Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.lucene.store; import java.io.Closeable; import java.io.FileNotFoundException; import java.io.IOException; import java.nio.file.AccessDeniedException; import java.nio.file.FileAlreadyExistsException; import java.nio.file.NoSuchFileException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.IdentityHashMap; import java.util.Iterator; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Random; import java.util.Set; import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicInteger; import java.util.regex.Matcher; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.NoDeletionPolicy; import org.apache.lucene.index.SegmentInfos; import org.apache.lucene.util.CollectionUtil; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestUtil; import org.apache.lucene.util.ThrottledIndexOutput; /** * This is a Directory Wrapper that adds methods * intended to be used only by unit tests. * It also adds a number of features useful for testing: * <ul> * <li> Instances created by {@link LuceneTestCase#newDirectory()} are tracked * to ensure they are closed by the test. * <li> When a MockDirectoryWrapper is closed, it will throw an exception if * it has any open files against it (with a stacktrace indicating where * they were opened from). * <li> When a MockDirectoryWrapper is closed, it runs CheckIndex to test if * the index was corrupted. * <li> MockDirectoryWrapper simulates some "features" of Windows, such as * refusing to write/delete to open files. * </ul> */ public class MockDirectoryWrapper extends BaseDirectoryWrapper { long maxSize; // Max actual bytes used. This is set by MockRAMOutputStream: long maxUsedSize; double randomIOExceptionRate; double randomIOExceptionRateOnOpen; Random randomState; boolean assertNoDeleteOpenFile = false; boolean trackDiskUsage = false; boolean useSlowOpenClosers = LuceneTestCase.TEST_NIGHTLY; boolean allowRandomFileNotFoundException = true; boolean allowReadingFilesStillOpenForWrite = false; private Set<String> unSyncedFiles; private Set<String> createdFiles; private Set<String> openFilesForWrite = new HashSet<>(); ConcurrentMap<String, RuntimeException> openLocks = new ConcurrentHashMap<>(); volatile boolean crashed; private ThrottledIndexOutput throttledOutput; private Throttling throttling = LuceneTestCase.TEST_NIGHTLY ? Throttling.SOMETIMES : Throttling.NEVER; // for testing boolean alwaysCorrupt; final AtomicInteger inputCloneCount = new AtomicInteger(); // use this for tracking files for crash. // additionally: provides debugging information in case you leave one open private Map<Closeable, Exception> openFileHandles = Collections .synchronizedMap(new IdentityHashMap<Closeable, Exception>()); // NOTE: we cannot initialize the Map here due to the // order in which our constructor actually does this // member initialization vs when it calls super. It seems // like super is called, then our members are initialized: private Map<String, Integer> openFiles; // Only tracked if noDeleteOpenFile is true: if an attempt // is made to delete an open file, we enroll it here. private Set<String> openFilesDeleted; private synchronized void init() { if (openFiles == null) { openFiles = new HashMap<>(); openFilesDeleted = new HashSet<>(); } if (createdFiles == null) createdFiles = new HashSet<>(); if (unSyncedFiles == null) unSyncedFiles = new HashSet<>(); } public MockDirectoryWrapper(Random random, Directory delegate) { super(delegate); // must make a private random since our methods are // called from different threads; else test failures may // not be reproducible from the original seed this.randomState = new Random(random.nextInt()); this.throttledOutput = new ThrottledIndexOutput( ThrottledIndexOutput.mBitsToBytes(40 + randomState.nextInt(10)), 1 + randomState.nextInt(5), null); init(); } public int getInputCloneCount() { return inputCloneCount.get(); } boolean verboseClone; /** * If set to true, we print a fake exception * with filename and stacktrace on every indexinput clone() */ public void setVerboseClone(boolean v) { verboseClone = v; } public void setTrackDiskUsage(boolean v) { trackDiskUsage = v; } /** If set to true (the default), when we throw random * IOException on openInput or createOutput, we may * sometimes throw FileNotFoundException or * NoSuchFileException. */ public void setAllowRandomFileNotFoundException(boolean value) { allowRandomFileNotFoundException = value; } /** If set to true, you can open an inputstream on a file * that is still open for writes. */ public void setAllowReadingFilesStillOpenForWrite(boolean value) { allowReadingFilesStillOpenForWrite = value; } /** * Enum for controlling hard disk throttling. * Set via {@link MockDirectoryWrapper #setThrottling(Throttling)} * <p> * WARNING: can make tests very slow. */ public static enum Throttling { /** always emulate a slow hard disk. could be very slow! */ ALWAYS, /** sometimes (0.5% of the time) emulate a slow hard disk. */ SOMETIMES, /** never throttle output */ NEVER } public void setThrottling(Throttling throttling) { this.throttling = throttling; } /** * Add a rare small sleep to catch race conditions in open/close * <p> * You can enable this if you need it. */ public void setUseSlowOpenClosers(boolean v) { useSlowOpenClosers = v; } @Override public synchronized void sync(Collection<String> names) throws IOException { maybeYield(); maybeThrowDeterministicException(); if (crashed) { throw new IOException("cannot sync after crash"); } // always pass thru fsync, directories rely on this. // 90% of time, we use DisableFsyncFS which omits the real calls. for (String name : names) { // randomly fail with IOE on any file maybeThrowIOException(name); in.sync(Collections.singleton(name)); unSyncedFiles.remove(name); } } @Override public synchronized void rename(String source, String dest) throws IOException { maybeYield(); maybeThrowDeterministicException(); if (crashed) { throw new IOException("cannot rename after crash"); } if (openFiles.containsKey(source) && assertNoDeleteOpenFile) { throw fillOpenTrace( new AssertionError( "MockDirectoryWrapper: source file \"" + source + "\" is still open: cannot rename"), source, true); } if (openFiles.containsKey(dest) && assertNoDeleteOpenFile) { throw fillOpenTrace( new AssertionError( "MockDirectoryWrapper: dest file \"" + dest + "\" is still open: cannot rename"), dest, true); } boolean success = false; try { in.rename(source, dest); success = true; } finally { if (success) { // we don't do this stuff with lucene's commit, but it's just for completeness if (unSyncedFiles.contains(source)) { unSyncedFiles.remove(source); unSyncedFiles.add(dest); } openFilesDeleted.remove(source); createdFiles.remove(source); createdFiles.add(dest); } } } @Override public synchronized void syncMetaData() throws IOException { maybeYield(); maybeThrowDeterministicException(); if (crashed) { throw new IOException("cannot rename after crash"); } in.syncMetaData(); } public synchronized final long sizeInBytes() throws IOException { if (in instanceof RAMDirectory) return ((RAMDirectory) in).ramBytesUsed(); else { // hack long size = 0; for (String file : in.listAll()) { // hack 2: see TODO in ExtrasFS (ideally it would always return 0 byte // size for extras it creates, even though the size of non-regular files is not defined) if (!file.startsWith("extra")) { size += in.fileLength(file); } } return size; } } public synchronized void corruptUnknownFiles() throws IOException { System.out.println("MDW: corrupt unknown files"); Set<String> knownFiles = new HashSet<>(); for (String fileName : listAll()) { if (fileName.startsWith(IndexFileNames.SEGMENTS)) { System.out.println("MDW: read " + fileName + " to gather files it references"); SegmentInfos infos; try { infos = SegmentInfos.readCommit(this, fileName); } catch (IOException ioe) { System.out.println("MDW: exception reading segment infos " + fileName + "; files: " + Arrays.toString(listAll())); throw ioe; } knownFiles.addAll(infos.files(true)); } } Set<String> toCorrupt = new HashSet<>(); Matcher m = IndexFileNames.CODEC_FILE_PATTERN.matcher(""); for (String fileName : listAll()) { m.reset(fileName); if (knownFiles.contains(fileName) == false && fileName.endsWith("write.lock") == false && (m.matches() || fileName.startsWith(IndexFileNames.PENDING_SEGMENTS))) { toCorrupt.add(fileName); } } corruptFiles(toCorrupt); } public synchronized void corruptFiles(Collection<String> files) throws IOException { boolean disabled = TestUtil.disableVirusChecker(in); try { _corruptFiles(files); } finally { if (disabled) { TestUtil.enableVirusChecker(in); } } } private synchronized void _corruptFiles(Collection<String> files) throws IOException { // TODO: we should also mess with any recent file renames, file deletions, if // syncMetaData was not called!! // Must make a copy because we change the incoming unsyncedFiles // when we create temp files, delete, etc., below: final List<String> filesToCorrupt = new ArrayList<>(files); // sort the files otherwise we have reproducibility issues // across JVMs if the incoming collection is a hashSet etc. CollectionUtil.timSort(filesToCorrupt); for (String name : filesToCorrupt) { int damage = randomState.nextInt(6); if (alwaysCorrupt && damage == 3) { damage = 4; } String action = null; switch (damage) { case 0: action = "deleted"; deleteFile(name); break; case 1: action = "zeroed"; // Zero out file entirely long length; try { length = fileLength(name); } catch (IOException ioe) { throw new RuntimeException("hit unexpected IOException while trying to corrupt file " + name, ioe); } // Delete original and write zeros back: deleteFile(name); byte[] zeroes = new byte[256]; long upto = 0; try (IndexOutput out = in.createOutput(name, LuceneTestCase.newIOContext(randomState))) { while (upto < length) { final int limit = (int) Math.min(length - upto, zeroes.length); out.writeBytes(zeroes, 0, limit); upto += limit; } } catch (IOException ioe) { throw new RuntimeException("hit unexpected IOException while trying to corrupt file " + name, ioe); } break; case 2: { action = "partially truncated"; // Partially Truncate the file: // First, make temp file and copy only half this // file over: String tempFileName = null; try (IndexOutput tempOut = in.createTempOutput("name", "mdw_corrupt", LuceneTestCase.newIOContext(randomState)); IndexInput ii = in.openInput(name, LuceneTestCase.newIOContext(randomState))) { tempFileName = tempOut.getName(); tempOut.copyBytes(ii, ii.length() / 2); } catch (IOException ioe) { throw new RuntimeException("hit unexpected IOException while trying to corrupt file " + name, ioe); } // Delete original and copy bytes back: deleteFile(name); try (IndexOutput out = in.createOutput(name, LuceneTestCase.newIOContext(randomState)); IndexInput ii = in.openInput(tempFileName, LuceneTestCase.newIOContext(randomState))) { out.copyBytes(ii, ii.length()); } catch (IOException ioe) { throw new RuntimeException("hit unexpected IOException while trying to corrupt file " + name, ioe); } deleteFile(tempFileName); } break; case 3: // The file survived intact: action = "didn't change"; break; case 4: // Corrupt one bit randomly in the file: { String tempFileName = null; try (IndexOutput tempOut = in.createTempOutput("name", "mdw_corrupt", LuceneTestCase.newIOContext(randomState)); IndexInput ii = in.openInput(name, LuceneTestCase.newIOContext(randomState))) { tempFileName = tempOut.getName(); if (ii.length() > 0) { // Copy first part unchanged: long byteToCorrupt = (long) (randomState.nextDouble() * ii.length()); if (byteToCorrupt > 0) { tempOut.copyBytes(ii, byteToCorrupt); } // Randomly flip one bit from this byte: byte b = ii.readByte(); int bitToFlip = randomState.nextInt(8); b = (byte) (b ^ (1 << bitToFlip)); tempOut.writeByte(b); action = "flip bit " + bitToFlip + " of byte " + byteToCorrupt + " out of " + ii.length() + " bytes"; // Copy last part unchanged: long bytesLeft = ii.length() - byteToCorrupt - 1; if (bytesLeft > 0) { tempOut.copyBytes(ii, bytesLeft); } } else { action = "didn't change"; } } catch (IOException ioe) { throw new RuntimeException("hit unexpected IOException while trying to corrupt file " + name, ioe); } // Delete original and copy bytes back: deleteFile(name); try (IndexOutput out = in.createOutput(name, LuceneTestCase.newIOContext(randomState)); IndexInput ii = in.openInput(tempFileName, LuceneTestCase.newIOContext(randomState))) { out.copyBytes(ii, ii.length()); } catch (IOException ioe) { throw new RuntimeException("hit unexpected IOException while trying to corrupt file " + name, ioe); } deleteFile(tempFileName); } break; case 5: action = "fully truncated"; // Totally truncate the file to zero bytes deleteFile(name); try (IndexOutput out = in.createOutput(name, LuceneTestCase.newIOContext(randomState))) { out.getFilePointer(); // just fake access to prevent compiler warning } catch (IOException ioe) { throw new RuntimeException("hit unexpected IOException while trying to corrupt file " + name, ioe); } break; default: throw new AssertionError(); } if (LuceneTestCase.VERBOSE) { System.out.println("MockDirectoryWrapper: " + action + " unsynced file: " + name); } } } /** Simulates a crash of OS or machine by overwriting * unsynced files. */ public synchronized void crash() throws IOException { openFiles = new HashMap<>(); openFilesForWrite = new HashSet<>(); openFilesDeleted = new HashSet<>(); // first force-close all files, so we can corrupt on windows etc. // clone the file map, as these guys want to remove themselves on close. Map<Closeable, Exception> m = new IdentityHashMap<>(openFileHandles); for (Closeable f : m.keySet()) { try { f.close(); } catch (Exception ignored) { } } corruptFiles(unSyncedFiles); crashed = true; unSyncedFiles = new HashSet<>(); } public synchronized void clearCrash() { crashed = false; openLocks.clear(); } public void setMaxSizeInBytes(long maxSize) { this.maxSize = maxSize; } public long getMaxSizeInBytes() { return this.maxSize; } /** * Returns the peek actual storage used (bytes) in this * directory. */ public long getMaxUsedSizeInBytes() { return this.maxUsedSize; } public void resetMaxUsedSizeInBytes() throws IOException { this.maxUsedSize = getRecomputedActualSizeInBytes(); } /** * Trip a test assert if there is an attempt * to delete an open file. */ public void setAssertNoDeleteOpenFile(boolean value) { this.assertNoDeleteOpenFile = value; } public boolean getAssertNoDeleteOpenFile() { return assertNoDeleteOpenFile; } /** * If 0.0, no exceptions will be thrown. Else this should * be a double 0.0 - 1.0. We will randomly throw an * IOException on the first write to an OutputStream based * on this probability. */ public void setRandomIOExceptionRate(double rate) { randomIOExceptionRate = rate; } public double getRandomIOExceptionRate() { return randomIOExceptionRate; } /** * If 0.0, no exceptions will be thrown during openInput * and createOutput. Else this should * be a double 0.0 - 1.0 and we will randomly throw an * IOException in openInput and createOutput with * this probability. */ public void setRandomIOExceptionRateOnOpen(double rate) { randomIOExceptionRateOnOpen = rate; } public double getRandomIOExceptionRateOnOpen() { return randomIOExceptionRateOnOpen; } void maybeThrowIOException(String message) throws IOException { if (randomState.nextDouble() < randomIOExceptionRate) { IOException ioe = new IOException( "a random IOException" + (message == null ? "" : " (" + message + ")")); if (LuceneTestCase.VERBOSE) { System.out.println( Thread.currentThread().getName() + ": MockDirectoryWrapper: now throw random exception" + (message == null ? "" : " (" + message + ")")); ioe.printStackTrace(System.out); } throw ioe; } } void maybeThrowIOExceptionOnOpen(String name) throws IOException { if (randomState.nextDouble() < randomIOExceptionRateOnOpen) { if (LuceneTestCase.VERBOSE) { System.out.println(Thread.currentThread().getName() + ": MockDirectoryWrapper: now throw random exception during open file=" + name); new Throwable().printStackTrace(System.out); } if (allowRandomFileNotFoundException == false || randomState.nextBoolean()) { throw new IOException("a random IOException (" + name + ")"); } else { throw randomState.nextBoolean() ? new FileNotFoundException("a random IOException (" + name + ")") : new NoSuchFileException("a random IOException (" + name + ")"); } } } /** returns current open file handle count */ public synchronized long getFileHandleCount() { return openFileHandles.size(); } @Override public synchronized void deleteFile(String name) throws IOException { maybeYield(); maybeThrowDeterministicException(); if (crashed) { throw new IOException("cannot delete after crash"); } if (openFiles.containsKey(name)) { openFilesDeleted.add(name); if (assertNoDeleteOpenFile) { throw fillOpenTrace( new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open: cannot delete"), name, true); } } else { openFilesDeleted.remove(name); } unSyncedFiles.remove(name); in.deleteFile(name); createdFiles.remove(name); } // sets the cause of the incoming ioe to be the stack // trace when the offending file name was opened private synchronized <T extends Throwable> T fillOpenTrace(T t, String name, boolean input) { for (Map.Entry<Closeable, Exception> ent : openFileHandles.entrySet()) { if (input && ent.getKey() instanceof MockIndexInputWrapper && ((MockIndexInputWrapper) ent.getKey()).name.equals(name)) { t.initCause(ent.getValue()); break; } else if (!input && ent.getKey() instanceof MockIndexOutputWrapper && ((MockIndexOutputWrapper) ent.getKey()).name.equals(name)) { t.initCause(ent.getValue()); break; } } return t; } private void maybeYield() { if (randomState.nextBoolean()) { Thread.yield(); } } public synchronized Set<String> getOpenDeletedFiles() { return new HashSet<>(openFilesDeleted); } private boolean failOnCreateOutput = true; public void setFailOnCreateOutput(boolean v) { failOnCreateOutput = v; } @Override public synchronized IndexOutput createOutput(String name, IOContext context) throws IOException { maybeThrowDeterministicException(); maybeThrowIOExceptionOnOpen(name); maybeYield(); if (failOnCreateOutput) { maybeThrowDeterministicException(); } if (crashed) { throw new IOException("cannot createOutput after crash"); } init(); if (createdFiles.contains(name)) { throw new FileAlreadyExistsException("File \"" + name + "\" was already written to."); } if (assertNoDeleteOpenFile && openFiles.containsKey(name)) { throw new AssertionError("MockDirectoryWrapper: file \"" + name + "\" is still open: cannot overwrite"); } unSyncedFiles.add(name); createdFiles.add(name); //System.out.println(Thread.currentThread().getName() + ": MDW: create " + name); IndexOutput delegateOutput = in.createOutput(name, LuceneTestCase.newIOContext(randomState, context)); final IndexOutput io = new MockIndexOutputWrapper(this, delegateOutput, name); addFileHandle(io, name, Handle.Output); openFilesForWrite.add(name); return maybeThrottle(name, io); } private IndexOutput maybeThrottle(String name, IndexOutput output) { // throttling REALLY slows down tests, so don't do it very often for SOMETIMES. if (throttling == Throttling.ALWAYS || (throttling == Throttling.SOMETIMES && randomState.nextInt(200) == 0)) { if (LuceneTestCase.VERBOSE) { System.out.println("MockDirectoryWrapper: throttling indexOutput (" + name + ")"); } return throttledOutput.newFromDelegate(output); } else { return output; } } @Override public synchronized IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException { maybeThrowDeterministicException(); maybeThrowIOExceptionOnOpen("temp: prefix=" + prefix + " suffix=" + suffix); maybeYield(); if (failOnCreateOutput) { maybeThrowDeterministicException(); } if (crashed) { throw new IOException("cannot createTempOutput after crash"); } init(); IndexOutput delegateOutput = in.createTempOutput(prefix, suffix, LuceneTestCase.newIOContext(randomState, context)); String name = delegateOutput.getName(); if (name.toLowerCase(Locale.ROOT).endsWith(".tmp") == false) { throw new IllegalStateException("wrapped directory failed to use .tmp extension: got: " + name); } unSyncedFiles.add(name); createdFiles.add(name); final IndexOutput io = new MockIndexOutputWrapper(this, delegateOutput, name); addFileHandle(io, name, Handle.Output); openFilesForWrite.add(name); return maybeThrottle(name, io); } private static enum Handle { Input, Output, Slice } synchronized void addFileHandle(Closeable c, String name, Handle handle) { Integer v = openFiles.get(name); if (v != null) { v = Integer.valueOf(v.intValue() + 1); openFiles.put(name, v); } else { openFiles.put(name, Integer.valueOf(1)); } openFileHandles.put(c, new RuntimeException("unclosed Index" + handle.name() + ": " + name)); } private boolean failOnOpenInput = true; public void setFailOnOpenInput(boolean v) { failOnOpenInput = v; } @Override public synchronized IndexInput openInput(String name, IOContext context) throws IOException { maybeThrowDeterministicException(); maybeThrowIOExceptionOnOpen(name); maybeYield(); if (failOnOpenInput) { maybeThrowDeterministicException(); } if (!LuceneTestCase.slowFileExists(in, name)) { throw randomState.nextBoolean() ? new FileNotFoundException(name + " in dir=" + in) : new NoSuchFileException(name + " in dir=" + in); } // cannot open a file for input if it's still open for output. if (!allowReadingFilesStillOpenForWrite && openFilesForWrite.contains(name)) { throw fillOpenTrace(new AccessDeniedException( "MockDirectoryWrapper: file \"" + name + "\" is still open for writing"), name, false); } IndexInput delegateInput = in.openInput(name, LuceneTestCase.newIOContext(randomState, context)); final IndexInput ii; int randomInt = randomState.nextInt(500); if (useSlowOpenClosers && randomInt == 0) { if (LuceneTestCase.VERBOSE) { System.out.println("MockDirectoryWrapper: using SlowClosingMockIndexInputWrapper for file " + name); } ii = new SlowClosingMockIndexInputWrapper(this, name, delegateInput); } else if (useSlowOpenClosers && randomInt == 1) { if (LuceneTestCase.VERBOSE) { System.out.println("MockDirectoryWrapper: using SlowOpeningMockIndexInputWrapper for file " + name); } ii = new SlowOpeningMockIndexInputWrapper(this, name, delegateInput); } else { ii = new MockIndexInputWrapper(this, name, delegateInput, null); } addFileHandle(ii, name, Handle.Input); return ii; } /** Provided for testing purposes. Use sizeInBytes() instead. */ public synchronized final long getRecomputedSizeInBytes() throws IOException { if (!(in instanceof RAMDirectory)) return sizeInBytes(); long size = 0; for (final RAMFile file : ((RAMDirectory) in).fileMap.values()) { size += file.ramBytesUsed(); } return size; } /** Like getRecomputedSizeInBytes(), but, uses actual file * lengths rather than buffer allocations (which are * quantized up to nearest * RAMOutputStream.BUFFER_SIZE (now 1024) bytes. */ public final synchronized long getRecomputedActualSizeInBytes() throws IOException { if (!(in instanceof RAMDirectory)) return sizeInBytes(); long size = 0; for (final RAMFile file : ((RAMDirectory) in).fileMap.values()) size += file.length; return size; } // NOTE: This is off by default; see LUCENE-5574 private volatile boolean assertNoUnreferencedFilesOnClose; public void setAssertNoUnrefencedFilesOnClose(boolean v) { assertNoUnreferencedFilesOnClose = v; } @Override public synchronized void close() throws IOException { if (isOpen) { isOpen = false; } else { in.close(); // but call it again on our wrapped dir return; } boolean success = false; try { // files that we tried to delete, but couldn't because readers were open. // all that matters is that we tried! (they will eventually go away) // still open when we tried to delete maybeYield(); if (openFiles == null) { openFiles = new HashMap<>(); openFilesDeleted = new HashSet<>(); } if (openFiles.size() > 0) { // print the first one as it's very verbose otherwise Exception cause = null; Iterator<Exception> stacktraces = openFileHandles.values().iterator(); if (stacktraces.hasNext()) { cause = stacktraces.next(); } // RuntimeException instead of IOException because // super() does not throw IOException currently: throw new RuntimeException("MockDirectoryWrapper: cannot close: there are still " + openFiles.size() + " open files: " + openFiles, cause); } if (openLocks.size() > 0) { Exception cause = null; Iterator<RuntimeException> stacktraces = openLocks.values().iterator(); if (stacktraces.hasNext()) { cause = stacktraces.next(); } throw new RuntimeException( "MockDirectoryWrapper: cannot close: there are still open locks: " + openLocks, cause); } randomIOExceptionRate = 0.0; randomIOExceptionRateOnOpen = 0.0; if ((getCheckIndexOnClose() || assertNoUnreferencedFilesOnClose) && DirectoryReader.indexExists(this)) { if (getCheckIndexOnClose()) { if (LuceneTestCase.VERBOSE) { System.out.println("\nNOTE: MockDirectoryWrapper: now crush"); } crash(); // corrupt any unsynced-files if (LuceneTestCase.VERBOSE) { System.out.println("\nNOTE: MockDirectoryWrapper: now run CheckIndex"); } TestUtil.checkIndex(this, getCrossCheckTermVectorsOnClose(), true, null); } // TODO: factor this out / share w/ TestIW.assertNoUnreferencedFiles if (assertNoUnreferencedFilesOnClose) { System.out.println("MDW: now assert no unref'd files at close"); // now look for unreferenced files: discount ones that we tried to delete but could not Set<String> allFiles = new HashSet<>(Arrays.asList(listAll())); String[] startFiles = allFiles.toArray(new String[0]); IndexWriterConfig iwc = new IndexWriterConfig(null); iwc.setIndexDeletionPolicy(NoDeletionPolicy.INSTANCE); // We must do this before opening writer otherwise writer will be angry if there are pending deletions: TestUtil.disableVirusChecker(in); new IndexWriter(in, iwc).rollback(); String[] endFiles = in.listAll(); Set<String> startSet = new TreeSet<>(Arrays.asList(startFiles)); Set<String> endSet = new TreeSet<>(Arrays.asList(endFiles)); startFiles = startSet.toArray(new String[0]); endFiles = endSet.toArray(new String[0]); if (!Arrays.equals(startFiles, endFiles)) { List<String> removed = new ArrayList<>(); for (String fileName : startFiles) { if (!endSet.contains(fileName)) { removed.add(fileName); } } List<String> added = new ArrayList<>(); for (String fileName : endFiles) { if (!startSet.contains(fileName)) { added.add(fileName); } } String extras; if (removed.size() != 0) { extras = "\n\nThese files were removed: " + removed; } else { extras = ""; } if (added.size() != 0) { extras += "\n\nThese files were added (waaaaaaaaaat!): " + added; } throw new RuntimeException( "unreferenced files: before delete:\n " + Arrays.toString(startFiles) + "\n after delete:\n " + Arrays.toString(endFiles) + extras); } DirectoryReader ir1 = DirectoryReader.open(this); int numDocs1 = ir1.numDocs(); ir1.close(); new IndexWriter(this, new IndexWriterConfig(null)).close(); DirectoryReader ir2 = DirectoryReader.open(this); int numDocs2 = ir2.numDocs(); ir2.close(); assert numDocs1 == numDocs2 : "numDocs changed after opening/closing IW: before=" + numDocs1 + " after=" + numDocs2; } } success = true; } finally { if (success) { IOUtils.close(in); } else { IOUtils.closeWhileHandlingException(in); } } } synchronized void removeOpenFile(Closeable c, String name) { Integer v = openFiles.get(name); // Could be null when crash() was called if (v != null) { if (v.intValue() == 1) { openFiles.remove(name); } else { v = Integer.valueOf(v.intValue() - 1); openFiles.put(name, v); } } openFileHandles.remove(c); } public synchronized void removeIndexOutput(IndexOutput out, String name) { openFilesForWrite.remove(name); removeOpenFile(out, name); } public synchronized void removeIndexInput(IndexInput in, String name) { removeOpenFile(in, name); } /** * Objects that represent fail-able conditions. Objects of a derived * class are created and registered with the mock directory. After * register, each object will be invoked once for each first write * of a file, giving the object a chance to throw an IOException. */ public static class Failure { /** * eval is called on the first write of every new file. */ public void eval(MockDirectoryWrapper dir) throws IOException { } /** * reset should set the state of the failure to its default * (freshly constructed) state. Reset is convenient for tests * that want to create one failure object and then reuse it in * multiple cases. This, combined with the fact that Failure * subclasses are often anonymous classes makes reset difficult to * do otherwise. * * A typical example of use is * Failure failure = new Failure() { ... }; * ... * mock.failOn(failure.reset()) */ public Failure reset() { return this; } protected boolean doFail; public void setDoFail() { doFail = true; } public void clearDoFail() { doFail = false; } } ArrayList<Failure> failures; /** * add a Failure object to the list of objects to be evaluated * at every potential failure point */ synchronized public void failOn(Failure fail) { if (failures == null) { failures = new ArrayList<>(); } failures.add(fail); } /** * Iterate through the failures list, giving each object a * chance to throw an IOE */ synchronized void maybeThrowDeterministicException() throws IOException { if (failures != null) { for (int i = 0; i < failures.size(); i++) { try { failures.get(i).eval(this); } catch (Throwable t) { if (LuceneTestCase.VERBOSE) { System.out.println("MockDirectoryWrapper: throw exc"); t.printStackTrace(System.out); } throw IOUtils.rethrowAlways(t); } } } } @Override public synchronized String[] listAll() throws IOException { maybeYield(); return in.listAll(); } @Override public synchronized long fileLength(String name) throws IOException { maybeYield(); return in.fileLength(name); } @Override public synchronized Lock obtainLock(String name) throws IOException { maybeYield(); return super.obtainLock(name); // TODO: consider mocking locks, but not all the time, can hide bugs } /** Use this when throwing fake {@code IOException}, * e.g. from {@link MockDirectoryWrapper.Failure}. */ public static class FakeIOException extends IOException { } @Override public String toString() { if (maxSize != 0) { return "MockDirectoryWrapper(" + in + ", current=" + maxUsedSize + ",max=" + maxSize + ")"; } else { return super.toString(); } } // don't override optional methods like copyFrom: we need the default impl for things like disk // full checks. we randomly exercise "raw" directories anyway. We ensure default impls are used: @Override public final ChecksumIndexInput openChecksumInput(String name, IOContext context) throws IOException { return super.openChecksumInput(name, context); } @Override public final void copyFrom(Directory from, String src, String dest, IOContext context) throws IOException { super.copyFrom(from, src, dest, context); } @Override protected final void ensureOpen() throws AlreadyClosedException { super.ensureOpen(); } }