List of usage examples for org.apache.hadoop.fs LocalFileSystem initialize
@Override
public void initialize(URI name, Configuration conf) throws IOException
From source file:org.apache.flink.contrib.streaming.state.RocksDBAsyncKVSnapshotTest.java
License:Apache License
/** * This ensures that asynchronous state handles are actually materialized asynchonously. * * <p>We use latches to block at various stages and see if the code still continues through * the parts that are not asynchronous. If the checkpoint is not done asynchronously the * test will simply lock forever.//from w w w.j a v a 2 s .co m */ @Test public void testAsyncCheckpoints() throws Exception { LocalFileSystem localFS = new LocalFileSystem(); localFS.initialize(new URI("file:///"), new Configuration()); PowerMockito.stub(PowerMockito.method(FileSystem.class, "get", URI.class, Configuration.class)) .toReturn(localFS); final OneShotLatch delayCheckpointLatch = new OneShotLatch(); final OneShotLatch ensureCheckpointLatch = new OneShotLatch(); final OneInputStreamTask<String, String> task = new OneInputStreamTask<>(); final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(task, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO); testHarness.configureForKeyedStream(new KeySelector<String, String>() { @Override public String getKey(String value) throws Exception { return value; } }, BasicTypeInfo.STRING_TYPE_INFO); StreamConfig streamConfig = testHarness.getStreamConfig(); File dbDir = new File(new File(ConfigConstants.DEFAULT_TASK_MANAGER_TMP_PATH, UUID.randomUUID().toString()), "state"); File chkDir = new File( new File(ConfigConstants.DEFAULT_TASK_MANAGER_TMP_PATH, UUID.randomUUID().toString()), "snapshots"); RocksDBStateBackend backend = new RocksDBStateBackend(chkDir.getAbsoluteFile().toURI(), new MemoryStateBackend()); backend.setDbStoragePath(dbDir.getAbsolutePath()); streamConfig.setStateBackend(backend); streamConfig.setStreamOperator(new AsyncCheckpointOperator()); StreamMockEnvironment mockEnv = new StreamMockEnvironment(testHarness.jobConfig, testHarness.taskConfig, testHarness.memorySize, new MockInputSplitProvider(), testHarness.bufferSize) { @Override public void acknowledgeCheckpoint(long checkpointId) { super.acknowledgeCheckpoint(checkpointId); } @Override public void acknowledgeCheckpoint(long checkpointId, StateHandle<?> state) { super.acknowledgeCheckpoint(checkpointId, state); // block on the latch, to verify that triggerCheckpoint returns below, // even though the async checkpoint would not finish try { delayCheckpointLatch.await(); } catch (InterruptedException e) { e.printStackTrace(); } assertTrue(state instanceof StreamTaskStateList); StreamTaskStateList stateList = (StreamTaskStateList) state; // should be only one k/v state StreamTaskState taskState = stateList.getState(this.getUserClassLoader())[0]; assertEquals(1, taskState.getKvStates().size()); assertTrue(taskState.getKvStates() .get("count") instanceof AbstractRocksDBState.AbstractRocksDBSnapshot); // we now know that the checkpoint went through ensureCheckpointLatch.trigger(); } }; testHarness.invoke(mockEnv); // wait for the task to be running for (Field field : StreamTask.class.getDeclaredFields()) { if (field.getName().equals("isRunning")) { field.setAccessible(true); while (!field.getBoolean(task)) { Thread.sleep(10); } } } testHarness.processElement(new StreamRecord<>("Wohoo", 0)); task.triggerCheckpoint(42, 17); // now we allow the checkpoint delayCheckpointLatch.trigger(); // wait for the checkpoint to go through ensureCheckpointLatch.await(); testHarness.endInput(); testHarness.waitForTaskCompletion(); }
From source file:org.apache.flink.contrib.streaming.state.RocksDBAsyncSnapshotTest.java
License:Apache License
/** * This ensures that asynchronous state handles are actually materialized asynchronously. * * <p>We use latches to block at various stages and see if the code still continues through * the parts that are not asynchronous. If the checkpoint is not done asynchronously the * test will simply lock forever./*from w ww . j a v a 2s.c o m*/ */ @Test public void testFullyAsyncSnapshot() throws Exception { LocalFileSystem localFS = new LocalFileSystem(); localFS.initialize(new URI("file:///"), new Configuration()); PowerMockito.stub(PowerMockito.method(FileSystem.class, "get", URI.class, Configuration.class)) .toReturn(localFS); final OneInputStreamTask<String, String> task = new OneInputStreamTask<>(); final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(task, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO); testHarness.setupOutputForSingletonOperatorChain(); testHarness.configureForKeyedStream(new KeySelector<String, String>() { @Override public String getKey(String value) throws Exception { return value; } }, BasicTypeInfo.STRING_TYPE_INFO); StreamConfig streamConfig = testHarness.getStreamConfig(); File dbDir = new File(new File(ConfigConstants.DEFAULT_TASK_MANAGER_TMP_PATH, UUID.randomUUID().toString()), "state"); RocksDBStateBackend backend = new RocksDBStateBackend(new MemoryStateBackend()); backend.setDbStoragePath(dbDir.getAbsolutePath()); streamConfig.setStateBackend(backend); streamConfig.setStreamOperator(new AsyncCheckpointOperator()); final OneShotLatch delayCheckpointLatch = new OneShotLatch(); final OneShotLatch ensureCheckpointLatch = new OneShotLatch(); StreamMockEnvironment mockEnv = new StreamMockEnvironment(testHarness.jobConfig, testHarness.taskConfig, testHarness.memorySize, new MockInputSplitProvider(), testHarness.bufferSize) { @Override public void acknowledgeCheckpoint(long checkpointId, CheckpointMetrics checkpointMetrics, SubtaskState checkpointStateHandles) { super.acknowledgeCheckpoint(checkpointId, checkpointMetrics); // block on the latch, to verify that triggerCheckpoint returns below, // even though the async checkpoint would not finish try { delayCheckpointLatch.await(); } catch (InterruptedException e) { throw new RuntimeException(e); } // should be one k/v state assertNotNull(checkpointStateHandles.getManagedKeyedState()); // we now know that the checkpoint went through ensureCheckpointLatch.trigger(); } }; testHarness.invoke(mockEnv); // wait for the task to be running for (Field field : StreamTask.class.getDeclaredFields()) { if (field.getName().equals("isRunning")) { field.setAccessible(true); while (!field.getBoolean(task)) { Thread.sleep(10); } } } task.triggerCheckpoint(new CheckpointMetaData(42, 17), CheckpointOptions.forFullCheckpoint()); testHarness.processElement(new StreamRecord<>("Wohoo", 0)); // now we allow the checkpoint delayCheckpointLatch.trigger(); // wait for the checkpoint to go through ensureCheckpointLatch.await(); testHarness.endInput(); ExecutorService threadPool = task.getAsyncOperationsThreadPool(); threadPool.shutdown(); Assert.assertTrue(threadPool.awaitTermination(60_000, TimeUnit.MILLISECONDS)); testHarness.waitForTaskCompletion(); if (mockEnv.wasFailedExternally()) { fail("Unexpected exception during execution."); } }
From source file:org.apache.flink.contrib.streaming.state.RocksDBAsyncSnapshotTest.java
License:Apache License
/** * This tests ensures that canceling of asynchronous snapshots works as expected and does not block. * @throws Exception//from ww w . ja v a2 s . c o m */ @Test @Ignore public void testCancelFullyAsyncCheckpoints() throws Exception { LocalFileSystem localFS = new LocalFileSystem(); localFS.initialize(new URI("file:///"), new Configuration()); PowerMockito.stub(PowerMockito.method(FileSystem.class, "get", URI.class, Configuration.class)) .toReturn(localFS); final OneInputStreamTask<String, String> task = new OneInputStreamTask<>(); final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(task, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO); testHarness.setupOutputForSingletonOperatorChain(); testHarness.configureForKeyedStream(new KeySelector<String, String>() { @Override public String getKey(String value) throws Exception { return value; } }, BasicTypeInfo.STRING_TYPE_INFO); StreamConfig streamConfig = testHarness.getStreamConfig(); File dbDir = new File(new File(ConfigConstants.DEFAULT_TASK_MANAGER_TMP_PATH, UUID.randomUUID().toString()), "state"); BlockingStreamMemoryStateBackend memoryStateBackend = new BlockingStreamMemoryStateBackend(); RocksDBStateBackend backend = new RocksDBStateBackend(memoryStateBackend); backend.setDbStoragePath(dbDir.getAbsolutePath()); streamConfig.setStateBackend(backend); streamConfig.setStreamOperator(new AsyncCheckpointOperator()); StreamMockEnvironment mockEnv = new StreamMockEnvironment(testHarness.jobConfig, testHarness.taskConfig, testHarness.memorySize, new MockInputSplitProvider(), testHarness.bufferSize); BlockingStreamMemoryStateBackend.waitFirstWriteLatch = new OneShotLatch(); BlockingStreamMemoryStateBackend.unblockCancelLatch = new OneShotLatch(); testHarness.invoke(mockEnv); // wait for the task to be running for (Field field : StreamTask.class.getDeclaredFields()) { if (field.getName().equals("isRunning")) { field.setAccessible(true); while (!field.getBoolean(task)) { Thread.sleep(10); } } } task.triggerCheckpoint(new CheckpointMetaData(42, 17), CheckpointOptions.forFullCheckpoint()); testHarness.processElement(new StreamRecord<>("Wohoo", 0)); BlockingStreamMemoryStateBackend.waitFirstWriteLatch.await(); task.cancel(); BlockingStreamMemoryStateBackend.unblockCancelLatch.trigger(); testHarness.endInput(); try { ExecutorService threadPool = task.getAsyncOperationsThreadPool(); threadPool.shutdown(); Assert.assertTrue(threadPool.awaitTermination(60_000, TimeUnit.MILLISECONDS)); testHarness.waitForTaskCompletion(); if (mockEnv.wasFailedExternally()) { throw new AsynchronousException(new InterruptedException("Exception was thrown as expected.")); } fail("Operation completed. Cancel failed."); } catch (Exception expected) { AsynchronousException asynchronousException = null; if (expected instanceof AsynchronousException) { asynchronousException = (AsynchronousException) expected; } else if (expected.getCause() instanceof AsynchronousException) { asynchronousException = (AsynchronousException) expected.getCause(); } else { fail("Unexpected exception: " + expected); } // we expect the exception from canceling snapshots Throwable innerCause = asynchronousException.getCause(); Assert.assertTrue("Unexpected inner cause: " + innerCause, innerCause instanceof CancellationException //future canceled || innerCause instanceof InterruptedException); //thread interrupted } }