List of usage examples for org.apache.hadoop.fs LocalFileSystem LocalFileSystem
public LocalFileSystem()
From source file:org.apache.falcon.hadoop.JailedFileSystem.java
License:Apache License
public JailedFileSystem() { localFS = new LocalFileSystem(); this.workingDir = new Path("/user", System.getProperty("user.name")); }
From source file:org.apache.flink.contrib.streaming.state.RocksDBAsyncKVSnapshotTest.java
License:Apache License
/** * This ensures that asynchronous state handles are actually materialized asynchonously. * * <p>We use latches to block at various stages and see if the code still continues through * the parts that are not asynchronous. If the checkpoint is not done asynchronously the * test will simply lock forever.//from w w w . ja v a 2s.co m */ @Test public void testAsyncCheckpoints() throws Exception { LocalFileSystem localFS = new LocalFileSystem(); localFS.initialize(new URI("file:///"), new Configuration()); PowerMockito.stub(PowerMockito.method(FileSystem.class, "get", URI.class, Configuration.class)) .toReturn(localFS); final OneShotLatch delayCheckpointLatch = new OneShotLatch(); final OneShotLatch ensureCheckpointLatch = new OneShotLatch(); final OneInputStreamTask<String, String> task = new OneInputStreamTask<>(); final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(task, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO); testHarness.configureForKeyedStream(new KeySelector<String, String>() { @Override public String getKey(String value) throws Exception { return value; } }, BasicTypeInfo.STRING_TYPE_INFO); StreamConfig streamConfig = testHarness.getStreamConfig(); File dbDir = new File(new File(ConfigConstants.DEFAULT_TASK_MANAGER_TMP_PATH, UUID.randomUUID().toString()), "state"); File chkDir = new File( new File(ConfigConstants.DEFAULT_TASK_MANAGER_TMP_PATH, UUID.randomUUID().toString()), "snapshots"); RocksDBStateBackend backend = new RocksDBStateBackend(chkDir.getAbsoluteFile().toURI(), new MemoryStateBackend()); backend.setDbStoragePath(dbDir.getAbsolutePath()); streamConfig.setStateBackend(backend); streamConfig.setStreamOperator(new AsyncCheckpointOperator()); StreamMockEnvironment mockEnv = new StreamMockEnvironment(testHarness.jobConfig, testHarness.taskConfig, testHarness.memorySize, new MockInputSplitProvider(), testHarness.bufferSize) { @Override public void acknowledgeCheckpoint(long checkpointId) { super.acknowledgeCheckpoint(checkpointId); } @Override public void acknowledgeCheckpoint(long checkpointId, StateHandle<?> state) { super.acknowledgeCheckpoint(checkpointId, state); // block on the latch, to verify that triggerCheckpoint returns below, // even though the async checkpoint would not finish try { delayCheckpointLatch.await(); } catch (InterruptedException e) { e.printStackTrace(); } assertTrue(state instanceof StreamTaskStateList); StreamTaskStateList stateList = (StreamTaskStateList) state; // should be only one k/v state StreamTaskState taskState = stateList.getState(this.getUserClassLoader())[0]; assertEquals(1, taskState.getKvStates().size()); assertTrue(taskState.getKvStates() .get("count") instanceof AbstractRocksDBState.AbstractRocksDBSnapshot); // we now know that the checkpoint went through ensureCheckpointLatch.trigger(); } }; testHarness.invoke(mockEnv); // wait for the task to be running for (Field field : StreamTask.class.getDeclaredFields()) { if (field.getName().equals("isRunning")) { field.setAccessible(true); while (!field.getBoolean(task)) { Thread.sleep(10); } } } testHarness.processElement(new StreamRecord<>("Wohoo", 0)); task.triggerCheckpoint(42, 17); // now we allow the checkpoint delayCheckpointLatch.trigger(); // wait for the checkpoint to go through ensureCheckpointLatch.await(); testHarness.endInput(); testHarness.waitForTaskCompletion(); }
From source file:org.apache.flink.contrib.streaming.state.RocksDBAsyncSnapshotTest.java
License:Apache License
/** * This ensures that asynchronous state handles are actually materialized asynchronously. * * <p>We use latches to block at various stages and see if the code still continues through * the parts that are not asynchronous. If the checkpoint is not done asynchronously the * test will simply lock forever.//w w w . j ava 2 s. c o m */ @Test public void testFullyAsyncSnapshot() throws Exception { LocalFileSystem localFS = new LocalFileSystem(); localFS.initialize(new URI("file:///"), new Configuration()); PowerMockito.stub(PowerMockito.method(FileSystem.class, "get", URI.class, Configuration.class)) .toReturn(localFS); final OneInputStreamTask<String, String> task = new OneInputStreamTask<>(); final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(task, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO); testHarness.setupOutputForSingletonOperatorChain(); testHarness.configureForKeyedStream(new KeySelector<String, String>() { @Override public String getKey(String value) throws Exception { return value; } }, BasicTypeInfo.STRING_TYPE_INFO); StreamConfig streamConfig = testHarness.getStreamConfig(); File dbDir = new File(new File(ConfigConstants.DEFAULT_TASK_MANAGER_TMP_PATH, UUID.randomUUID().toString()), "state"); RocksDBStateBackend backend = new RocksDBStateBackend(new MemoryStateBackend()); backend.setDbStoragePath(dbDir.getAbsolutePath()); streamConfig.setStateBackend(backend); streamConfig.setStreamOperator(new AsyncCheckpointOperator()); final OneShotLatch delayCheckpointLatch = new OneShotLatch(); final OneShotLatch ensureCheckpointLatch = new OneShotLatch(); StreamMockEnvironment mockEnv = new StreamMockEnvironment(testHarness.jobConfig, testHarness.taskConfig, testHarness.memorySize, new MockInputSplitProvider(), testHarness.bufferSize) { @Override public void acknowledgeCheckpoint(long checkpointId, CheckpointMetrics checkpointMetrics, SubtaskState checkpointStateHandles) { super.acknowledgeCheckpoint(checkpointId, checkpointMetrics); // block on the latch, to verify that triggerCheckpoint returns below, // even though the async checkpoint would not finish try { delayCheckpointLatch.await(); } catch (InterruptedException e) { throw new RuntimeException(e); } // should be one k/v state assertNotNull(checkpointStateHandles.getManagedKeyedState()); // we now know that the checkpoint went through ensureCheckpointLatch.trigger(); } }; testHarness.invoke(mockEnv); // wait for the task to be running for (Field field : StreamTask.class.getDeclaredFields()) { if (field.getName().equals("isRunning")) { field.setAccessible(true); while (!field.getBoolean(task)) { Thread.sleep(10); } } } task.triggerCheckpoint(new CheckpointMetaData(42, 17), CheckpointOptions.forFullCheckpoint()); testHarness.processElement(new StreamRecord<>("Wohoo", 0)); // now we allow the checkpoint delayCheckpointLatch.trigger(); // wait for the checkpoint to go through ensureCheckpointLatch.await(); testHarness.endInput(); ExecutorService threadPool = task.getAsyncOperationsThreadPool(); threadPool.shutdown(); Assert.assertTrue(threadPool.awaitTermination(60_000, TimeUnit.MILLISECONDS)); testHarness.waitForTaskCompletion(); if (mockEnv.wasFailedExternally()) { fail("Unexpected exception during execution."); } }
From source file:org.apache.flink.contrib.streaming.state.RocksDBAsyncSnapshotTest.java
License:Apache License
/** * This tests ensures that canceling of asynchronous snapshots works as expected and does not block. * @throws Exception// ww w .j av a2s .c o m */ @Test @Ignore public void testCancelFullyAsyncCheckpoints() throws Exception { LocalFileSystem localFS = new LocalFileSystem(); localFS.initialize(new URI("file:///"), new Configuration()); PowerMockito.stub(PowerMockito.method(FileSystem.class, "get", URI.class, Configuration.class)) .toReturn(localFS); final OneInputStreamTask<String, String> task = new OneInputStreamTask<>(); final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(task, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO); testHarness.setupOutputForSingletonOperatorChain(); testHarness.configureForKeyedStream(new KeySelector<String, String>() { @Override public String getKey(String value) throws Exception { return value; } }, BasicTypeInfo.STRING_TYPE_INFO); StreamConfig streamConfig = testHarness.getStreamConfig(); File dbDir = new File(new File(ConfigConstants.DEFAULT_TASK_MANAGER_TMP_PATH, UUID.randomUUID().toString()), "state"); BlockingStreamMemoryStateBackend memoryStateBackend = new BlockingStreamMemoryStateBackend(); RocksDBStateBackend backend = new RocksDBStateBackend(memoryStateBackend); backend.setDbStoragePath(dbDir.getAbsolutePath()); streamConfig.setStateBackend(backend); streamConfig.setStreamOperator(new AsyncCheckpointOperator()); StreamMockEnvironment mockEnv = new StreamMockEnvironment(testHarness.jobConfig, testHarness.taskConfig, testHarness.memorySize, new MockInputSplitProvider(), testHarness.bufferSize); BlockingStreamMemoryStateBackend.waitFirstWriteLatch = new OneShotLatch(); BlockingStreamMemoryStateBackend.unblockCancelLatch = new OneShotLatch(); testHarness.invoke(mockEnv); // wait for the task to be running for (Field field : StreamTask.class.getDeclaredFields()) { if (field.getName().equals("isRunning")) { field.setAccessible(true); while (!field.getBoolean(task)) { Thread.sleep(10); } } } task.triggerCheckpoint(new CheckpointMetaData(42, 17), CheckpointOptions.forFullCheckpoint()); testHarness.processElement(new StreamRecord<>("Wohoo", 0)); BlockingStreamMemoryStateBackend.waitFirstWriteLatch.await(); task.cancel(); BlockingStreamMemoryStateBackend.unblockCancelLatch.trigger(); testHarness.endInput(); try { ExecutorService threadPool = task.getAsyncOperationsThreadPool(); threadPool.shutdown(); Assert.assertTrue(threadPool.awaitTermination(60_000, TimeUnit.MILLISECONDS)); testHarness.waitForTaskCompletion(); if (mockEnv.wasFailedExternally()) { throw new AsynchronousException(new InterruptedException("Exception was thrown as expected.")); } fail("Operation completed. Cancel failed."); } catch (Exception expected) { AsynchronousException asynchronousException = null; if (expected instanceof AsynchronousException) { asynchronousException = (AsynchronousException) expected; } else if (expected.getCause() instanceof AsynchronousException) { asynchronousException = (AsynchronousException) expected.getCause(); } else { fail("Unexpected exception: " + expected); } // we expect the exception from canceling snapshots Throwable innerCause = asynchronousException.getCause(); Assert.assertTrue("Unexpected inner cause: " + innerCause, innerCause instanceof CancellationException //future canceled || innerCause instanceof InterruptedException); //thread interrupted } }
From source file:org.apache.hcatalog.mapreduce.HCatMapReduceTest.java
License:Apache License
@BeforeClass public static void setUpOneTime() throws Exception { fs = new LocalFileSystem(); fs.initialize(fs.getWorkingDirectory().toUri(), new Configuration()); HiveConf hiveConf = new HiveConf(); hiveConf.setInt(HCatConstants.HCAT_HIVE_CLIENT_EXPIRY_TIME, 0); // Hack to initialize cache with 0 expiry time causing it to return a new hive client every time // Otherwise the cache doesn't play well with the second test method with the client gets closed() in the // tearDown() of the previous test HCatUtil.getHiveClient(hiveConf);//from w ww . j a v a 2s .com MapCreate.writeCount = 0; MapRead.readCount = 0; }
From source file:org.apache.hive.hcatalog.mapreduce.HCatMapReduceTest.java
License:Apache License
@BeforeClass public static void setUpOneTime() throws Exception { fs = new LocalFileSystem(); fs.initialize(fs.getWorkingDirectory().toUri(), new Configuration()); HiveConf hiveConf = new HiveConf(); hiveConf.setInt(HCatConstants.HCAT_HIVE_CLIENT_EXPIRY_TIME, 0); // Hack to initialize cache with 0 expiry time causing it to return a new hive client every time // Otherwise the cache doesn't play well with the second test method with the client gets closed() in the // tearDown() of the previous test HCatUtil.getHiveMetastoreClient(hiveConf); MapCreate.writeCount = 0;/*ww w .j av a 2 s.co m*/ MapRead.readCount = 0; }
From source file:org.apache.hive.hcatalog.pig.TestHCatLoaderEncryption.java
License:Apache License
@Test public void testReadDataFromEncryptedHiveTableByHCatMR() throws Exception { assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); readRecords.clear();/*from ww w .j a v a 2 s . co m*/ Configuration conf = new Configuration(); Job job = new Job(conf, "hcat mapreduce read encryption test"); job.setJarByClass(this.getClass()); job.setMapperClass(TestHCatLoaderEncryption.MapRead.class); // input/output settings job.setInputFormatClass(HCatInputFormat.class); job.setOutputFormatClass(TextOutputFormat.class); HCatInputFormat.setInput(job, MetaStoreUtils.DEFAULT_DATABASE_NAME, ENCRYPTED_TABLE, null); job.setMapOutputKeyClass(BytesWritable.class); job.setMapOutputValueClass(Text.class); job.setNumReduceTasks(0); FileSystem fs = new LocalFileSystem(); String pathLoc = TEST_DATA_DIR + "/testHCatMREncryptionOutput"; Path path = new Path(pathLoc); if (fs.exists(path)) { fs.delete(path, true); } TextOutputFormat.setOutputPath(job, new Path(WindowsPathUtil.getHdfsUriString(pathLoc))); job.waitForCompletion(true); int numTuplesRead = 0; for (HCatRecord hCatRecord : readRecords) { assertEquals(2, hCatRecord.size()); assertNotNull(hCatRecord.get(0)); assertNotNull(hCatRecord.get(1)); assertTrue(hCatRecord.get(0).getClass() == Integer.class); assertTrue(hCatRecord.get(1).getClass() == String.class); assertEquals(hCatRecord.get(0), basicInputData.get(numTuplesRead).first); assertEquals(hCatRecord.get(1), basicInputData.get(numTuplesRead).second); numTuplesRead++; } assertEquals("failed HCat MR read with storage format: " + this.storageFormat, basicInputData.size(), numTuplesRead); }
From source file:org.apache.tajo.pullserver.PullServerAuxService.java
License:Apache License
@Override public synchronized void init(Configuration conf) { try {// w w w. ja va 2 s . c o m manageOsCache = conf.getBoolean(SHUFFLE_MANAGE_OS_CACHE, DEFAULT_SHUFFLE_MANAGE_OS_CACHE); readaheadLength = conf.getInt(SHUFFLE_READAHEAD_BYTES, DEFAULT_SHUFFLE_READAHEAD_BYTES); ThreadFactory bossFactory = new ThreadFactoryBuilder() .setNameFormat("PullServerAuxService Netty Boss #%d").build(); ThreadFactory workerFactory = new ThreadFactoryBuilder() .setNameFormat("PullServerAuxService Netty Worker #%d").build(); selector = new NioServerSocketChannelFactory(Executors.newCachedThreadPool(bossFactory), Executors.newCachedThreadPool(workerFactory)); localFS = new LocalFileSystem(); super.init(new Configuration(conf)); } catch (Throwable t) { LOG.error(t); } }
From source file:org.apache.tajo.pullserver.TajoPullServerService.java
License:Apache License
@Override public void init(Configuration conf) { try {/* w w w. j a v a 2 s . c o m*/ manageOsCache = conf.getBoolean(SHUFFLE_MANAGE_OS_CACHE, DEFAULT_SHUFFLE_MANAGE_OS_CACHE); readaheadLength = conf.getInt(SHUFFLE_READAHEAD_BYTES, DEFAULT_SHUFFLE_READAHEAD_BYTES); int workerNum = conf.getInt("tajo.shuffle.rpc.server.worker-thread-num", Runtime.getRuntime().availableProcessors() * 2); selector = RpcChannelFactory.createServerChannelFactory("TajoPullServerService", workerNum) .option(ChannelOption.TCP_NODELAY, true) .childOption(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT) .childOption(ChannelOption.TCP_NODELAY, true); localFS = new LocalFileSystem(); conf.setInt(TajoConf.ConfVars.PULLSERVER_PORT.varname, TajoConf.ConfVars.PULLSERVER_PORT.defaultIntVal); super.init(conf); LOG.info("Tajo PullServer initialized: readaheadLength=" + readaheadLength); } catch (Throwable t) { LOG.error(t, t); } }
From source file:org.apache.tajo.worker.LocalFetcher.java
License:Apache License
@VisibleForTesting public LocalFetcher(TajoConf conf, URI uri, String tableName) throws IOException { super(conf, uri); this.maxUrlLength = conf.getIntVar(ConfVars.PULLSERVER_FETCH_URL_MAX_LENGTH); this.tableName = tableName; this.localFileSystem = new LocalFileSystem(); this.localDirAllocator = new LocalDirAllocator(ConfVars.WORKER_TEMPORAL_DIR.varname); this.pullServerService = null; String scheme = uri.getScheme() == null ? "http" : uri.getScheme(); this.host = uri.getHost() == null ? "localhost" : uri.getHost(); this.port = uri.getPort(); if (port == -1) { if (scheme.equalsIgnoreCase("http")) { this.port = 80; } else if (scheme.equalsIgnoreCase("https")) { this.port = 443; }/*from w w w .jav a2 s.c om*/ } bootstrap = new Bootstrap() .group(NettyUtils.getSharedEventLoopGroup(NettyUtils.GROUP.FETCHER, conf.getIntVar(ConfVars.SHUFFLE_RPC_CLIENT_WORKER_THREAD_NUM))) .channel(NioSocketChannel.class).option(ChannelOption.ALLOCATOR, NettyUtils.ALLOCATOR) .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, conf.getIntVar(ConfVars.SHUFFLE_FETCHER_CONNECT_TIMEOUT) * 1000) .option(ChannelOption.SO_RCVBUF, 1048576) // set 1M .option(ChannelOption.TCP_NODELAY, true); }