List of usage examples for org.apache.hadoop.fs FileSystem getLocal
public static LocalFileSystem getLocal(Configuration conf) throws IOException
From source file:org.apache.blur.mapreduce.lib.BlurMapReduceUtil.java
License:Apache License
/** * Add the jars containing the given classes to the job's configuration such * that JobClient will ship them to the cluster and add them to the * DistributedCache./*from w w w . j ava2 s . com*/ */ public static void addDependencyJars(Configuration conf, Class<?>... classes) throws IOException { FileSystem localFs = FileSystem.getLocal(conf); Set<String> jars = new HashSet<String>(); // Add jars that are already in the tmpjars variable jars.addAll(conf.getStringCollection("tmpjars")); // Add jars containing the specified classes for (Class<?> clazz : classes) { if (clazz == null) { continue; } String pathStr = findOrCreateJar(clazz); if (pathStr == null) { LOG.warn("Could not find jar for class " + clazz + " in order to ship it to the cluster."); continue; } Path path = new Path(pathStr); if (!localFs.exists(path)) { LOG.warn("Could not validate jar file " + path + " for class " + clazz); continue; } jars.add(path.makeQualified(localFs.getUri(), localFs.getWorkingDirectory()).toString()); } if (jars.isEmpty()) { return; } conf.set("tmpjars", StringUtils.arrayToString(jars.toArray(new String[0]))); }
From source file:org.apache.blur.mapreduce.lib.BlurOutputFormatMiniClusterTest.java
License:Apache License
@BeforeClass public static void setupTest() throws Exception { GCWatcher.init(0.60);//from ww w .j a v a 2 s .c o m JavaHome.checkJavaHome(); LocalFileSystem localFS = FileSystem.getLocal(new Configuration()); File testDirectory = new File(TMPDIR, "blur-cluster-test").getAbsoluteFile(); testDirectory.mkdirs(); Path directory = new Path(testDirectory.getPath()); FsPermission dirPermissions = localFS.getFileStatus(directory).getPermission(); FsAction userAction = dirPermissions.getUserAction(); FsAction groupAction = dirPermissions.getGroupAction(); FsAction otherAction = dirPermissions.getOtherAction(); StringBuilder builder = new StringBuilder(); builder.append(userAction.ordinal()); builder.append(groupAction.ordinal()); builder.append(otherAction.ordinal()); String dirPermissionNum = builder.toString(); System.setProperty("dfs.datanode.data.dir.perm", dirPermissionNum); testDirectory.delete(); miniCluster = new MiniCluster(); miniCluster.startBlurCluster(new File(testDirectory, "cluster").getAbsolutePath(), 2, 3, true, false); TEST_ROOT_DIR = new Path(miniCluster.getFileSystemUri().toString() + "/blur_test"); System.setProperty("hadoop.log.dir", "./target/BlurOutputFormatTest/hadoop_log"); try { fileSystem = TEST_ROOT_DIR.getFileSystem(conf); } catch (IOException io) { throw new RuntimeException("problem getting local fs", io); } FileSystem.setDefaultUri(conf, miniCluster.getFileSystemUri()); miniCluster.startMrMiniCluster(); conf = miniCluster.getMRConfiguration(); BufferStore.initNewBuffer(128, 128 * 128); }
From source file:org.apache.blur.mapreduce.lib.v2.DirectIndexingDriverMiniClusterTest.java
License:Apache License
@BeforeClass public static void setupTest() throws Exception { System.setProperty("test.build.data", "./target/DirectIndexingDriverTest/data"); System.setProperty("hadoop.log.dir", "./target/DirectIndexingDriverTest/hadoop_log"); try {/*from ww w . ja v a2 s. c om*/ localFs = FileSystem.getLocal(conf); } catch (IOException io) { throw new RuntimeException("problem getting local fs", io); } mr = new MiniMRCluster(1, "file:///", 1); jobConf = mr.createJobConf(); BufferStore.initNewBuffer(128, 128 * 128); }
From source file:org.apache.blur.slur.SolrLookingBlurServerTest.java
License:Apache License
@BeforeClass public static void startCluster() throws IOException { GCWatcher.init(0.60);// ww w.j a v a 2 s . c om LocalFileSystem localFS = FileSystem.getLocal(new Configuration()); File testDirectory = new File(TMPDIR, "blur-cluster-test").getAbsoluteFile(); testDirectory.mkdirs(); Path directory = new Path(testDirectory.getPath()); FsPermission dirPermissions = localFS.getFileStatus(directory).getPermission(); FsAction userAction = dirPermissions.getUserAction(); FsAction groupAction = dirPermissions.getGroupAction(); FsAction otherAction = dirPermissions.getOtherAction(); StringBuilder builder = new StringBuilder(); builder.append(userAction.ordinal()); builder.append(groupAction.ordinal()); builder.append(otherAction.ordinal()); String dirPermissionNum = builder.toString(); System.setProperty("dfs.datanode.data.dir.perm", dirPermissionNum); testDirectory.delete(); miniCluster = new MiniCluster(); miniCluster.startBlurCluster(new File(testDirectory, "cluster").getAbsolutePath(), 2, 3, true); connectionStr = miniCluster.getControllerConnectionStr(); }
From source file:org.apache.blur.thrift.BlurClusterTest.java
License:Apache License
@BeforeClass public static void startCluster() throws IOException { GCWatcher.init(0.60);//from w ww. j a v a2 s . co m LocalFileSystem localFS = FileSystem.getLocal(new Configuration()); File testDirectory = new File(TMPDIR, "blur-cluster-test").getAbsoluteFile(); testDirectory.mkdirs(); Path directory = new Path(testDirectory.getPath()); FsPermission dirPermissions = localFS.getFileStatus(directory).getPermission(); FsAction userAction = dirPermissions.getUserAction(); FsAction groupAction = dirPermissions.getGroupAction(); FsAction otherAction = dirPermissions.getOtherAction(); StringBuilder builder = new StringBuilder(); builder.append(userAction.ordinal()); builder.append(groupAction.ordinal()); builder.append(otherAction.ordinal()); String dirPermissionNum = builder.toString(); System.setProperty("dfs.datanode.data.dir.perm", dirPermissionNum); testDirectory.delete(); miniCluster = new MiniCluster(); miniCluster.startBlurCluster(new File(testDirectory, "cluster").getAbsolutePath(), 2, 3, true); }
From source file:org.apache.blur.thrift.BlurClusterTestBase.java
License:Apache License
@BeforeClass public static void startCluster() throws IOException { GCWatcher.init(0.60);/*from w w w . java2 s .co m*/ LocalFileSystem localFS = FileSystem.getLocal(new Configuration()); File testDirectory = new File(TMPDIR, "blur-cluster-test").getAbsoluteFile(); testDirectory.mkdirs(); Path directory = new Path(testDirectory.getPath()); FsPermission dirPermissions = localFS.getFileStatus(directory).getPermission(); FsAction userAction = dirPermissions.getUserAction(); FsAction groupAction = dirPermissions.getGroupAction(); FsAction otherAction = dirPermissions.getOtherAction(); StringBuilder builder = new StringBuilder(); builder.append(userAction.ordinal()); builder.append(groupAction.ordinal()); builder.append(otherAction.ordinal()); String dirPermissionNum = builder.toString(); System.setProperty("dfs.datanode.data.dir.perm", dirPermissionNum); testDirectory.delete(); miniCluster = new MiniCluster(); miniCluster.startBlurCluster(new File(testDirectory, "cluster").getAbsolutePath(), 2, 3, true, externalProcesses); }
From source file:org.apache.blur.trace.hdfs.HdfsTraceStorageTest.java
License:Apache License
@Before public void setUp() throws IOException, InterruptedException { rmr(TMPDIR);//from w w w . java 2 s . c om LocalFileSystem localFS = FileSystem.getLocal(new Configuration()); File testDirectory = new File(TMPDIR, "HdfsTraceStorageTest").getAbsoluteFile(); testDirectory.mkdirs(); Path directory = new Path(testDirectory.getPath()); FsPermission dirPermissions = localFS.getFileStatus(directory).getPermission(); FsAction userAction = dirPermissions.getUserAction(); FsAction groupAction = dirPermissions.getGroupAction(); FsAction otherAction = dirPermissions.getOtherAction(); StringBuilder builder = new StringBuilder(); builder.append(userAction.ordinal()); builder.append(groupAction.ordinal()); builder.append(otherAction.ordinal()); String dirPermissionNum = builder.toString(); System.setProperty("dfs.datanode.data.dir.perm", dirPermissionNum); configuration = new BlurConfiguration(); configuration.set(BLUR_HDFS_TRACE_PATH, directory.makeQualified(localFS).toString()); _storage = new HdfsTraceStorage(configuration); _storage.init(new Configuration()); }
From source file:org.apache.crunch.impl.mem.MemPipelineFileReadingWritingIT.java
License:Apache License
private void createTestSequenceFile(final File seqFile) throws IOException { SequenceFile.Writer writer = null; writer = new Writer(FileSystem.getLocal(baseTmpDir.getDefaultConfiguration()), baseTmpDir.getDefaultConfiguration(), new Path(seqFile.toString()), IntWritable.class, Text.class); writer.append(new IntWritable(1), new Text("hello")); writer.append(new IntWritable(2), new Text("world")); writer.close();/*from ww w .j av a 2s . c o m*/ }
From source file:org.apache.crunch.impl.mem.MemPipelineFileReadingWritingIT.java
License:Apache License
@Test public void testMemPipelineWriteSequenceFile_PCollection() throws IOException { // write// www . ja v a2s . com PCollection<String> collection = MemPipeline.typedCollectionOf(Writables.strings(), EXPECTED_COLLECTION); final Target target = To.sequenceFile(outputFile.toString()); MemPipeline.getInstance().write(collection, target); // read final SequenceFile.Reader reader = new Reader(FileSystem.getLocal(baseTmpDir.getDefaultConfiguration()), new Path(outputFile.toString()), baseTmpDir.getDefaultConfiguration()); final List<String> actual = Lists.newArrayList(); final NullWritable key = NullWritable.get(); final Text value = new Text(); while (reader.next(key, value)) { actual.add(value.toString()); } reader.close(); // assert read same as written assertEquals(EXPECTED_COLLECTION, actual); }