List of usage examples for org.apache.hadoop.fs FileSystem getLocal
public static LocalFileSystem getLocal(Configuration conf) throws IOException
From source file:com.tdunning.plume.local.lazy.MapRedTwoSequentialGBKTest.java
License:Apache License
@Test public void test() throws Exception { String outputPath = "/tmp/output-plume-twosequentialgbktest"; String inputPath = "/tmp/input-wordcount.txt"; // Prepare input for test FileSystem system = FileSystem.getLocal(new Configuration()); system.copyFromLocalFile(new Path(Resources.getResource("simple-text.txt").getPath()), new Path(inputPath)); // Prepare output for test system.delete(new Path(outputPath), true); // Prepare workflow TwoSequentialGBKWorkflow workFlow = new TwoSequentialGBKWorkflow(); // Execute it MapRedExecutor executor = new MapRedExecutor(); executor.execute(workFlow, outputPath); String outputId = ((LazyCollection<?>) workFlow.getOutputs().get(0)).getPlumeId(); List<String> str = Files.readLines(new File(outputPath + "/" + outputId + "/1-r-00000"), Charsets.UTF_8); Map<String, String> m = Maps.newHashMap(); for (String line : str) { m.put(line.split("\t")[0], line.split("\t")[1]); // not super-optimal, but less code }/* w w w. j a v a 2 s . c o m*/ assertEquals("bar 2", m.get("To test text processing with some simple")); assertEquals("bar 2", m.get("examples is some simple")); assertEquals("bar 2", m.get("is is")); assertEquals("bar 2", m.get("some simple text")); }
From source file:com.tdunning.plume.local.lazy.MapRedWordCountTest.java
License:Apache License
/** * The wordcount example to test with local hadoop * //from w w w . j a v a2 s . co m * @throws IOException * @throws ClassNotFoundException * @throws InterruptedException */ @Test public void testWordCount() throws IOException, InterruptedException, ClassNotFoundException { String inputPath = "/tmp/input-wordcount.txt"; String outputPath = "/tmp/output-plume-wordcount"; // Prepare input for test FileSystem system = FileSystem.getLocal(new Configuration()); system.copyFromLocalFile(new Path(Resources.getResource("simple-text.txt").getPath()), new Path(inputPath)); // Prepare output for test system.delete(new Path(outputPath), true); // Prepare workflow WordCountWorkflow workFlow = new WordCountWorkflow(); // Execute it MapRedExecutor executor = new MapRedExecutor(); executor.execute(workFlow, outputPath); List<String> str = Files.readLines(new File(outputPath + "/1_1/1-r-00000"), Charsets.UTF_8); Map<String, String> m = Maps.newHashMap(); for (String line : str) { m.put(line.split("\t")[0], line.split("\t")[1]); // not super-optimal, but less code } assertEquals(3 + "", m.get("is")); assertEquals(3 + "", m.get("some")); assertEquals(3 + "", m.get("simple")); assertEquals(1 + "", m.get("examples")); assertEquals(2 + "", m.get("text")); }
From source file:com.tomslabs.grid.avro.HadoopTestBase.java
License:Apache License
@BeforeClass public static void beforeClass() throws Exception { System.setProperty("hadoop.tmp.dir", new File("target/hadoop-test").getAbsolutePath()); System.setProperty("test.build.data", new File("target/hadoop-test").getAbsolutePath()); localConf = new Configuration(); localConf.set("fs.default.name", "file://" + new File("target/hadoop-test/dfs").getAbsolutePath()); localFs = FileSystem.getLocal(localConf); localConf.set("hadoop.log.dir", new Path("target/hadoop-test/logs").makeQualified(localFs).toString()); localConf.set("mapred.system.dir", new Path("target/hadoop-test/mapred/sys").makeQualified(localFs).toString()); localConf.set("mapred.local.dir", new File("target/hadoop-test/mapred/local").getAbsolutePath()); localConf.set("mapred.temp.dir", new File("target/hadoop-test/mapred/tmp").getAbsolutePath()); System.setProperty("hadoop.log.dir", localConf.get("hadoop.log.dir")); }
From source file:com.uber.hoodie.common.io.storage.HoodieWrapperFileSystem.java
License:Apache License
private Path convertToLocalPath(Path oldPath) { try {//from w ww .j av a2s . c o m return convertPathWithScheme(oldPath, FileSystem.getLocal(getConf()).getScheme()); } catch (IOException e) { throw new HoodieIOException(e.getMessage(), e); } }
From source file:com.xiaomi.linden.hadoop.indexing.reduce.ShardWriter.java
License:Apache License
/** * Constructor/* w w w. j a v a2s .c o m*/ * @param fs * @param shard * @param tempDir * @param conf * @throws IOException */ public ShardWriter(FileSystem fs, Shard shard, String tempDir, Configuration conf) throws IOException { logger.info("Construct a shard writer"); this.conf = conf; this.fs = fs; localFs = FileSystem.getLocal(conf); perm = new Path(shard.getDirectory()); taxoPerm = new Path(shard.getDirectory() + ".taxonomy"); String indexDir = tempDir + "/" + "index"; String taxoDir = tempDir + "/" + "taxo"; temp = new Path(indexDir); taxoTemp = new Path(taxoDir); if (localFs.exists(temp)) { File tempFile = new File(temp.getName()); if (tempFile.exists()) { LindenReducer.deleteDir(tempFile); } } if (!fs.exists(perm)) { fs.mkdirs(perm); } else { moveToTrash(conf, perm); fs.mkdirs(perm); } if (!fs.exists(taxoPerm)) { fs.mkdirs(taxoPerm); } else { moveToTrash(conf, taxoPerm); fs.mkdirs(taxoPerm); } IndexWriterConfig config = new IndexWriterConfig(Version.LATEST, null); config.setIndexDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()); writer = new IndexWriter(FSDirectory.open(new File(indexDir)), config); taxoWriter = new DirectoryTaxonomyWriter(FSDirectory.open(new File(taxoDir))); }
From source file:com.yolodata.tbana.cascading.shuttl.ShuttlCsvTest.java
License:Open Source License
@Override public void setUp() throws Exception { fs = FileSystem.getLocal(new Configuration()); fs.delete(new Path(TestUtils.TEST_FILE_PATH), true); }
From source file:com.yolodata.tbana.hadoop.mapred.shuttl.bucket.search.BucketFinderTest.java
License:Open Source License
@Before public void setUp() throws Exception { fs = FileSystem.getLocal(new Configuration()); fs.delete(new Path(TestUtils.TEST_FILE_PATH), true); index = createIndexWithBuckets();/*from w w w. java 2 s .com*/ }
From source file:com.yolodata.tbana.util.search.filter.BucketTimestampFilterTest.java
License:Open Source License
@Test public void testFilter() throws Exception { FileSystem fileSystem = FileSystem.getLocal(new Configuration()); String[] buckets = { "db_3000_0_index", "db_1501_1500_index", "db_5000_4000_index", "db_3000_1500_index", "db_3000_1500_index", "db_2000_1000_index" }; Path testRoot = FileSystemTestUtils.createEmptyDir(fileSystem); List<Path> bucketPaths = FileSystemTestUtils.createDirectories(fileSystem, testRoot, buckets); long earliest = 1000; long latest = 2000; BucketTimestampFilter filter = new BucketTimestampFilter(fileSystem, earliest, latest); assertEquals(true, filter.accept(bucketPaths.get(0).toString())); assertEquals(true, filter.accept(bucketPaths.get(1).toString())); assertEquals(false, filter.accept(bucketPaths.get(2).toString())); assertEquals(true, filter.accept(bucketPaths.get(3).toString())); assertEquals(true, filter.accept(bucketPaths.get(4).toString())); assertEquals(true, filter.accept(bucketPaths.get(5).toString())); }
From source file:com.yolodata.tbana.util.search.HadoopPathFinderTest.java
License:Open Source License
@Before public void setUp() throws Exception { fileSystem = FileSystem.getLocal(new Configuration()); finder = new HadoopPathFinder(fileSystem); }
From source file:com.yolodata.tbana.util.search.ShuttlDirectoryTreeFactory.java
License:Open Source License
public ShuttlDirectoryTreeFactory() throws IOException { fileSystem = FileSystem.getLocal(new Configuration()); root = FileSystemTestUtils.createEmptyDir(fileSystem); indexPaths = new ArrayList<Path>(); createBaseStructure();/*from w w w . j a va2 s .c o m*/ }