List of usage examples for org.apache.hadoop.fs LocalFileSystem LocalFileSystem
public LocalFileSystem()
From source file:com.datatorrent.stram.debug.TupleRecorderTest.java
License:Apache License
@Test public void testRecorder() throws IOException { FileSystem fs = new LocalFileSystem(); try {/* w ww .j a v a 2 s.c o m*/ TupleRecorder recorder = new TupleRecorder(null, "application_test_id_1"); recorder.getStorage().setBytesPerPartFile(4096); recorder.getStorage().setLocalMode(true); recorder.getStorage().setBasePath("file://" + testWorkDir.getAbsolutePath() + "/recordings"); recorder.addInputPortInfo("ip1", "str1"); recorder.addInputPortInfo("ip2", "str2"); recorder.addInputPortInfo("ip3", "str3"); recorder.addOutputPortInfo("op1", "str4"); recorder.setup(null, null); recorder.beginWindow(1000); recorder.beginWindow(1000); recorder.beginWindow(1000); Tuple t1 = new Tuple(); t1.key = "speed"; t1.value = "5m/h"; recorder.writeTuple(t1, "ip1"); recorder.endWindow(); Tuple t2 = new Tuple(); t2.key = "speed"; t2.value = "4m/h"; recorder.writeTuple(t2, "ip3"); recorder.endWindow(); Tuple t3 = new Tuple(); t3.key = "speed"; t3.value = "6m/h"; recorder.writeTuple(t3, "ip2"); recorder.endWindow(); recorder.beginWindow(1000); Tuple t4 = new Tuple(); t4.key = "speed"; t4.value = "2m/h"; recorder.writeTuple(t4, "op1"); recorder.endWindow(); recorder.teardown(); fs.initialize((new Path(recorder.getStorage().getBasePath()).toUri()), new Configuration()); Path path; FSDataInputStream is; String line; BufferedReader br; path = new Path(recorder.getStorage().getBasePath(), FSPartFileCollection.INDEX_FILE); is = fs.open(path); br = new BufferedReader(new InputStreamReader(is)); line = br.readLine(); // Assert.assertEquals("check index", "B:1000:T:0:part0.txt", line); Assert.assertTrue("check index", line.matches( "F:part0.txt:\\d+-\\d+:4:T:1000-1000:33:\\{\"3\":\"1\",\"1\":\"1\",\"0\":\"1\",\"2\":\"1\"\\}")); path = new Path(recorder.getStorage().getBasePath(), FSPartFileCollection.META_FILE); is = fs.open(path); br = new BufferedReader(new InputStreamReader(is)); ObjectMapper mapper = new ObjectMapper(); line = br.readLine(); Assert.assertEquals("check version", "1.2", line); br.readLine(); // RecordInfo //RecordInfo ri = mapper.readValue(line, RecordInfo.class); line = br.readLine(); PortInfo pi = mapper.readValue(line, PortInfo.class); Assert.assertEquals("port1", recorder.getPortInfoMap().get(pi.name).id, pi.id); Assert.assertEquals("port1", recorder.getPortInfoMap().get(pi.name).type, pi.type); line = br.readLine(); pi = mapper.readValue(line, PortInfo.class); Assert.assertEquals("port2", recorder.getPortInfoMap().get(pi.name).id, pi.id); Assert.assertEquals("port2", recorder.getPortInfoMap().get(pi.name).type, pi.type); line = br.readLine(); pi = mapper.readValue(line, PortInfo.class); Assert.assertEquals("port3", recorder.getPortInfoMap().get(pi.name).id, pi.id); Assert.assertEquals("port3", recorder.getPortInfoMap().get(pi.name).type, pi.type); line = br.readLine(); pi = mapper.readValue(line, PortInfo.class); Assert.assertEquals("port4", recorder.getPortInfoMap().get(pi.name).id, pi.id); Assert.assertEquals("port4", recorder.getPortInfoMap().get(pi.name).type, pi.type); Assert.assertEquals("port size", 4, recorder.getPortInfoMap().size()); //line = br.readLine(); path = new Path(recorder.getStorage().getBasePath(), "part0.txt"); is = fs.open(path); br = new BufferedReader(new InputStreamReader(is)); line = br.readLine(); Assert.assertTrue("check part0", line.startsWith("B:")); Assert.assertTrue("check part0", line.endsWith(":1000")); line = br.readLine(); Assert.assertTrue("check part0 1", line.startsWith("T:")); Assert.assertTrue("check part0 1", line.endsWith(":0:30:{\"key\":\"speed\",\"value\":\"5m/h\"}")); line = br.readLine(); Assert.assertTrue("check part0 2", line.startsWith("T:")); Assert.assertTrue("check part0 2", line.endsWith(":2:30:{\"key\":\"speed\",\"value\":\"4m/h\"}")); line = br.readLine(); Assert.assertTrue("check part0 3", line.startsWith("T:")); Assert.assertTrue("check part0 3", line.endsWith(":1:30:{\"key\":\"speed\",\"value\":\"6m/h\"}")); line = br.readLine(); Assert.assertTrue("check part0 4", line.startsWith("T:")); Assert.assertTrue("check part0 4", line.endsWith(":3:30:{\"key\":\"speed\",\"value\":\"2m/h\"}")); line = br.readLine(); Assert.assertTrue("check part0 5", line.startsWith("E:")); Assert.assertTrue("check part0 5", line.endsWith(":1000")); } catch (IOException ex) { throw new RuntimeException(ex); } finally { fs.close(); } }
From source file:com.metamx.druid.indexer.HadoopDruidIndexerConfigTest.java
License:Open Source License
@Test public void shouldMakeDefaultSegmentOutputPathIfNotHDFS() { final HadoopDruidIndexerConfig cfg; try {/* w w w . j a v a 2s . co m*/ cfg = jsonReadWriteRead("{" + "\"dataSource\": \"the:data:source\"," + " \"granularitySpec\":{" + " \"type\":\"uniform\"," + " \"gran\":\"hour\"," + " \"intervals\":[\"2012-07-10/P1D\"]" + " }," + "\"segmentOutputPath\": \"/tmp/dru:id/data:test\"" + "}", HadoopDruidIndexerConfig.class); } catch (Exception e) { throw Throwables.propagate(e); } cfg.setVersion("some:brand:new:version"); Bucket bucket = new Bucket(4711, new DateTime(2012, 07, 10, 5, 30), 4712); Path path = cfg.makeSegmentOutputPath(new LocalFileSystem(), bucket); Assert.assertEquals( "/tmp/dru:id/data:test/the:data:source/2012-07-10T05:00:00.000Z_2012-07-10T06:00:00.000Z/some:brand:new:version/4712", path.toString()); }
From source file:com.pentaho.big.data.bundles.impl.shim.hdfs.HadoopFileSystemFactoryImplTest.java
License:Apache License
@Test(expected = IOException.class) public void testLocalFileSystem() throws IOException { when(namedCluster.isMapr()).thenReturn(true); when(fileSystem.getDelegate()).thenReturn(new LocalFileSystem()); hadoopFileSystemFactory.create(namedCluster); }
From source file:com.thinkbiganalytics.kylo.hadoop.FileSystemUtilTest.java
License:Apache License
/** * Verify adding file systems to Hadoop configuration. */// w w w.j a va 2s .c o m @Test public void registerFileSystems() { final Configuration conf = new Configuration(false); FileSystemUtil.registerFileSystems( Arrays.asList(new LocalFileSystem(), new MockFileSystem(), new RawLocalFileSystem()), conf); Assert.assertEquals(LocalFileSystem.class.getName(), conf.get("fs.file.impl")); Assert.assertEquals(MockFileSystem.class.getName(), conf.get("fs.mock.impl")); }
From source file:gobblin.util.filesystem.InstrumentedLocalFileSystem.java
License:Apache License
public InstrumentedLocalFileSystem() { super(SCHEME, new LocalFileSystem()); }
From source file:gobblin.util.io.StreamCopierSharedLimiterKey.java
License:Apache License
private static String getFSIdentifier(URI uri) { if (new LocalFileSystem().getScheme().equals(uri.getScheme())) { return "localhost"; } else {// w w w . j av a 2 s . c om return ClustersNames.getInstance().getClusterName(uri.toString()); } }
From source file:io.druid.indexer.HadoopDruidIndexerConfigTest.java
License:Apache License
@Test public void shouldMakeDefaultSegmentOutputPathIfNotHDFS() { final HadoopIngestionSpec schema; try {//from www. j av a2s .c o m schema = jsonReadWriteRead( "{\n" + " \"dataSchema\": {\n" + " \"dataSource\": \"the:data:source\",\n" + " \"metricsSpec\": [],\n" + " \"granularitySpec\": {\n" + " \"type\": \"uniform\",\n" + " \"segmentGranularity\": \"hour\",\n" + " \"intervals\": [\"2012-07-10/P1D\"]\n" + " }\n" + " },\n" + " \"ioConfig\": {\n" + " \"type\": \"hadoop\",\n" + " \"segmentOutputPath\": \"/tmp/dru:id/data:test\"\n" + " }\n" + "}", HadoopIngestionSpec.class); } catch (Exception e) { throw Throwables.propagate(e); } HadoopDruidIndexerConfig cfg = new HadoopDruidIndexerConfig( schema.withTuningConfig(schema.getTuningConfig().withVersion("some:brand:new:version"))); Bucket bucket = new Bucket(4711, new DateTime(2012, 07, 10, 5, 30), 4712); Path path = JobHelper.makeSegmentOutputPath(new Path(cfg.getSchema().getIOConfig().getSegmentOutputPath()), new LocalFileSystem(), cfg.getSchema().getDataSchema().getDataSource(), cfg.getSchema().getTuningConfig().getVersion(), cfg.getSchema().getDataSchema().getGranularitySpec().bucketInterval(bucket.time).get(), bucket.partitionNum); Assert.assertEquals( "file:/tmp/dru:id/data:test/the:data:source/2012-07-10T05:00:00.000Z_2012-07-10T06:00:00.000Z/some:brand:new:version/4712", path.toString()); }
From source file:io.druid.storage.hdfs.HdfsDataSegmentPuller.java
License:Apache License
public FileUtils.FileCopyResult getSegmentFiles(final Path path, final File outDir) throws SegmentLoadingException { final LocalFileSystem localFileSystem = new LocalFileSystem(); try {/* w ww . ja v a 2 s . c o m*/ final FileSystem fs = path.getFileSystem(config); if (fs.isDirectory(path)) { // -------- directory --------- try { return RetryUtils.retry(new Callable<FileUtils.FileCopyResult>() { @Override public FileUtils.FileCopyResult call() throws Exception { if (!fs.exists(path)) { throw new SegmentLoadingException("No files found at [%s]", path.toString()); } final RemoteIterator<LocatedFileStatus> children = fs.listFiles(path, false); final ArrayList<FileUtils.FileCopyResult> localChildren = new ArrayList<>(); final FileUtils.FileCopyResult result = new FileUtils.FileCopyResult(); while (children.hasNext()) { final LocatedFileStatus child = children.next(); final Path childPath = child.getPath(); final String fname = childPath.getName(); if (fs.isDirectory(childPath)) { log.warn("[%s] is a child directory, skipping", childPath.toString()); } else { final File outFile = new File(outDir, fname); // Actual copy fs.copyToLocalFile(childPath, new Path(outFile.toURI())); result.addFile(outFile); } } log.info("Copied %d bytes from [%s] to [%s]", result.size(), path.toString(), outDir.getAbsolutePath()); return result; } }, shouldRetryPredicate(), DEFAULT_RETRY_COUNT); } catch (Exception e) { throw Throwables.propagate(e); } } else if (CompressionUtils.isZip(path.getName())) { // -------- zip --------- final FileUtils.FileCopyResult result = CompressionUtils.unzip(new ByteSource() { @Override public InputStream openStream() throws IOException { return getInputStream(path); } }, outDir, shouldRetryPredicate(), false); log.info("Unzipped %d bytes from [%s] to [%s]", result.size(), path.toString(), outDir.getAbsolutePath()); return result; } else if (CompressionUtils.isGz(path.getName())) { // -------- gzip --------- final String fname = path.getName(); final File outFile = new File(outDir, CompressionUtils.getGzBaseName(fname)); final FileUtils.FileCopyResult result = CompressionUtils.gunzip(new ByteSource() { @Override public InputStream openStream() throws IOException { return getInputStream(path); } }, outFile); log.info("Gunzipped %d bytes from [%s] to [%s]", result.size(), path.toString(), outFile.getAbsolutePath()); return result; } else { throw new SegmentLoadingException("Do not know how to handle file type at [%s]", path.toString()); } } catch (IOException e) { throw new SegmentLoadingException(e, "Error loading [%s]", path.toString()); } }
From source file:io.druid.storage.hdfs.HdfsDataSegmentPusherTest.java
License:Apache License
@Test public void shouldMakeDefaultSegmentOutputPathIfNotHDFS() { final HadoopIngestionSpec schema; try {//from w w w . j a v a 2 s. c o m schema = objectMapper.readValue( "{\n" + " \"dataSchema\": {\n" + " \"dataSource\": \"the:data:source\",\n" + " \"metricsSpec\": [],\n" + " \"granularitySpec\": {\n" + " \"type\": \"uniform\",\n" + " \"segmentGranularity\": \"hour\",\n" + " \"intervals\": [\"2012-07-10/P1D\"]\n" + " }\n" + " },\n" + " \"ioConfig\": {\n" + " \"type\": \"hadoop\",\n" + " \"segmentOutputPath\": \"/tmp/dru:id/data:test\"\n" + " }\n" + "}", HadoopIngestionSpec.class); } catch (Exception e) { throw Throwables.propagate(e); } HadoopDruidIndexerConfig cfg = new HadoopDruidIndexerConfig( schema.withTuningConfig(schema.getTuningConfig().withVersion("some:brand:new:version"))); Bucket bucket = new Bucket(4711, new DateTime(2012, 07, 10, 5, 30), 4712); Path path = JobHelper .makeFileNamePath(new Path(cfg.getSchema().getIOConfig().getSegmentOutputPath()), new LocalFileSystem(), new DataSegment(cfg.getSchema().getDataSchema().getDataSource(), cfg.getSchema().getDataSchema().getGranularitySpec().bucketInterval(bucket.time) .get(), cfg.getSchema().getTuningConfig().getVersion(), null, null, null, new NumberedShardSpec(bucket.partitionNum, 5000), -1, -1), JobHelper.INDEX_ZIP, new LocalDataSegmentPusher(new LocalDataSegmentPusherConfig(), objectMapper)); Assert.assertEquals( "file:/tmp/dru:id/data:test/the:data:source/2012-07-10T05:00:00.000Z_2012-07-10T06:00:00.000Z/some:brand:new:" + "version/4712/index.zip", path.toString()); path = JobHelper.makeFileNamePath(new Path(cfg.getSchema().getIOConfig().getSegmentOutputPath()), new LocalFileSystem(), new DataSegment(cfg.getSchema().getDataSchema().getDataSource(), cfg.getSchema().getDataSchema().getGranularitySpec().bucketInterval(bucket.time).get(), cfg.getSchema().getTuningConfig().getVersion(), null, null, null, new NumberedShardSpec(bucket.partitionNum, 5000), -1, -1), JobHelper.DESCRIPTOR_JSON, new LocalDataSegmentPusher(new LocalDataSegmentPusherConfig(), objectMapper)); Assert.assertEquals( "file:/tmp/dru:id/data:test/the:data:source/2012-07-10T05:00:00.000Z_2012-07-10T06:00:00.000Z/some:brand:new:" + "version/4712/descriptor.json", path.toString()); path = JobHelper .makeTmpPath(new Path(cfg.getSchema().getIOConfig().getSegmentOutputPath()), new LocalFileSystem(), new DataSegment(cfg.getSchema().getDataSchema().getDataSource(), cfg.getSchema().getDataSchema().getGranularitySpec().bucketInterval(bucket.time) .get(), cfg.getSchema().getTuningConfig().getVersion(), null, null, null, new NumberedShardSpec(bucket.partitionNum, 5000), -1, -1), new TaskAttemptID("abc", 123, TaskType.REDUCE, 1, 0), new LocalDataSegmentPusher(new LocalDataSegmentPusherConfig(), objectMapper)); Assert.assertEquals( "file:/tmp/dru:id/data:test/the:data:source/2012-07-10T05:00:00.000Z_2012-07-10T06:00:00.000Z/some:brand:new:" + "version/4712/index.zip.0", path.toString()); }
From source file:org.apache.druid.storage.hdfs.HdfsDataSegmentPusherTest.java
License:Apache License
@Test public void shouldMakeDefaultSegmentOutputPathIfNotHDFS() { final HadoopIngestionSpec schema; try {//from ww w. j a v a 2 s. c o m schema = objectMapper.readValue( "{\n" + " \"dataSchema\": {\n" + " \"dataSource\": \"the:data:source\",\n" + " \"metricsSpec\": [],\n" + " \"granularitySpec\": {\n" + " \"type\": \"uniform\",\n" + " \"segmentGranularity\": \"hour\",\n" + " \"intervals\": [\"2012-07-10/P1D\"]\n" + " }\n" + " },\n" + " \"ioConfig\": {\n" + " \"type\": \"hadoop\",\n" + " \"segmentOutputPath\": \"/tmp/dru:id/data:test\"\n" + " }\n" + "}", HadoopIngestionSpec.class); } catch (Exception e) { throw new RuntimeException(e); } HadoopDruidIndexerConfig cfg = new HadoopDruidIndexerConfig( schema.withTuningConfig(schema.getTuningConfig().withVersion("some:brand:new:version"))); Bucket bucket = new Bucket(4711, new DateTime(2012, 07, 10, 5, 30, ISOChronology.getInstanceUTC()), 4712); Path path = JobHelper .makeFileNamePath(new Path(cfg.getSchema().getIOConfig().getSegmentOutputPath()), new LocalFileSystem(), new DataSegment(cfg.getSchema().getDataSchema().getDataSource(), cfg.getSchema().getDataSchema().getGranularitySpec().bucketInterval(bucket.time) .get(), cfg.getSchema().getTuningConfig().getVersion(), null, null, null, new NumberedShardSpec(bucket.partitionNum, 5000), -1, -1), JobHelper.INDEX_ZIP, new LocalDataSegmentPusher(new LocalDataSegmentPusherConfig())); Assert.assertEquals( "file:/tmp/dru:id/data:test/the:data:source/2012-07-10T05:00:00.000Z_2012-07-10T06:00:00.000Z/some:brand:new:" + "version/4712/index.zip", path.toString()); path = JobHelper .makeTmpPath(new Path(cfg.getSchema().getIOConfig().getSegmentOutputPath()), new LocalFileSystem(), new DataSegment(cfg.getSchema().getDataSchema().getDataSource(), cfg.getSchema().getDataSchema().getGranularitySpec().bucketInterval(bucket.time) .get(), cfg.getSchema().getTuningConfig().getVersion(), null, null, null, new NumberedShardSpec(bucket.partitionNum, 5000), -1, -1), new TaskAttemptID("abc", 123, TaskType.REDUCE, 1, 0), new LocalDataSegmentPusher(new LocalDataSegmentPusherConfig())); Assert.assertEquals( "file:/tmp/dru:id/data:test/the:data:source/2012-07-10T05:00:00.000Z_2012-07-10T06:00:00.000Z/some:brand:new:" + "version/4712/index.zip.0", path.toString()); }