List of usage examples for org.apache.hadoop.fs FileSystem newInstance
public static FileSystem newInstance(Configuration conf) throws IOException
From source file:org.apache.apex.engine.security.TokenRenewer.java
License:Apache License
private long renewTokens(final boolean refresh, boolean checkOnly) throws IOException { logger.info("{}", checkOnly ? "Checking renewal" : (refresh ? "Refreshing tokens" : "Renewing tokens")); long expiryTime = System.currentTimeMillis() + (refresh ? tokenLifeTime : tokenRenewalInterval); final String tokenRenewer = UserGroupInformation.getCurrentUser().getUserName(); logger.debug("Token renewer {}", tokenRenewer); File keyTabFile = null;//www . j a v a 2s. com try (FileSystem fs = FileSystem.newInstance(conf)) { String destinationDir = FileUtils.getTempDirectoryPath(); keyTabFile = FSUtil.copyToLocalFileSystem(fs, destinationDir, destinationFile, hdfsKeyTabFile, conf); if (principal == null) { //principal = UserGroupInformation.getCurrentUser().getUserName(); principal = UserGroupInformation.getLoginUser().getUserName(); } logger.debug("Principal {}", principal); UserGroupInformation ugi = UserGroupInformation.loginUserFromKeytabAndReturnUGI(principal, keyTabFile.getAbsolutePath()); if (!checkOnly) { try { UserGroupInformation currUGI = UserGroupInformation.createProxyUser(tokenRenewer, ugi); currUGI.doAs(new PrivilegedExceptionAction<Object>() { @Override public Object run() throws Exception { if (refresh) { Credentials creds = new Credentials(); try (FileSystem fs1 = FileSystem.newInstance(conf)) { logger.info("Refreshing fs tokens"); fs1.addDelegationTokens(tokenRenewer, creds); logger.info("Refreshed tokens"); } if (renewRMToken) { try (YarnClient yarnClient = StramClientUtils.createYarnClient(conf)) { logger.info("Refreshing rm tokens"); new StramClientUtils.ClientRMHelper(yarnClient, conf) .addRMDelegationToken(tokenRenewer, creds); logger.info("Refreshed tokens"); } } credentials.addAll(creds); } else { Collection<Token<? extends TokenIdentifier>> tokens = credentials.getAllTokens(); for (Token<? extends TokenIdentifier> token : tokens) { logger.debug("Token {}", token); if (token.getKind().equals(HDFS_TOKEN_KIND) || (renewRMToken && token.getKind().equals(RMDelegationTokenIdentifier.KIND_NAME))) { logger.info("Renewing token {}", token.getKind()); token.renew(conf); logger.info("Renewed token"); } } } return null; } }); UserGroupInformation.getCurrentUser().addCredentials(credentials); } catch (InterruptedException e) { logger.error("Error while renewing tokens ", e); expiryTime = System.currentTimeMillis(); } catch (IOException e) { logger.error("Error while renewing tokens ", e); expiryTime = System.currentTimeMillis(); } } if (logger.isDebugEnabled()) { logger.debug("number of tokens: {}", credentials.getAllTokens().size()); Iterator<Token<?>> iter = credentials.getAllTokens().iterator(); while (iter.hasNext()) { Token<?> token = iter.next(); logger.debug("updated token: {}", token); } } } finally { if (keyTabFile != null) { keyTabFile.delete(); } } return expiryTime; }
From source file:org.apache.apex.malhar.lib.state.managed.IncrementalCheckpointManagerTest.java
License:Apache License
@Test public void testPurge() throws IOException, InterruptedException { FileSystem fileSystem = FileSystem.newInstance(new Configuration()); testTransferWindowFiles();//from w w w. j av a 2 s . c om RemoteIterator<LocatedFileStatus> iterator = fileSystem .listLocatedStatus(new Path(testMeta.applicationPath + "/bucket_data")); Assert.assertTrue(iterator.hasNext()); testMeta.managedStateContext.getBucketsFileSystem().deleteTimeBucketsLessThanEqualTo(200); iterator = fileSystem.listLocatedStatus(new Path(testMeta.applicationPath + "/bucket_data")); if (iterator.hasNext()) { Assert.fail("All buckets should be deleted"); } }
From source file:org.apache.crunch.kafka.offset.hdfs.HDFSOffsetReaderTest.java
License:Apache License
@Before public void setup() throws IOException { Configuration config = new Configuration(); config.set(FileSystem.DEFAULT_FS, tempFolder.newFolder().getAbsolutePath()); fileSystem = FileSystem.newInstance(config); basePath = new Path(tempFolder.newFolder().toString(), testName.getMethodName()); writer = new HDFSOffsetWriter(config, basePath); reader = new HDFSOffsetReader(config, basePath); }
From source file:org.apache.crunch.kafka.offset.hdfs.HDFSOffsetWriterTest.java
License:Apache License
@Before public void setup() throws IOException { config = new Configuration(); config.set(FileSystem.DEFAULT_FS, tempFolder.newFolder().getAbsolutePath()); fileSystem = FileSystem.newInstance(config); basePath = new Path(tempFolder.newFolder().toString(), testName.getMethodName()); writer = new HDFSOffsetWriter(config, basePath); }
From source file:org.apache.druid.storage.hdfs.HdfsDataSegmentPusherTest.java
License:Apache License
private void testUsingSchemeForMultipleSegments(final String scheme, final int numberOfSegments) throws Exception { Configuration conf = new Configuration(true); DataSegment[] segments = new DataSegment[numberOfSegments]; // Create a mock segment on disk File segmentDir = tempFolder.newFolder(); File tmp = new File(segmentDir, "version.bin"); final byte[] data = new byte[] { 0x0, 0x0, 0x0, 0x1 }; Files.write(data, tmp);//w w w . jav a 2 s .c o m final long size = data.length; HdfsDataSegmentPusherConfig config = new HdfsDataSegmentPusherConfig(); final File storageDirectory = tempFolder.newFolder(); config.setStorageDirectory( scheme != null ? StringUtils.format("%s://%s", scheme, storageDirectory.getAbsolutePath()) : storageDirectory.getAbsolutePath()); HdfsDataSegmentPusher pusher = new HdfsDataSegmentPusher(config, conf, new DefaultObjectMapper()); for (int i = 0; i < numberOfSegments; i++) { segments[i] = new DataSegment("foo", Intervals.of("2015/2016"), "0", new HashMap<>(), new ArrayList<>(), new ArrayList<>(), new NumberedShardSpec(i, i), 0, size); } for (int i = 0; i < numberOfSegments; i++) { final DataSegment pushedSegment = pusher.push(segmentDir, segments[i], false); String indexUri = StringUtils.format("%s/%s/%d_index.zip", FileSystem.newInstance(conf).makeQualified(new Path(config.getStorageDirectory())).toUri() .toString(), pusher.getStorageDir(segments[i], false), segments[i].getShardSpec().getPartitionNum()); Assert.assertEquals(segments[i].getSize(), pushedSegment.getSize()); Assert.assertEquals(segments[i], pushedSegment); Assert.assertEquals(ImmutableMap.of("type", "hdfs", "path", indexUri), pushedSegment.getLoadSpec()); // rename directory after push String segmentPath = pusher.getStorageDir(pushedSegment, false); File indexFile = new File(StringUtils.format("%s/%s/%d_index.zip", storageDirectory, segmentPath, pushedSegment.getShardSpec().getPartitionNum())); Assert.assertTrue(indexFile.exists()); Assert.assertEquals(segments[i].getSize(), pushedSegment.getSize()); Assert.assertEquals(segments[i], pushedSegment); indexFile = new File(StringUtils.format("%s/%s/%d_index.zip", storageDirectory, segmentPath, pushedSegment.getShardSpec().getPartitionNum())); Assert.assertTrue(indexFile.exists()); // push twice will fail and temp dir cleaned File outDir = new File(StringUtils.format("%s/%s", config.getStorageDirectory(), segmentPath)); outDir.setReadOnly(); try { pusher.push(segmentDir, segments[i], false); } catch (IOException e) { Assert.fail("should not throw exception"); } } }
From source file:org.apache.druid.storage.hdfs.HdfsDataSegmentPusherTest.java
License:Apache License
private void testUsingScheme(final String scheme) throws Exception { Configuration conf = new Configuration(true); // Create a mock segment on disk File segmentDir = tempFolder.newFolder(); File tmp = new File(segmentDir, "version.bin"); final byte[] data = new byte[] { 0x0, 0x0, 0x0, 0x1 }; Files.write(data, tmp);//w w w. jav a2s .c om final long size = data.length; HdfsDataSegmentPusherConfig config = new HdfsDataSegmentPusherConfig(); final File storageDirectory = tempFolder.newFolder(); config.setStorageDirectory( scheme != null ? StringUtils.format("%s://%s", scheme, storageDirectory.getAbsolutePath()) : storageDirectory.getAbsolutePath()); HdfsDataSegmentPusher pusher = new HdfsDataSegmentPusher(config, conf, new DefaultObjectMapper()); DataSegment segmentToPush = new DataSegment("foo", Intervals.of("2015/2016"), "0", new HashMap<>(), new ArrayList<>(), new ArrayList<>(), NoneShardSpec.instance(), 0, size); DataSegment segment = pusher.push(segmentDir, segmentToPush, false); String indexUri = StringUtils.format("%s/%s/%d_index.zip", FileSystem.newInstance(conf).makeQualified(new Path(config.getStorageDirectory())).toUri() .toString(), pusher.getStorageDir(segmentToPush, false), segmentToPush.getShardSpec().getPartitionNum()); Assert.assertEquals(segmentToPush.getSize(), segment.getSize()); Assert.assertEquals(segmentToPush, segment); Assert.assertEquals(ImmutableMap.of("type", "hdfs", "path", indexUri), segment.getLoadSpec()); // rename directory after push final String segmentPath = pusher.getStorageDir(segment, false); File indexFile = new File(StringUtils.format("%s/%s/%d_index.zip", storageDirectory, segmentPath, segment.getShardSpec().getPartitionNum())); Assert.assertTrue(indexFile.exists()); // push twice will fail and temp dir cleaned File outDir = new File(StringUtils.format("%s/%s", config.getStorageDirectory(), segmentPath)); outDir.setReadOnly(); try { pusher.push(segmentDir, segmentToPush, false); } catch (IOException e) { Assert.fail("should not throw exception"); } }
From source file:org.apache.gobblin.compliance.HivePartitionVersionFinder.java
License:Apache License
private static List<Partition> getPartitions(String completeTableName) { List<String> tableList = At_SPLITTER.splitToList(completeTableName); if (tableList.size() != 2) { log.warn("Invalid table name " + completeTableName); return Collections.EMPTY_LIST; }//ww w. jav a2 s.c o m try (AutoReturnableObject<IMetaStoreClient> client = ComplianceRetentionJob.pool.getClient()) { Table table = client.get().getTable(tableList.get(0), tableList.get(1)); HiveDataset dataset = new HiveDataset(FileSystem.newInstance(new Configuration()), ComplianceRetentionJob.pool, new org.apache.hadoop.hive.ql.metadata.Table(table), new Properties()); return dataset.getPartitionsFromDataset(); } catch (IOException | TException e) { log.warn("Unable to get Partitions for table " + completeTableName + " " + e.getMessage()); } return Collections.EMPTY_LIST; }
From source file:org.apache.gobblin.data.management.retention.Avro2OrcStaleDatasetCleaner.java
License:Apache License
public Avro2OrcStaleDatasetCleaner(String jobId, Properties props) throws IOException { super(jobId, log); props.setProperty(HiveDatasetFinder.HIVE_DATASET_CONFIG_PREFIX_KEY, HIVE_DATASET_CONFIG_AVRO_PREFIX); this.graceTimeInMillis = TimeUnit.DAYS .toMillis(Long.parseLong(props.getProperty(HIVE_PARTITION_DELETION_GRACE_TIME_IN_DAYS, DEFAULT_HIVE_PARTITION_DELETION_GRACE_TIME_IN_DAYS))); Config config = ConfigFactory.parseProperties(props); this.fs = FileSystem.newInstance(new Configuration()); this.metricContext = Instrumented.getMetricContext(ConfigUtils.configToState(config), ValidationJob.class); this.eventSubmitter = new EventSubmitter.Builder(this.metricContext, EventConstants.CONVERSION_NAMESPACE) .build();/*from ww w .ja v a 2 s. c o m*/ this.datasetFinder = new ConvertibleHiveDatasetFinder(this.fs, props, this.eventSubmitter); }
From source file:org.apache.kylin.monitor.FileUtils.java
License:Apache License
public static FileSystem getHdfsFileSystem() throws IOException { Configuration conf = new Configuration(); //conf.set("dfs.client.block.write.replace-datanode-on-failure.policy", "NEVER"); FileSystem fs = null;/* w w w . j av a 2s . com*/ try { fs = FileSystem.newInstance(conf); } catch (IOException e) { if (fs != null) { fs.close(); } logger.info("Failed to get hdfs FileSystem", e); } return fs; }
From source file:org.apache.kylin.rest.service.PerformService.java
License:Apache License
private List<String[]> readHdfsFile(String filePath) throws IOException { List<String[]> allRows = null; CSVReader reader = null;//from ww w. java 2 s .c o m FileSystem fs = null; Configuration conf = new Configuration(); try { fs = FileSystem.newInstance(conf); FSDataInputStream inputStream = fs.open(new Path(filePath)); reader = new CSVReader(new InputStreamReader(inputStream), '|'); //Read all rows at once allRows = reader.readAll(); } catch (IOException e) { logger.info("failed to read hdfs file:", e); } finally { fs.close(); } return allRows; }