List of usage examples for org.apache.hadoop.fs FileSystem setPermission
public void setPermission(Path p, FsPermission permission) throws IOException
From source file:org.apache.hive.hcatalog.mapreduce.TestHCatMultiOutputFormat.java
License:Apache License
private static void createTable(String tableName, String tablePerm) throws Exception { Table tbl = new Table(); tbl.setDbName(DATABASE);/*from w ww . j av a2 s. c o m*/ tbl.setTableName(tableName); StorageDescriptor sd = new StorageDescriptor(); sd.setCols(ColumnHolder.colMapping.get(tableName)); tbl.setSd(sd); sd.setParameters(new HashMap<String, String>()); sd.setSerdeInfo(new SerDeInfo()); sd.getSerdeInfo().setName(tbl.getTableName()); sd.getSerdeInfo().setParameters(new HashMap<String, String>()); sd.setInputFormat(org.apache.hadoop.hive.ql.io.RCFileInputFormat.class.getName()); sd.setOutputFormat(org.apache.hadoop.hive.ql.io.RCFileOutputFormat.class.getName()); sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_FORMAT, "1"); sd.getSerdeInfo().setSerializationLib(org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe.class.getName()); tbl.setPartitionKeys(ColumnHolder.partitionCols); hmsc.createTable(tbl); Path path = new Path(warehousedir, tableName); FileSystem fs = path.getFileSystem(hiveConf); fs.setPermission(path, new FsPermission(tablePerm)); }
From source file:org.apache.hive.service.TestDFSErrorHandling.java
License:Apache License
@Test public void testAccessDenied() throws Exception { assertTrue("Test setup failed. MiniHS2 is not initialized", miniHS2 != null && miniHS2.isStarted()); Class.forName(MiniHS2.getJdbcDriverName()); Path scratchDir = new Path(HiveConf.getVar(hiveConf, HiveConf.ConfVars.SCRATCHDIR)); MiniDFSShim dfs = miniHS2.getDfs();//from ww w .j a va2 s . com FileSystem fs = dfs.getFileSystem(); Path stickyBitDir = new Path(scratchDir, "stickyBitDir"); fs.mkdirs(stickyBitDir); String dataFileDir = hiveConf.get("test.data.files").replace('\\', '/').replace("c:", "").replace("C:", "") .replace("D:", "").replace("d:", ""); Path dataFilePath = new Path(dataFileDir, "kv1.txt"); fs.copyFromLocalFile(dataFilePath, stickyBitDir); FsPermission fsPermission = new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL, true); // Sets the sticky bit on stickyBitDir - now removing file kv1.txt from stickyBitDir by // unprivileged user will result in a DFS error. fs.setPermission(stickyBitDir, fsPermission); FileStatus[] files = fs.listStatus(stickyBitDir); // Connecting to HS2 as foo. Connection hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL(), "foo", "bar"); Statement stmt = hs2Conn.createStatement(); String tableName = "stickyBitTable"; stmt.execute("drop table if exists " + tableName); stmt.execute("create table " + tableName + " (foo int, bar string)"); try { // This statement will attempt to move kv1.txt out of stickyBitDir as user foo. HS2 is // expected to return 20009. stmt.execute("LOAD DATA INPATH '" + stickyBitDir.toUri().getPath() + "/kv1.txt' " + "OVERWRITE INTO TABLE " + tableName); } catch (Exception e) { if (e instanceof SQLException) { SQLException se = (SQLException) e; Assert.assertEquals("Unexpected error code", 20009, se.getErrorCode()); System.out.println(String.format("Error Message: %s", se.getMessage())); } else throw e; } stmt.execute("drop table if exists " + tableName); stmt.close(); hs2Conn.close(); }
From source file:org.apache.ignite.internal.processors.hadoop.impl.delegate.HadoopIgfsSecondaryFileSystemDelegateImpl.java
License:Apache License
/** {@inheritDoc} */ @Nullable/*from ww w . ja v a2 s . c om*/ @Override public IgfsFile update(IgfsPath path, Map<String, String> props) { HadoopIgfsProperties props0 = new HadoopIgfsProperties(props); final FileSystem fileSys = fileSystemForUser(); Path hadoopPath = convert(path); try { if (!fileSys.exists(hadoopPath)) return null; if (props0.userName() != null || props0.groupName() != null) fileSys.setOwner(hadoopPath, props0.userName(), props0.groupName()); if (props0.permission() != null) fileSys.setPermission(hadoopPath, props0.permission()); } catch (IOException e) { throw handleSecondaryFsError(e, "Failed to update file properties [path=" + path + "]"); } return info(path); }
From source file:org.apache.impala.analysis.AnalyzeDDLTest.java
License:Apache License
/** * Validate if location path analysis issues proper warnings when directory * permissions/existence checks fail./* www .ja v a 2 s . c om*/ */ @Test public void TestPermissionValidation() throws AnalysisException { String location = "/test-warehouse/.tmp_" + UUID.randomUUID().toString(); Path parentPath = FileSystemUtil.createFullyQualifiedPath(new Path(location)); FileSystem fs = null; try { fs = parentPath.getFileSystem(FileSystemUtil.getConfiguration()); // Test location doesn't exist AnalyzesOk(String.format("create table new_table (col INT) location '%s/new_table'", location), String.format("Path '%s' cannot be reached: Path does not exist.", parentPath)); // Test localtion path with trailing slash. AnalyzesOk(String.format("create table new_table (col INT) location " + "'%s/new_table/'", location), String.format("Path '%s' cannot be reached: Path does not exist.", parentPath)); AnalyzesOk( String.format("create table new_table location '%s/new_table' " + "as select 1, 1", location), String.format("Path '%s' cannot be reached: Path does not exist.", parentPath)); AnalyzesOk( String.format("create table new_table like functional.alltypes " + "location '%s/new_table'", location), String.format("Path '%s' cannot be reached: Path does not exist.", parentPath)); AnalyzesOk(String.format("create database new_db location '%s/new_db'", location), String.format("Path '%s' cannot be reached: Path does not exist.", parentPath)); fs.mkdirs(parentPath); // Create a test data file for load data test FSDataOutputStream out = fs.create(new Path(parentPath, "test_loaddata/testdata.txt")); out.close(); fs.setPermission(parentPath, new FsPermission(FsAction.NONE, FsAction.NONE, FsAction.NONE)); // Test location exists but Impala doesn't have sufficient permission AnalyzesOk( String.format("create data Source serverlog location " + "'%s/foo.jar' class 'foo.Bar' API_VERSION 'V1'", location), String.format("Impala does not have READ access to path '%s'", parentPath)); AnalyzesOk( String.format("create external table new_table (col INT) location " + "'%s/new_table'", location), String.format("Impala does not have READ_WRITE access to path '%s'", parentPath)); AnalyzesOk( String.format("alter table functional.insert_string_partitioned " + "add partition (s2='hello') location '%s/new_partition'", location), String.format("Impala does not have READ_WRITE access to path '%s'", parentPath)); AnalyzesOk( String.format( "alter table functional.stringpartitionkey " + "partition(string_col = 'partition1') set location '%s/new_part_loc'", location), String.format("Impala does not have READ_WRITE access to path '%s'", parentPath)); // Test location exists and Impala does have sufficient permission fs.setPermission(parentPath, new FsPermission(FsAction.READ_WRITE, FsAction.NONE, FsAction.NONE)); AnalyzesOk(String.format("create external table new_table (col INT) location " + "'%s/new_table'", location)); } catch (IOException e) { throw new AnalysisException(e.getMessage(), e); } finally { // Clean up try { if (fs != null && fs.exists(parentPath)) { fs.delete(parentPath, true); } } catch (IOException e) { // Ignore } } }
From source file:org.apache.ivory.converter.AbstractOozieEntityMapper.java
License:Apache License
private void createTempDir(Cluster cluster, Path coordPath) throws IvoryException { try {/* w w w.ja v a2s . com*/ FileSystem fs = coordPath.getFileSystem(ClusterHelper.getConfiguration(cluster)); Path tempDir = new Path(coordPath, "../../logs"); fs.mkdirs(tempDir); fs.setPermission(tempDir, new FsPermission((short) 511)); } catch (Exception e) { throw new IvoryException("Unable to create temp dir in " + coordPath, e); } }
From source file:org.apache.kylin.common.metrics.metrics2.JsonFileMetricsReporter.java
License:Apache License
@Override public void start() { final Path tmpPath = new Path(pathString + ".tmp"); URI tmpPathURI = tmpPath.toUri(); final FileSystem fs; try {/* w w w . j a va 2 s . c o m*/ if (tmpPathURI.getScheme() == null && tmpPathURI.getAuthority() == null) { //default local fs = FileSystem.getLocal(new Configuration()); } else { fs = FileSystem.get(tmpPathURI, new Configuration()); } } catch (IOException e) { LOGGER.error("Unable to access filesystem for path " + tmpPath + ". Aborting reporting", e); return; } Runnable task = new Runnable() { public void run() { try { String json = null; try { json = jsonWriter.writeValueAsString(metricRegistry); } catch (JsonProcessingException e) { LOGGER.error("Unable to convert json to string ", e); return; } BufferedWriter bw = null; try { fs.delete(tmpPath, true); bw = new BufferedWriter( new OutputStreamWriter(fs.create(tmpPath, true), StandardCharsets.UTF_8)); bw.write(json); fs.setPermission(tmpPath, FsPermission.createImmutable((short) 0644)); } catch (IOException e) { LOGGER.error("Unable to write to temp file " + tmpPath, e); return; } finally { if (bw != null) { bw.close(); } } try { fs.rename(tmpPath, path); fs.setPermission(path, FsPermission.createImmutable((short) 0644)); } catch (IOException e) { LOGGER.error("Unable to rename temp file " + tmpPath + " to " + pathString, e); return; } } catch (Throwable t) { // catch all errors (throwable and execptions to prevent subsequent tasks from being suppressed) LOGGER.error("Error executing scheduled task ", t); } } }; executorService.scheduleWithFixedDelay(task, 0, frequency, TimeUnit.MILLISECONDS); }
From source file:org.apache.kylin.job.hadoop.hbase.BulkLoadJob.java
License:Apache License
@Override public int run(String[] args) throws Exception { Options options = new Options(); try {/* w ww . j a v a2 s .c o m*/ options.addOption(OPTION_INPUT_PATH); options.addOption(OPTION_HTABLE_NAME); options.addOption(OPTION_CUBE_NAME); parseOptions(options, args); String tableName = getOptionValue(OPTION_HTABLE_NAME).toUpperCase(); // e.g // /tmp/kylin-3f150b00-3332-41ca-9d3d-652f67f044d7/test_kylin_cube_with_slr_ready_2_segments/hfile/ // end with "/" String input = getOptionValue(OPTION_INPUT_PATH); Configuration conf = HBaseConfiguration.create(getConf()); FileSystem fs = FileSystem.get(conf); String cubeName = getOptionValue(OPTION_CUBE_NAME).toUpperCase(); KylinConfig config = KylinConfig.getInstanceFromEnv(); CubeManager cubeMgr = CubeManager.getInstance(config); CubeInstance cube = cubeMgr.getCube(cubeName); CubeDesc cubeDesc = cube.getDescriptor(); FsPermission permission = new FsPermission((short) 0777); for (HBaseColumnFamilyDesc cf : cubeDesc.getHBaseMapping().getColumnFamily()) { String cfName = cf.getName(); fs.setPermission(new Path(input + cfName), permission); } String[] newArgs = new String[2]; newArgs[0] = input; newArgs[1] = tableName; log.debug("Start to run LoadIncrementalHFiles"); int ret = ToolRunner.run(new LoadIncrementalHFiles(conf), newArgs); log.debug("End to run LoadIncrementalHFiles"); return ret; } catch (Exception e) { printUsage(options); throw e; } }
From source file:org.apache.kylin.job.hadoop.invertedindex.IIBulkLoadJob.java
License:Apache License
@Override public int run(String[] args) throws Exception { Options options = new Options(); try {//from w w w . j a va2s.com options.addOption(OPTION_INPUT_PATH); options.addOption(OPTION_HTABLE_NAME); options.addOption(OPTION_II_NAME); parseOptions(options, args); String tableName = getOptionValue(OPTION_HTABLE_NAME); String input = getOptionValue(OPTION_INPUT_PATH); String iiname = getOptionValue(OPTION_II_NAME); FileSystem fs = FileSystem.get(getConf()); FsPermission permission = new FsPermission((short) 0777); fs.setPermission(new Path(input, IIDesc.HBASE_FAMILY), permission); int hbaseExitCode = ToolRunner.run(new LoadIncrementalHFiles(getConf()), new String[] { input, tableName }); IIManager mgr = IIManager.getInstance(KylinConfig.getInstanceFromEnv()); IIInstance ii = mgr.getII(iiname); IISegment seg = ii.getFirstSegment(); seg.setStorageLocationIdentifier(tableName); seg.setStatus(SegmentStatusEnum.READY); mgr.updateII(ii); return hbaseExitCode; } catch (Exception e) { printUsage(options); throw e; } }
From source file:org.apache.kylin.storage.hbase.ii.IIBulkLoadJob.java
License:Apache License
@Override public int run(String[] args) throws Exception { Options options = new Options(); try {/* w ww. j a va2s . co m*/ options.addOption(OPTION_INPUT_PATH); options.addOption(OPTION_HTABLE_NAME); options.addOption(OPTION_II_NAME); parseOptions(options, args); String tableName = getOptionValue(OPTION_HTABLE_NAME).toUpperCase(); String input = getOptionValue(OPTION_INPUT_PATH); Configuration conf = HBaseConnection.getCurrentHBaseConfiguration(); FileSystem fs = FileSystem.get(conf); Path columnFamilyPath = new Path(input, IIDesc.HBASE_FAMILY); // File may have already been auto-loaded (in the case of MapR DB) if (fs.exists(columnFamilyPath)) { FsPermission permission = new FsPermission((short) 0777); fs.setPermission(columnFamilyPath, permission); } return ToolRunner.run(new LoadIncrementalHFiles(conf), new String[] { input, tableName }); } catch (Exception e) { printUsage(options); throw e; } }
From source file:org.apache.oozie.action.hadoop.FsActionExecutor.java
License:Apache License
private void doFsOperation(String op, FileSystem fs, Path p, Map<String, String> argsMap) throws ActionExecutorException, IOException { if (op.equals("chmod")) { String permissions = argsMap.get("permissions"); FsPermission newFsPermission = createShortPermission(permissions, p); fs.setPermission(p, newFsPermission); } else if (op.equals("chgrp")) { String user = argsMap.get("user"); String group = argsMap.get("group"); fs.setOwner(p, user, group);// www . jav a2 s. c o m } }