List of usage examples for org.apache.hadoop.fs FileSystem create
public FSDataOutputStream create(Path f, FsPermission permission, EnumSet<CreateFlag> flags, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException
From source file:com.cloudera.hoop.client.fs.TestHoopFileSystem.java
License:Open Source License
private void testCreate(Path path, boolean override) throws Exception { Configuration conf = new Configuration(); conf.set("fs.http.impl", HoopFileSystem.class.getName()); FileSystem fs = FileSystem.get(getJettyURL().toURI(), conf); FsPermission permission = new FsPermission(FsAction.READ_WRITE, FsAction.NONE, FsAction.NONE); OutputStream os = fs.create(new Path(path.toUri().getPath()), permission, override, 1024, (short) 2, 100 * 1024 * 1024, null);/*from w w w .j a va 2s . co m*/ os.write(1); os.close(); fs.close(); fs = FileSystem.get(getHadoopConf()); FileStatus status = fs.getFileStatus(path); Assert.assertEquals(status.getReplication(), 2); Assert.assertEquals(status.getBlockSize(), 100 * 1024 * 1024); Assert.assertEquals(status.getPermission(), permission); InputStream is = fs.open(path); Assert.assertEquals(is.read(), 1); is.close(); fs.close(); }
From source file:com.cloudera.hoop.fs.FSCreate.java
License:Open Source License
/** * Executes the filesystem operation./*from w ww . j av a 2 s . co m*/ * * @param fs filesystem instance to use. * @return The URI of the created file. * @throws IOException thrown if an IO error occured. */ @Override public URI execute(FileSystem fs) throws IOException { if (replication == -1) { replication = (short) fs.getConf().getInt("dfs.replication", 3); } if (blockSize == -1) { blockSize = fs.getConf().getInt("dfs.block.size", 67108864); } FsPermission fsPermission = FSUtils.getPermission(permission); int bufferSize = fs.getConf().getInt("hoop.buffer.size", 4096); OutputStream os = fs.create(path, fsPermission, override, bufferSize, replication, blockSize, null); IOUtils.copy(is, os); os.close(); return FSUtils.convertPathToHoop(path, HoopServer.get().getBaseUrl()).toUri(); }
From source file:com.trendmicro.hdfs.webdav.test.TestCopySimple.java
License:Apache License
@BeforeClass public static void setup() throws Exception { Configuration conf = minicluster.getConfiguration(); conf.set("hadoop.proxyuser." + UserGroupInformation.getCurrentUser().getShortUserName() + ".groups", "users"); conf.set("hadoop.proxyuser." + UserGroupInformation.getCurrentUser().getShortUserName() + ".hosts", "localhost"); conf.set("hadoop.webdav.authentication.type", "simple"); conf.setBoolean("hadoop.webdav.authentication.simple.anonymous.allowed", true); minicluster.startMiniCluster(gatewayUser); LOG.info("Gateway started on port " + minicluster.getGatewayPort()); FsPermission.setUMask(conf, new FsPermission((short) 0)); FileSystem fs = minicluster.getTestFileSystem(); Path path = new Path("/test"); assertTrue(fs.mkdirs(path, new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL))); fs.setOwner(path, ownerUser.getShortUserName(), ownerUser.getGroupNames()[0]); ownerUser.doAs(new PrivilegedExceptionAction<Void>() { public Void run() throws Exception { FileSystem fs = minicluster.getTestFileSystem(); assertTrue(fs.mkdirs(new Path("/test/rw"), new FsPermission(FsAction.ALL, FsAction.WRITE_EXECUTE, FsAction.NONE))); assertTrue(fs.mkdirs(new Path("/test/ro"), new FsPermission(FsAction.READ_EXECUTE, FsAction.NONE, FsAction.NONE))); assertTrue(fs.mkdirs(new Path("/test/public"), new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL))); FSDataOutputStream os = fs.create(new Path("/test/rw/file1"), new FsPermission(FsAction.ALL, FsAction.READ, FsAction.NONE), true, 4096, (short) 1, 65536, null);/*w w w.ja va 2 s . c o m*/ assertNotNull(os); os.write(testData.getBytes()); os.close(); return null; } }); }
From source file:com.trendmicro.hdfs.webdav.test.TestDeleteSimple.java
License:Apache License
@BeforeClass public static void setup() throws Exception { Configuration conf = minicluster.getConfiguration(); conf.set("hadoop.proxyuser." + UserGroupInformation.getCurrentUser().getShortUserName() + ".groups", "users"); conf.set("hadoop.proxyuser." + UserGroupInformation.getCurrentUser().getShortUserName() + ".hosts", "localhost"); conf.set("hadoop.webdav.authentication.type", "simple"); conf.setBoolean("hadoop.webdav.authentication.simple.anonymous.allowed", true); minicluster.startMiniCluster(gatewayUser); LOG.info("Gateway started on port " + minicluster.getGatewayPort()); FsPermission.setUMask(conf, new FsPermission((short) 0)); FileSystem fs = minicluster.getTestFileSystem(); Path path = new Path("/test"); assertTrue(fs.mkdirs(path, new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL))); fs.setOwner(path, ownerUser.getShortUserName(), ownerUser.getGroupNames()[0]); ownerUser.doAs(new PrivilegedExceptionAction<Void>() { public Void run() throws Exception { FileSystem fs = minicluster.getTestFileSystem(); assertTrue(fs.mkdirs(new Path("/test/private"), new FsPermission(FsAction.ALL, FsAction.READ_EXECUTE, FsAction.NONE))); assertTrue(fs.mkdirs(new Path("/test/public"), new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL))); FSDataOutputStream os = fs.create(new Path("/test/private/file1"), new FsPermission(FsAction.ALL, FsAction.READ, FsAction.NONE), true, 4096, (short) 1, 65536, null);/*from ww w. jav a 2 s.c o m*/ assertNotNull(os); os.write(testData.getBytes()); os.close(); os = fs.create(new Path("/test/private/file2"), new FsPermission(FsAction.ALL, FsAction.READ, FsAction.NONE), true, 4096, (short) 1, 65536, null); assertNotNull(os); os.write(testData.getBytes()); os.close(); os = fs.create(new Path("/test/public/file3"), new FsPermission(FsAction.ALL, FsAction.READ, FsAction.READ), true, 4096, (short) 1, 65536, null); assertNotNull(os); os.write(testData.getBytes()); os.close(); os = fs.create(new Path("/test/public/file4"), new FsPermission(FsAction.ALL, FsAction.READ, FsAction.READ), true, 4096, (short) 1, 65536, null); assertNotNull(os); os.write(testData.getBytes()); os.close(); return null; } }); }
From source file:com.trendmicro.hdfs.webdav.test.TestGetSimple.java
License:Apache License
@BeforeClass public static void setup() throws Exception { Configuration conf = minicluster.getConfiguration(); conf.set("hadoop.proxyuser." + UserGroupInformation.getCurrentUser().getShortUserName() + ".groups", "users"); conf.set("hadoop.proxyuser." + UserGroupInformation.getCurrentUser().getShortUserName() + ".hosts", "localhost"); conf.set("hadoop.webdav.authentication.type", "simple"); conf.setBoolean("hadoop.webdav.authentication.simple.anonymous.allowed", true); minicluster.startMiniCluster(gatewayUser); LOG.info("Gateway started on port " + minicluster.getGatewayPort()); FsPermission.setUMask(conf, new FsPermission((short) 0)); FileSystem fs = minicluster.getTestFileSystem(); Path path = new Path("/test"); assertTrue(fs.mkdirs(path, new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL))); fs.setOwner(path, ownerUser.getShortUserName(), ownerUser.getGroupNames()[0]); ownerUser.doAs(new PrivilegedExceptionAction<Void>() { public Void run() throws Exception { FileSystem fs = minicluster.getTestFileSystem(); FSDataOutputStream os;//w w w . j av a 2 s .co m os = fs.create(new Path("/test/pubdata"), new FsPermission(FsAction.ALL, FsAction.READ, FsAction.NONE), true, 4096, (short) 1, 65536, null); assertNotNull(os); os.write(testPublicData.getBytes()); os.close(); os = fs.create(new Path("/test/privdata"), new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE), true, 4096, (short) 1, 65536, null); assertNotNull(os); os.write(testPrivateData.getBytes()); os.close(); return null; } }); }
From source file:com.trendmicro.hdfs.webdav.test.TestMoveSimple.java
License:Apache License
@BeforeClass public static void setup() throws Exception { Configuration conf = minicluster.getConfiguration(); conf.set("hadoop.proxyuser." + UserGroupInformation.getCurrentUser().getShortUserName() + ".groups", "users"); conf.set("hadoop.proxyuser." + UserGroupInformation.getCurrentUser().getShortUserName() + ".hosts", "localhost"); conf.set("hadoop.webdav.authentication.type", "simple"); conf.setBoolean("hadoop.webdav.authentication.simple.anonymous.allowed", true); minicluster.startMiniCluster(gatewayUser); LOG.info("Gateway started on port " + minicluster.getGatewayPort()); FsPermission.setUMask(conf, new FsPermission((short) 0)); FileSystem fs = minicluster.getTestFileSystem(); Path path = new Path("/test"); assertTrue(fs.mkdirs(path, new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL))); fs.setOwner(path, ownerUser.getShortUserName(), ownerUser.getGroupNames()[0]); ownerUser.doAs(new PrivilegedExceptionAction<Void>() { public Void run() throws Exception { FileSystem fs = minicluster.getTestFileSystem(); assertTrue(fs.mkdirs(new Path("/test/owner"), new FsPermission(FsAction.ALL, FsAction.READ_EXECUTE, FsAction.NONE))); assertTrue(fs.mkdirs(new Path("/test/public"), new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL))); FSDataOutputStream os = fs.create(new Path("/test/owner/file1"), new FsPermission(FsAction.ALL, FsAction.READ, FsAction.NONE), true, 4096, (short) 1, 65536, null);/*from w ww. j av a2 s . com*/ assertNotNull(os); os.write(testData.getBytes()); os.close(); os = fs.create(new Path("/test/public/file1"), new FsPermission(FsAction.ALL, FsAction.READ, FsAction.NONE), true, 4096, (short) 1, 65536, null); assertNotNull(os); os.write(testData.getBytes()); os.close(); return null; } }); }
From source file:com.trendmicro.hdfs.webdav.test.TestPropfindSimple.java
License:Apache License
@BeforeClass public static void setup() throws Exception { Configuration conf = minicluster.getConfiguration(); conf.set("hadoop.proxyuser." + UserGroupInformation.getCurrentUser().getShortUserName() + ".groups", "users"); conf.set("hadoop.proxyuser." + UserGroupInformation.getCurrentUser().getShortUserName() + ".hosts", "localhost"); conf.set("hadoop.webdav.authentication.type", "simple"); conf.setBoolean("hadoop.webdav.authentication.simple.anonymous.allowed", true); minicluster.startMiniCluster(gatewayUser); LOG.info("Gateway started on port " + minicluster.getGatewayPort()); FsPermission.setUMask(conf, new FsPermission((short) 0)); FileSystem fs = minicluster.getTestFileSystem(); Path path = new Path("/test"); assertTrue(fs.mkdirs(path, new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL))); fs.setOwner(path, ownerUser.getShortUserName(), ownerUser.getGroupNames()[0]); ownerUser.doAs(new PrivilegedExceptionAction<Void>() { public Void run() throws Exception { FileSystem fs = minicluster.getTestFileSystem(); for (Path dir : publicDirPaths) { assertTrue(//from w ww .j av a 2 s . co m fs.mkdirs(dir, new FsPermission(FsAction.ALL, FsAction.READ_EXECUTE, FsAction.NONE))); } for (Path dir : privateDirPaths) { assertTrue(fs.mkdirs(dir, new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE))); } for (Path path : publicFilePaths) { FSDataOutputStream os = fs.create(path, new FsPermission(FsAction.ALL, FsAction.READ, FsAction.NONE), true, 4096, (short) 1, 65536, null); assertNotNull(os); os.write(testPublicData.getBytes()); os.close(); } for (Path path : privateFilePaths) { FSDataOutputStream os = fs.create(path, new FsPermission(FsAction.ALL, FsAction.READ, FsAction.NONE), true, 4096, (short) 1, 65536, null); assertNotNull(os); os.write(testPrivateData.getBytes()); os.close(); } return null; } }); }