Java tutorial
/* * Copyright 2011 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.springframework.data.hadoop.fs; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import java.io.ByteArrayInputStream; import java.io.InputStream; import java.io.OutputStream; import java.net.URI; import java.net.URL; import java.util.Arrays; import java.util.UUID; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FsUrlStreamHandlerFactory; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.compress.DecompressorStream; import org.apache.hadoop.io.compress.DefaultCodec; import org.apache.hadoop.security.UserGroupInformation; import org.junit.After; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.io.Resource; import org.springframework.data.hadoop.TestUtils; import org.springframework.data.hadoop.configuration.ConfigurationFactoryBean; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; /** * Test for interacting with Hadoop HDFS. * * @author Costin Leau */ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration("/org/springframework/data/hadoop/hadoop-ctx.xml") public class HdfsResourceLoaderLegacyTest { @Autowired private Configuration cfg; private FileSystem fs; private HdfsResourceLoader loader; @Before public void before() throws Exception { FileSystemFactoryBean fsf = new FileSystemFactoryBean(); fsf.setConfiguration(cfg); fsf.afterPropertiesSet(); fs = fsf.getObject(); System.out.println("Current user is " + UserGroupInformation.getCurrentUser()); System.out.println("Home dir is " + fs.getHomeDirectory().toString()); loader = new HdfsResourceLoader(cfg, null); } @After public void after() throws Exception { if (fs != null) { fs.close(); } } @Test public void testPatternResolver() throws Exception { Resource[] resources = loader.getResources("**/*"); System.out.println(resources.length); System.out.println(Arrays.toString(resources)); Resource[] homeResources = loader.getResources("~/**/*"); System.out.println(homeResources.length); System.out.println(Arrays.toString(resources)); assertArrayEquals(resources, homeResources); Resource[] res = loader.getResources("/**/user/**/*"); System.out.println(Arrays.toString(res)); System.out.println(Arrays.toString(loader.getResources("/**/user/**/*.xml"))); } @Test public void testConnectExistingFS() throws Exception { System.out.println(fs.toString()); System.out.println(fs.getUri()); System.out.println(fs.getHomeDirectory()); FileStatus[] listStatus = fs.listStatus(new Path("/")); for (FileStatus fileStatus : listStatus) { System.out.println(fileStatus.getPath()); } Resource resource = loader.getResource("/"); System.out.println(resource.isReadable()); System.out.println(resource.contentLength()); } @Test public void testWriteable() throws Exception { String name = "test-" + UUID.randomUUID() + ".file"; Path path = new Path(name); try { Resource resource = loader.getResource(name); System.out.println(resource.toString()); assertFalse(resource.exists()); assertFalse(resource.isReadable()); assertFalse(resource.isOpen()); resource = TestUtils.writeToFS(loader, name); assertTrue(resource.exists()); assertTrue(resource.isReadable()); assertTrue(resource.isOpen()); InputStream in = resource.getInputStream(); byte[] copy = new byte[name.length()]; in.read(copy); in.close(); assertArrayEquals(name.getBytes(), copy); } finally { fs.delete(path, true); } } @Test public void testResolve() throws Exception { Resource resource = loader.getResource("/test"); try { URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory(fs.getConf())); } catch (Error err) { //somebody already registered an URL handler... // get out } System.out.println(((HdfsResource) resource).getPath().makeQualified(fs)); System.out.println(resource.getURI()); resource = loader.getResource("test"); System.out.println(resource.getURI()); System.out.println(resource.getURL()); } @Test public void testPathWithFragment() throws Exception { String name = "test-" + UUID.randomUUID() + ".file#fragment"; Path path = new Path(name); Resource resource = loader.getResource(name); try { System.out.println(resource.toString()); assertFalse(resource.exists()); assertFalse(resource.isReadable()); assertFalse(resource.isOpen()); resource = TestUtils.writeToFS(loader, resource.getURI().toString()); // assertTrue(resource instanceof WritableResource); // WritableResource wr = (WritableResource) resource; // assertTrue(wr.isWritable()); // // byte[] bytes = name.getBytes(); // OutputStream out = wr.getOutputStream(); // // out.write(bytes); // out.close(); assertTrue(resource.exists()); assertTrue(resource.isReadable()); assertTrue(resource.isOpen()); URI uri = resource.getURI(); assertEquals("fragment", uri.getFragment()); } finally { fs.delete(path, true); } } @Test public void testFilesMatch() throws Exception { try { TestUtils.writeToFS(loader, "local/" + UUID.randomUUID() + ".txt"); TestUtils.writeToFS(loader, "local/" + UUID.randomUUID() + ".txt"); TestUtils.writeToFS(loader, "local/" + UUID.randomUUID() + ".txt"); Resource[] resources = loader.getResources("local/*.txt"); assertTrue(resources.length >= 3); } finally { fs.delete(new Path("local/"), true); } } @Test public void testFilesMatchWithHomePrefix() throws Exception { try { TestUtils.writeToFS(loader, "local/" + UUID.randomUUID() + ".txt"); TestUtils.writeToFS(loader, "local/" + UUID.randomUUID() + ".txt"); TestUtils.writeToFS(loader, "local/" + UUID.randomUUID() + ".txt"); Resource[] resources = loader.getResources("~/local/*.txt"); assertTrue(resources.length >= 3); } finally { fs.delete(new Path("local/"), true); } } @Test public void testFilesMatchWithPrefix() throws Exception { try { TestUtils.writeToFS(loader, "local/" + UUID.randomUUID() + ".txt"); TestUtils.writeToFS(loader, "local/" + UUID.randomUUID() + ".txt"); TestUtils.writeToFS(loader, "local/" + UUID.randomUUID() + ".txt"); Resource[] resources = loader.getResources("./local/../local/*.txt"); assertTrue(resources.length >= 3); } finally { fs.delete(new Path("local/"), true); } } @Test public void testDecompressedStream() throws Exception { DefaultCodec codec = new DefaultCodec(); codec.setConf(fs.getConf()); String name = "local/" + UUID.randomUUID() + codec.getDefaultExtension(); OutputStream outputStream = codec.createOutputStream(fs.create(new Path(name))); byte[] content = name.getBytes(); outputStream.write(content); outputStream.close(); Resource resource = loader.getResource(name); assertNotNull(resource); InputStream inputStream = resource.getInputStream(); assertEquals(DecompressorStream.class, inputStream.getClass()); assertTrue(TestUtils.compareStreams(new ByteArrayInputStream(content), inputStream)); } @Test public void testCompressedStream() throws Exception { DefaultCodec codec = new DefaultCodec(); codec.setConf(fs.getConf()); String name = "local/" + UUID.randomUUID() + codec.getDefaultExtension(); OutputStream outputStream = codec.createOutputStream(fs.create(new Path(name))); byte[] content = name.getBytes(); outputStream.write(content); outputStream.close(); loader.setUseCodecs(false); try { Resource resource = loader.getResource(name); assertNotNull(resource); InputStream inputStream = resource.getInputStream(); System.out.println(inputStream.getClass()); assertFalse(DecompressorStream.class.equals(inputStream.getClass())); assertFalse(TestUtils.compareStreams(new ByteArrayInputStream(content), inputStream)); } finally { loader.setUseCodecs(true); } } @Test public void testURLCycle() throws Exception { ConfigurationFactoryBean cfactory; cfactory = new ConfigurationFactoryBean(); cfactory.setRegisterUrlHandler(true); cfactory.afterPropertiesSet(); /* create hdfs resource loader */ HdfsResourceLoader ldr = new HdfsResourceLoader(cfactory.getObject(), null, null); assertNotNull(ldr); } }