Java tutorial
/* * Copyright 2014 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.springframework.batch.integration.x; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import java.io.ByteArrayInputStream; import java.util.HashMap; import java.util.Map; import java.util.Properties; import org.apache.commons.net.ftp.FTPFile; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.junit.After; import org.junit.Before; import org.junit.ClassRule; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.batch.core.ExitStatus; import org.springframework.batch.core.Job; import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.JobParameter; import org.springframework.batch.core.JobParameters; import org.springframework.batch.core.launch.JobLauncher; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Configuration; import org.springframework.context.support.ClassPathXmlApplicationContext; import org.springframework.core.env.PropertiesPropertySource; import org.springframework.data.hadoop.test.context.HadoopDelegatingSmartContextLoader; import org.springframework.data.hadoop.test.context.MiniHadoopCluster; import org.springframework.integration.file.remote.session.Session; import org.springframework.integration.file.remote.session.SessionFactory; import org.springframework.messaging.MessageChannel; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import org.springframework.xd.dirt.integration.bus.MessageBus; import org.springframework.xd.dirt.integration.bus.local.LocalMessageBus; import org.springframework.xd.test.HostNotWindowsRule; /** * * @author Gary Russell */ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(loader = HadoopDelegatingSmartContextLoader.class, classes = RemoteFileToHadoopTests.EmptyConfig.class) @MiniHadoopCluster public class RemoteFileToHadoopTests { @ClassRule public static HostNotWindowsRule hostNotWindowsRule = new HostNotWindowsRule(); @Autowired private ApplicationContext context; @Autowired org.apache.hadoop.conf.Configuration configuration; private JobLauncher launcher; private Job job; private SessionFactory<?> sessionFactory; private MessageChannel requestsOut; private MessageChannel requestsIn; private MessageChannel repliesOut; private MessageChannel repliesIn; private MessageBus bus; @SuppressWarnings({ "unchecked", "rawtypes" }) @Before public void setup() throws Exception { byte[] bytes = "foobarbaz".getBytes(); // using this trick here by getting hadoop minicluster from main test // context and then using it to override 'hadoopConfiguration' bean // which is imported from ftphdfs.xml. ClassPathXmlApplicationContext ctx = new ClassPathXmlApplicationContext(); ctx.setConfigLocations("org/springframework/batch/integration/x/RemoteFileToHadoopTests-context.xml", "org/springframework/batch/integration/x/miniclusterconfig.xml"); Properties properties = new Properties(); properties.setProperty("restartable", "false"); properties.setProperty("xd.config.home", "file:../../config"); properties.setProperty("partitionResultsTimeout", "3600000"); PropertiesPropertySource propertiesPropertySource = new PropertiesPropertySource("props", properties); ctx.getEnvironment().getPropertySources().addLast(propertiesPropertySource); ctx.setParent(context); ctx.refresh(); this.sessionFactory = ctx.getBean(SessionFactory.class); Session session = mock(Session.class); when(this.sessionFactory.getSession()).thenReturn(session); when(session.readRaw("/foo/bar.txt")).thenReturn(new ByteArrayInputStream(bytes)); when(session.readRaw("/foo/baz.txt")).thenReturn(new ByteArrayInputStream(bytes)); when(session.finalizeRaw()).thenReturn(true); Object[] fileList = new FTPFile[2]; FTPFile file = new FTPFile(); file.setName("bar.txt"); fileList[0] = file; file = new FTPFile(); file.setName("baz.txt"); fileList[1] = file; when(session.list("/foo/")).thenReturn(fileList); this.launcher = ctx.getBean(JobLauncher.class); this.job = ctx.getBean(Job.class); this.requestsOut = ctx.getBean("stepExecutionRequests.output", MessageChannel.class); this.requestsIn = ctx.getBean("stepExecutionRequests.input", MessageChannel.class); this.repliesOut = ctx.getBean("stepExecutionReplies.output", MessageChannel.class); this.repliesIn = ctx.getBean("stepExecutionReplies.input", MessageChannel.class); this.bus = new LocalMessageBus(); ((LocalMessageBus) this.bus).setApplicationContext(ctx); this.bus.bindRequestor("foo", this.requestsOut, this.repliesIn, null); this.bus.bindReplier("foo", this.requestsIn, this.repliesOut, null); } @After public void tearDown() { this.bus.unbindConsumer("foo", this.requestsOut); this.bus.unbindConsumer("foo", this.requestsIn); this.bus.unbindConsumer("foo", this.repliesIn); this.bus.unbindConsumer("foo", this.repliesOut); } @Test public void testSimple() throws Exception { FileSystem fs = FileSystem.get(configuration); Path p1 = new Path("/qux/foo/bar.txt"); fs.delete(p1, true); Path p2 = new Path("/qux/foo/baz.txt"); fs.delete(p2, true); assertFalse(fs.exists(p1)); assertFalse(fs.exists(p2)); Map<String, JobParameter> params = new HashMap<String, JobParameter>(); params.put("remoteDirectory", new JobParameter("/foo/")); params.put("hdfsDirectory", new JobParameter("/qux")); JobParameters parameters = new JobParameters(params); JobExecution execution = launcher.run(job, parameters); assertEquals(ExitStatus.COMPLETED, execution.getExitStatus()); assertTrue(fs.exists(p1)); assertTrue(fs.exists(p2)); FSDataInputStream stream = fs.open(p1); byte[] out = new byte[9]; stream.readFully(out); stream.close(); assertEquals("foobarbaz", new String(out)); stream = fs.open(p2); stream.readFully(out); stream.close(); assertEquals("foobarbaz", new String(out)); } @Configuration static class EmptyConfig { } }