Java tutorial
/** * Copyright 2011-2016 Asakusa Framework Team. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.asakusafw.testdriver; import java.io.File; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter; import org.junit.Assert; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.asakusafw.compiler.flow.FlowDescriptionDriver; import com.asakusafw.compiler.flow.Location; import com.asakusafw.compiler.testing.DirectExporterDescription; import com.asakusafw.compiler.testing.DirectFlowCompiler; import com.asakusafw.compiler.testing.DirectImporterDescription; import com.asakusafw.compiler.testing.JobflowInfo; import com.asakusafw.runtime.util.VariableTable; import com.asakusafw.testdriver.hadoop.ConfigurationFactory; import com.asakusafw.testtools.TestUtils; import com.asakusafw.vocabulary.external.ExporterDescription; import com.asakusafw.vocabulary.external.ImporterDescription; import com.asakusafw.vocabulary.flow.FlowDescription; import com.asakusafw.vocabulary.flow.In; import com.asakusafw.vocabulary.flow.Out; import com.asakusafw.vocabulary.flow.graph.FlowGraph; /** * ??? * @deprecated legacy API */ @Deprecated public class FlowPartTestDriver extends TestDriverTestToolsBase { private static final Logger LOG = LoggerFactory.getLogger(FlowPartTestDriver.class); private final FlowDescriptionDriver flowDescriptionDriver = new FlowDescriptionDriver(); private final Map<String, List<String>> createInMap = new HashMap<>(); private final Map<String, List<String>> createOutMap = new HashMap<>(); private boolean createIndividually = false; private boolean loadIndividually = false; /** * * * @throws RuntimeException ?????? */ public FlowPartTestDriver() { super(); } /** * * * @param testDataFileList ??File? * @throws RuntimeException ?????? * @see FlowPartTestDriver#FlowPartTestDriver() */ public FlowPartTestDriver(List<File> testDataFileList) { super(testDataFileList); } /** * ???????? * * @param flowDescription ?? */ public void runTest(FlowDescription flowDescription) { try { // ? String batchId = "flow"; String flowId = "part"; File compileWorkDir = driverContext.getCompilerWorkingDirectory(); if (compileWorkDir.exists()) { FileUtils.forceDelete(compileWorkDir); } FlowGraph flowGraph = flowDescriptionDriver.createFlowGraph(flowDescription); JobflowInfo jobflowInfo = DirectFlowCompiler.compile(flowGraph, batchId, flowId, "test.flowpart", LegacyUtil.createWorkingLocation(driverContext), compileWorkDir, Arrays.asList(new File[] { DirectFlowCompiler.toLibraryPath(flowDescription.getClass()), }), flowDescription.getClass().getClassLoader(), LegacyUtil.toCompilerOptions(driverContext)); LegacyJobflowExecutor executor = new LegacyJobflowExecutor(driverContext); LegacyUtil.prepareIds(driverContext, jobflowInfo); // ? executor.cleanWorkingDirectory(); // ???Excel?????HDFS??? if (createIndividually) { prepareIndividually(); } else { createAllInput(); } executor.runJobflow(jobflowInfo); // ???Excel???? if (loadIndividually) { loadAndInspectResultsIndividually(); } else { loadAndInspectResults(); } } catch (Exception e) { throw new RuntimeException(e); } finally { driverContext.cleanUpTemporaryResources(); } } /** * ?????? * * @param <T> * @param tableName ?? * @return */ @SuppressWarnings("unchecked") public <T> In<T> createIn(String tableName) { Class<?> modelType = testUtils.getClassByTablename(tableName); return (In<T>) createIn(modelType); } /** * ?????? * * @param <T> * @param modelType * @return */ public <T> In<T> createIn(Class<T> modelType) { String tableName = testUtils.getTablenameByClass(modelType); addCreateIn(tableName, tableName); String path = LegacyUtil.createInputLocation(driverContext, tableName).toPath('/'); LOG.info("DirectImporterDescription?:Path=" + path); ImporterDescription desc = new DirectImporterDescription(modelType, path); return flowDescriptionDriver.createIn(tableName, desc); } /** * ?????? * <p> * ???????? ????? * </p> * * @param <T> * @param tableName ?? * @param excelFileName ??(.xls??) * @return */ @SuppressWarnings("unchecked") public <T> In<T> createIn(String tableName, String excelFileName) { Class<?> modelType = testUtils.getClassByTablename(tableName); return (In<T>) createIn(modelType, excelFileName); } /** * ?????? * <p> * ???????? ????? * </p> * * @param <T> * @param modelType * @param excelFileName ??(.xls??) * @return */ public <T> In<T> createIn(Class<T> modelType, String excelFileName) { String tableName = testUtils.getTablenameByClass(modelType); createIndividually = true; loadIndividually = true; addCreateIn(tableName, excelFileName); String path = LegacyUtil.createInputLocation(driverContext, excelFileName).toPath('/'); LOG.info("DirectImporterDescription?:Path=" + excelFileName); DirectImporterDescription desc = new DirectImporterDescription(modelType, path); String inName; int offset = excelFileName.lastIndexOf('/'); if (offset > -1) { inName = excelFileName.substring(offset + 1); } else { inName = excelFileName; } return flowDescriptionDriver.createIn(inName, desc); } /** * ?????? * * @param <T> * @param tableName ?? * @return */ @SuppressWarnings("unchecked") public <T> Out<T> createOut(String tableName) { Class<?> modelType = testUtils.getClassByTablename(tableName); return (Out<T>) createOut(modelType); } /** * ?????? * * @param <T> * @param modelType * @return */ public <T> Out<T> createOut(Class<T> modelType) { String tableName = testUtils.getTablenameByClass(modelType); addCreateOut(tableName, tableName); String path = LegacyUtil.createOutputLocation(driverContext, tableName).toPath('/'); LOG.info("DirectExporterDescription?:Path=" + path); ExporterDescription desc = new DirectExporterDescription(modelType, path); return flowDescriptionDriver.createOut(tableName, desc); } /** * ?????? * <p> * ???????? ????? * </p> * * @param <T> * @param tableName ?? * @param excelFileName ??(.xls??) * @return */ @SuppressWarnings("unchecked") public <T> Out<T> createOut(String tableName, String excelFileName) { Class<?> modelType = testUtils.getClassByTablename(tableName); return (Out<T>) createOut(modelType, excelFileName); } /** * ?????? * <p> * ???????? ????? * </p> * * @param <T> * @param modelType * @param excelFileName ??(.xls??) * @return */ public <T> Out<T> createOut(Class<T> modelType, String excelFileName) { String tableName = testUtils.getTablenameByClass(modelType); loadIndividually = true; addCreateOut(tableName, excelFileName); String path = LegacyUtil.createOutputLocation(driverContext, excelFileName).toPath('/'); LOG.info("DirectExporterDescription?:Path=" + excelFileName); DirectExporterDescription desc = new DirectExporterDescription(modelType, path); String outName; int offset = excelFileName.lastIndexOf(System.getProperty("file.separator")); if (offset > -1) { outName = excelFileName.substring(offset + 1); } else { outName = excelFileName; } return flowDescriptionDriver.createOut(outName, desc); } private void addCreateIn(String tableName, String fileName) { List<String> fileListPerTable = createInMap.get(tableName); if (fileListPerTable == null) { fileListPerTable = new ArrayList<>(); createInMap.put(tableName, fileListPerTable); } fileListPerTable.add(fileName); } private void addCreateOut(String tableName, String fileName) { List<String> fileListPerTable = createOutMap.get(tableName); if (fileListPerTable == null) { fileListPerTable = new ArrayList<>(); createOutMap.put(tableName, fileListPerTable); } fileListPerTable.add(fileName); } private void createAllInput() { for (String table : testUtils.getTablenames()) { createInput(table, table); } } private void prepareIndividually() { for (Map.Entry<String, List<String>> entry : createInMap.entrySet()) { String tablename = entry.getKey(); List<String> fileList = entry.getValue(); for (String excelFileName : fileList) { List<File> inFileList = new ArrayList<>(); if (testDataDir != null) { inFileList.add(new File(testDataDir, excelFileName + ".xls")); } else { for (File file : testDataFileList) { if (file.getPath().endsWith(excelFileName + ".xls")) { inFileList.add(file); break; } } } try { testUtils = new TestUtils(inFileList); } catch (IOException e) { throw new RuntimeException(e); } createInput(tablename, excelFileName); } } } private void createInput(String tablename, String excelFileName) { FileSystem fs = null; try { Configuration conf = ConfigurationFactory.getDefault().newInstance(); fs = FileSystem.get(conf); URI inputPath = new URI(computeInputPath(fs, excelFileName)); LOG.info("????:Path=" + inputPath); testUtils.storeToTemporary(tablename, conf, new Path(inputPath)); } catch (IOException e) { throw new RuntimeException(e); } catch (URISyntaxException e) { throw new RuntimeException(e); } } private void loadAndInspectResults() throws IOException { for (String table : testUtils.getTablenames()) { loadResult(table, table); } if (!testUtils.inspect()) { Assert.fail(testUtils.getCauseMessage()); } } private void loadAndInspectResultsIndividually() throws IOException { for (Map.Entry<String, List<String>> entry : createOutMap.entrySet()) { String tablename = entry.getKey(); List<String> fileList = entry.getValue(); for (String excelFileName : fileList) { List<File> outFileList = new ArrayList<>(); if (testDataDir != null) { outFileList.add(new File(testDataDir, excelFileName + ".xls")); } else { for (File file : testDataFileList) { if (file.getPath().endsWith(excelFileName + ".xls")) { outFileList.add(file); break; } } } try { testUtils = new TestUtils(outFileList); } catch (IOException e) { throw new RuntimeException(e); } loadResult(tablename, excelFileName); if (!testUtils.inspect()) { Assert.fail(testUtils.getCauseMessage()); return; } } } } private void loadResult(String tablename, String excelFileName) throws IOException { Configuration conf = ConfigurationFactory.getDefault().newInstance(); FileSystem fs = FileSystem.get(conf); FileStatus[] status = fs.globStatus(new Path(computeOutputPath(fs, excelFileName))); Path[] listedPaths = FileUtil.stat2Paths(status); for (Path path : listedPaths) { if (isSystemFile(path)) { continue; } LOG.info("????:Path=" + path); testUtils.loadFromTemporary(tablename, conf, path); } } private boolean isSystemFile(Path path) { assert path != null; String name = path.getName(); return name.equals(FileOutputCommitter.SUCCEEDED_FILE_NAME) || name.equals("_logs"); } private String computeInputPath(FileSystem fs, String tableName) { Location location = LegacyUtil.createInputLocation(driverContext, tableName); String path = new Path(fs.getWorkingDirectory(), location.toPath('/')).toString(); return resolvePath(path); } private String computeOutputPath(FileSystem fs, String tableName) { Location location = LegacyUtil.createOutputLocation(driverContext, tableName); String path = new Path(fs.getWorkingDirectory(), location.toPath('/')).toString(); return resolvePath(path); } private String resolvePath(String path) { assert path != null; Map<String, String> arguments = driverContext.getArguments(); VariableTable variables = new VariableTable(); variables.defineVariables(arguments); return variables.parse(path, false); } }