List of usage examples for org.apache.commons.configuration SubnodeConfiguration getString
public String getString(String key, String defaultValue)
From source file:com.oltpbenchmark.multitenancy.MuTeBench.java
/** * @param args// ww w. ja va 2s . c o m * @throws Exception */ public static void main(String[] args) throws Exception { String duration = null; String scenarioFile = null; // ------------------------------------------------------------------- // INITIALIZE LOGGING // ------------------------------------------------------------------- String log4jPath = System.getProperty("log4j.configuration"); if (log4jPath != null) { org.apache.log4j.PropertyConfigurator.configure(log4jPath); } else { throw new RuntimeException("Missing log4j.properties file"); } // ------------------------------------------------------------------- // PARSE COMMAND LINE PARAMETERS // ------------------------------------------------------------------- CommandLineParser parser = new PosixParser(); XMLConfiguration pluginConfig = null; try { pluginConfig = new XMLConfiguration("config/plugin.xml"); } catch (ConfigurationException e1) { LOG.info("Plugin configuration file config/plugin.xml is missing"); e1.printStackTrace(); } pluginConfig.setExpressionEngine(new XPathExpressionEngine()); Options options = new Options(); options.addOption("s", "scenario", true, "[required] Workload scenario file"); options.addOption("a", "analysis-buckets", true, "sampling buckets for result aggregation"); options.addOption("r", "runtime", true, "maximum runtime (no events will be started after finishing runtime)"); options.addOption("v", "verbose", false, "Display Messages"); options.addOption("g", "gui", false, "Show controlling GUI"); options.addOption("h", "help", false, "Print this help"); options.addOption("o", "output", true, "Output file (default System.out)"); options.addOption("b", "baseline", true, "Output files of previous baseline run"); options.addOption(null, "histograms", false, "Print txn histograms"); options.addOption("d", "dialects-export", true, "Export benchmark SQL to a dialects file"); // parse the command line arguments CommandLine argsLine = parser.parse(options, args); if (argsLine.hasOption("h")) { printUsage(options); return; } else if (!argsLine.hasOption("scenario")) { INIT_LOG.fatal("Missing scenario description file"); System.exit(-1); } else scenarioFile = argsLine.getOptionValue("scenario"); if (argsLine.hasOption("r")) duration = argsLine.getOptionValue("r"); if (argsLine.hasOption("runtime")) duration = argsLine.getOptionValue("runtime"); // ------------------------------------------------------------------- // CREATE TENANT SCHEDULE // ------------------------------------------------------------------- INIT_LOG.info("Create schedule"); Schedule schedule = new Schedule(duration, scenarioFile); HashMap<Integer, ScheduleEvents> tenantEvents = schedule.getTenantEvents(); ArrayList<Integer> tenantList = schedule.getTenantList(); List<BenchmarkModule> benchList = new ArrayList<BenchmarkModule>(); for (int tenInd = 0; tenInd < tenantList.size(); tenInd++) { int tenantID = tenantList.get(tenInd); for (int tenEvent = 0; tenEvent < tenantEvents.get(tenantID).size(); tenEvent++) { BenchmarkSettings benchmarkSettings = (BenchmarkSettings) tenantEvents.get(tenantID) .getBenchmarkSettings(tenEvent); // update benchmark Settings benchmarkSettings.setTenantID(tenantID); // ------------------------------------------------------------------- // GET PLUGIN LIST // ------------------------------------------------------------------- String plugins = benchmarkSettings.getBenchmark(); String[] pluginList = plugins.split(","); String configFile = benchmarkSettings.getConfigFile(); XMLConfiguration xmlConfig = new XMLConfiguration(configFile); xmlConfig.setExpressionEngine(new XPathExpressionEngine()); int lastTxnId = 0; for (String plugin : pluginList) { // ---------------------------------------------------------------- // WORKLOAD CONFIGURATION // ---------------------------------------------------------------- String pluginTest = ""; pluginTest = "[@bench='" + plugin + "']"; WorkloadConfiguration wrkld = new WorkloadConfiguration(); wrkld.setTenantId(tenantID); wrkld.setBenchmarkName(setTenantIDinString(plugin, tenantID)); wrkld.setXmlConfig(xmlConfig); wrkld.setDBType(DatabaseType.get(setTenantIDinString(xmlConfig.getString("dbtype"), tenantID))); wrkld.setDBDriver(setTenantIDinString(xmlConfig.getString("driver"), tenantID)); wrkld.setDBConnection(setTenantIDinString(xmlConfig.getString("DBUrl"), tenantID)); wrkld.setDBName(setTenantIDinString(xmlConfig.getString("DBName"), tenantID)); wrkld.setDBUsername(setTenantIDinString(xmlConfig.getString("username"), tenantID)); wrkld.setDBPassword(setTenantIDinString(xmlConfig.getString("password"), tenantID)); String terminalString = setTenantIDinString(xmlConfig.getString("terminals[not(@bench)]", "0"), tenantID); int terminals = Integer.parseInt(xmlConfig.getString("terminals" + pluginTest, terminalString)); wrkld.setTerminals(terminals); int taSize = Integer.parseInt(xmlConfig.getString("taSize", "1")); if (taSize < 0) INIT_LOG.fatal("taSize must not be negative!"); wrkld.setTaSize(taSize); wrkld.setProprietaryTaSyntax(xmlConfig.getBoolean("proprietaryTaSyntax", false)); wrkld.setIsolationMode(setTenantIDinString( xmlConfig.getString("isolation", "TRANSACTION_SERIALIZABLE"), tenantID)); wrkld.setScaleFactor(Double .parseDouble(setTenantIDinString(xmlConfig.getString("scalefactor", "1.0"), tenantID))); wrkld.setRecordAbortMessages(xmlConfig.getBoolean("recordabortmessages", false)); int size = xmlConfig.configurationsAt("/works/work").size(); for (int i = 1; i < size + 1; i++) { SubnodeConfiguration work = xmlConfig.configurationAt("works/work[" + i + "]"); List<String> weight_strings; // use a workaround if there multiple workloads or // single // attributed workload if (pluginList.length > 1 || work.containsKey("weights[@bench]")) { weight_strings = get_weights(plugin, work); } else { weight_strings = work.getList("weights[not(@bench)]"); } int rate = 1; boolean rateLimited = true; boolean disabled = false; // can be "disabled", "unlimited" or a number String rate_string; rate_string = setTenantIDinString(work.getString("rate[not(@bench)]", ""), tenantID); rate_string = setTenantIDinString(work.getString("rate" + pluginTest, rate_string), tenantID); if (rate_string.equals(RATE_DISABLED)) { disabled = true; } else if (rate_string.equals(RATE_UNLIMITED)) { rateLimited = false; } else if (rate_string.isEmpty()) { LOG.fatal(String.format( "Tenant " + tenantID + ": Please specify the rate for phase %d and workload %s", i, plugin)); System.exit(-1); } else { try { rate = Integer.parseInt(rate_string); if (rate < 1) { LOG.fatal("Tenant " + tenantID + ": Rate limit must be at least 1. Use unlimited or disabled values instead."); System.exit(-1); } } catch (NumberFormatException e) { LOG.fatal(String.format( "Tenant " + tenantID + ": Rate string must be '%s', '%s' or a number", RATE_DISABLED, RATE_UNLIMITED)); System.exit(-1); } } Phase.Arrival arrival = Phase.Arrival.REGULAR; String arrive = setTenantIDinString(work.getString("@arrival", "regular"), tenantID); if (arrive.toUpperCase().equals("POISSON")) arrival = Phase.Arrival.POISSON; int activeTerminals; activeTerminals = Integer.parseInt(setTenantIDinString( work.getString("active_terminals[not(@bench)]", String.valueOf(terminals)), tenantID)); activeTerminals = Integer.parseInt(setTenantIDinString( work.getString("active_terminals" + pluginTest, String.valueOf(activeTerminals)), tenantID)); if (activeTerminals > terminals) { LOG.fatal("Tenant " + tenantID + ": Configuration error in work " + i + ": number of active terminals" + "" + "is bigger than the total number of terminals"); System.exit(-1); } wrkld.addWork(Integer.parseInt(setTenantIDinString(work.getString("/time"), tenantID)), rate, weight_strings, rateLimited, disabled, activeTerminals, arrival); } // FOR int numTxnTypes = xmlConfig .configurationsAt("transactiontypes" + pluginTest + "/transactiontype").size(); if (numTxnTypes == 0 && pluginList.length == 1) { // if it is a single workload run, <transactiontypes /> // w/o attribute is used pluginTest = "[not(@bench)]"; numTxnTypes = xmlConfig .configurationsAt("transactiontypes" + pluginTest + "/transactiontype").size(); } wrkld.setNumTxnTypes(numTxnTypes); // CHECKING INPUT PHASES int j = 0; for (Phase p : wrkld.getAllPhases()) { j++; if (p.getWeightCount() != wrkld.getNumTxnTypes()) { LOG.fatal(String.format("Tenant " + tenantID + ": Configuration files is inconsistent, phase %d contains %d weights but you defined %d transaction types", j, p.getWeightCount(), wrkld.getNumTxnTypes())); System.exit(-1); } } // FOR // Generate the dialect map wrkld.init(); assert (wrkld.getNumTxnTypes() >= 0); assert (xmlConfig != null); // ---------------------------------------------------------------- // BENCHMARK MODULE // ---------------------------------------------------------------- String classname = pluginConfig.getString("/plugin[@name='" + plugin + "']"); if (classname == null) { throw new ParseException("Plugin " + plugin + " is undefined in config/plugin.xml"); } BenchmarkModule bench = ClassUtil.newInstance(classname, new Object[] { wrkld }, new Class<?>[] { WorkloadConfiguration.class }); assert (benchList.get(0) != null); Map<String, Object> initDebug = new ListOrderedMap<String, Object>(); initDebug.put("Benchmark", String.format("%s {%s}", plugin.toUpperCase(), classname)); initDebug.put("Configuration", configFile); initDebug.put("Type", wrkld.getDBType()); initDebug.put("Driver", wrkld.getDBDriver()); initDebug.put("URL", wrkld.getDBConnection()); initDebug.put("Isolation", setTenantIDinString( xmlConfig.getString("isolation", "TRANSACTION_SERIALIZABLE [DEFAULT]"), tenantID)); initDebug.put("Scale Factor", wrkld.getScaleFactor()); INIT_LOG.info(SINGLE_LINE + "\n\n" + StringUtil.formatMaps(initDebug)); INIT_LOG.info(SINGLE_LINE); // Load TransactionTypes List<TransactionType> ttypes = new ArrayList<TransactionType>(); // Always add an INVALID type for Carlo ttypes.add(TransactionType.INVALID); int txnIdOffset = lastTxnId; for (int i = 1; i < wrkld.getNumTxnTypes() + 1; i++) { String key = "transactiontypes" + pluginTest + "/transactiontype[" + i + "]"; String txnName = setTenantIDinString(xmlConfig.getString(key + "/name"), tenantID); int txnId = i + 1; if (xmlConfig.containsKey(key + "/id")) { txnId = Integer .parseInt(setTenantIDinString(xmlConfig.getString(key + "/id"), tenantID)); } ttypes.add(bench.initTransactionType(txnName, txnId + txnIdOffset)); lastTxnId = i; } // FOR TransactionTypes tt = new TransactionTypes(ttypes); wrkld.setTransTypes(tt); if (benchmarkSettings.getBenchmarkSlaFile() != null) wrkld.setSlaFromFile(benchmarkSettings.getBenchmarkSlaFile()); LOG.debug("Tenant " + tenantID + ": Using the following transaction types: " + tt); bench.setTenantOffset(tenantEvents.get(tenantID).getTime(tenEvent)); bench.setTenantID(tenantID); bench.setBenchmarkSettings(benchmarkSettings); benchList.add(bench); } } } // create result collector ResultCollector rCollector = new ResultCollector(tenantList); // execute benchmarks in parallel ArrayList<Thread> benchThreads = new ArrayList<Thread>(); for (BenchmarkModule benchmark : benchList) { BenchmarkExecutor benchThread = new BenchmarkExecutor(benchmark, argsLine); Thread t = new Thread(benchThread); t.start(); benchThreads.add(t); benchmark.getWorkloadConfiguration().setrCollector(rCollector); } // waiting for completion of all benchmarks for (Thread t : benchThreads) { t.join(); } // print statistics int analysisBuckets = -1; if (argsLine.hasOption("analysis-buckets")) analysisBuckets = Integer.parseInt(argsLine.getOptionValue("analysis-buckets")); String output = null; if (argsLine.hasOption("o")) output = argsLine.getOptionValue("o"); String baseline = null; if (argsLine.hasOption("b")) baseline = argsLine.getOptionValue("b"); rCollector.printStatistics(output, analysisBuckets, argsLine.hasOption("histograms"), baseline); // create GUI if (argsLine.hasOption("g") && (!rCollector.getAllResults().isEmpty())) { try { Gui gui = new Gui(Integer.parseInt(argsLine.getOptionValue("analysis-buckets", "10")), rCollector, output); } catch (Exception e) { e.printStackTrace(); } } }
From source file:com.oltpbenchmark.DBWorkload.java
/** * @param args// www. java2 s. c o m * @throws Exception */ public static void main(String[] args) throws Exception { // Initialize log4j String log4jPath = System.getProperty("log4j.configuration"); if (log4jPath != null) { org.apache.log4j.PropertyConfigurator.configure(log4jPath); } else { throw new RuntimeException("Missing log4j.properties file"); } // create the command line parser CommandLineParser parser = new PosixParser(); XMLConfiguration pluginConfig = null; try { pluginConfig = new XMLConfiguration("config/plugin.xml"); } catch (ConfigurationException e1) { LOG.info("Plugin configuration file config/plugin.xml is missing"); e1.printStackTrace(); } pluginConfig.setExpressionEngine(new XPathExpressionEngine()); Options options = new Options(); options.addOption("b", "bench", true, "[required] Benchmark class. Currently supported: " + pluginConfig.getList("/plugin//@name")); options.addOption("c", "config", true, "[required] Workload configuration file"); options.addOption(null, "create", true, "Initialize the database for this benchmark"); options.addOption(null, "clear", true, "Clear all records in the database for this benchmark"); options.addOption(null, "load", true, "Load data using the benchmark's data loader"); options.addOption(null, "execute", true, "Execute the benchmark workload"); options.addOption(null, "runscript", true, "Run an SQL script"); options.addOption(null, "upload", true, "Upload the result"); options.addOption("v", "verbose", false, "Display Messages"); options.addOption("h", "help", false, "Print this help"); options.addOption("s", "sample", true, "Sampling window"); options.addOption("ss", false, "Verbose Sampling per Transaction"); options.addOption("o", "output", true, "Output file (default System.out)"); options.addOption("d", "directory", true, "Base directory for the result files, default is current directory"); options.addOption("t", "timestamp", false, "Each result file is prepended with a timestamp for the beginning of the experiment"); options.addOption(null, "histograms", false, "Print txn histograms"); options.addOption(null, "dialects-export", true, "Export benchmark SQL to a dialects file"); // parse the command line arguments CommandLine argsLine = parser.parse(options, args); if (argsLine.hasOption("h")) { printUsage(options); return; } else if (argsLine.hasOption("c") == false) { LOG.error("Missing Configuration file"); printUsage(options); return; } else if (argsLine.hasOption("b") == false) { LOG.fatal("Missing Benchmark Class to load"); printUsage(options); return; } // If an output directory is used, store the information String outputDirectory = ""; if (argsLine.hasOption("d")) { outputDirectory = argsLine.getOptionValue("d"); } String timestampValue = ""; if (argsLine.hasOption("t")) { timestampValue = String.valueOf(TimeUtil.getCurrentTime().getTime()) + "_"; } // ------------------------------------------------------------------- // GET PLUGIN LIST // ------------------------------------------------------------------- String plugins = argsLine.getOptionValue("b"); String[] pluginList = plugins.split(","); List<BenchmarkModule> benchList = new ArrayList<BenchmarkModule>(); // Use this list for filtering of the output List<TransactionType> activeTXTypes = new ArrayList<TransactionType>(); String configFile = argsLine.getOptionValue("c"); XMLConfiguration xmlConfig = new XMLConfiguration(configFile); xmlConfig.setExpressionEngine(new XPathExpressionEngine()); int lastTxnId = 0; for (String plugin : pluginList) { // ---------------------------------------------------------------- // WORKLOAD CONFIGURATION // ---------------------------------------------------------------- String pluginTest = ""; pluginTest = "[@bench='" + plugin + "']"; WorkloadConfiguration wrkld = new WorkloadConfiguration(); wrkld.setBenchmarkName(plugin); wrkld.setXmlConfig(xmlConfig); wrkld.setDBType(DatabaseType.get(xmlConfig.getString("dbtype"))); wrkld.setDBDriver(xmlConfig.getString("driver")); wrkld.setDBConnection(xmlConfig.getString("DBUrl")); wrkld.setDBName(xmlConfig.getString("DBName")); wrkld.setDBUsername(xmlConfig.getString("username")); wrkld.setDBPassword(xmlConfig.getString("password")); int terminals = xmlConfig.getInt("terminals[not(@bench)]", 0); terminals = xmlConfig.getInt("terminals" + pluginTest, terminals); wrkld.setTerminals(terminals); wrkld.setIsolationMode(xmlConfig.getString("isolation", "TRANSACTION_SERIALIZABLE")); wrkld.setScaleFactor(xmlConfig.getDouble("scalefactor", 1.0)); wrkld.setRecordAbortMessages(xmlConfig.getBoolean("recordabortmessages", false)); int size = xmlConfig.configurationsAt("/works/work").size(); for (int i = 1; i < size + 1; i++) { SubnodeConfiguration work = xmlConfig.configurationAt("works/work[" + i + "]"); List<String> weight_strings; // use a workaround if there multiple workloads or single // attributed workload if (pluginList.length > 1 || work.containsKey("weights[@bench]")) { weight_strings = get_weights(plugin, work); } else { weight_strings = work.getList("weights[not(@bench)]"); } int rate = 1; boolean rateLimited = true; boolean disabled = false; // can be "disabled", "unlimited" or a number String rate_string; rate_string = work.getString("rate[not(@bench)]", ""); rate_string = work.getString("rate" + pluginTest, rate_string); if (rate_string.equals(RATE_DISABLED)) { disabled = true; } else if (rate_string.equals(RATE_UNLIMITED)) { rateLimited = false; } else if (rate_string.isEmpty()) { LOG.fatal(String.format("Please specify the rate for phase %d and workload %s", i, plugin)); System.exit(-1); } else { try { rate = Integer.parseInt(rate_string); if (rate < 1) { LOG.fatal("Rate limit must be at least 1. Use unlimited or disabled values instead."); System.exit(-1); } } catch (NumberFormatException e) { LOG.fatal(String.format("Rate string must be '%s', '%s' or a number", RATE_DISABLED, RATE_UNLIMITED)); System.exit(-1); } } Phase.Arrival arrival = Phase.Arrival.REGULAR; String arrive = work.getString("@arrival", "regular"); if (arrive.toUpperCase().equals("POISSON")) arrival = Phase.Arrival.POISSON; int activeTerminals; activeTerminals = work.getInt("active_terminals[not(@bench)]", terminals); activeTerminals = work.getInt("active_terminals" + pluginTest, activeTerminals); if (activeTerminals > terminals) { System.out.println("Configuration error in work " + i + ": number of active terminals" + "" + "is bigger than the total number of terminals"); System.exit(-1); } wrkld.addWork(work.getInt("/time"), rate, weight_strings, rateLimited, disabled, activeTerminals, arrival); } // FOR int numTxnTypes = xmlConfig.configurationsAt("transactiontypes" + pluginTest + "/transactiontype") .size(); if (numTxnTypes == 0 && pluginList.length == 1) { //if it is a single workload run, <transactiontypes /> w/o attribute is used pluginTest = "[not(@bench)]"; numTxnTypes = xmlConfig.configurationsAt("transactiontypes" + pluginTest + "/transactiontype") .size(); } wrkld.setNumTxnTypes(numTxnTypes); // CHECKING INPUT PHASES int j = 0; for (Phase p : wrkld.getAllPhases()) { j++; if (p.getWeightCount() != wrkld.getNumTxnTypes()) { LOG.fatal(String.format( "Configuration files is inconsistent, phase %d contains %d weights but you defined %d transaction types", j, p.getWeightCount(), wrkld.getNumTxnTypes())); System.exit(-1); } } // FOR // Generate the dialect map wrkld.init(); assert (wrkld.getNumTxnTypes() >= 0); assert (xmlConfig != null); // ---------------------------------------------------------------- // BENCHMARK MODULE // ---------------------------------------------------------------- String classname = pluginConfig.getString("/plugin[@name='" + plugin + "']"); if (classname == null) { throw new ParseException("Plugin " + plugin + " is undefined in config/plugin.xml"); } BenchmarkModule bench = ClassUtil.newInstance(classname, new Object[] { wrkld }, new Class<?>[] { WorkloadConfiguration.class }); assert (benchList.get(0) != null); Map<String, Object> initDebug = new ListOrderedMap<String, Object>(); initDebug.put("Benchmark", String.format("%s {%s}", plugin.toUpperCase(), classname)); initDebug.put("Configuration", configFile); initDebug.put("Type", wrkld.getDBType()); initDebug.put("Driver", wrkld.getDBDriver()); initDebug.put("URL", wrkld.getDBConnection()); initDebug.put("Isolation", xmlConfig.getString("isolation", "TRANSACTION_SERIALIZABLE [DEFAULT]")); initDebug.put("Scale Factor", wrkld.getScaleFactor()); INIT_LOG.info(SINGLE_LINE + "\n\n" + StringUtil.formatMaps(initDebug)); INIT_LOG.info(SINGLE_LINE); // Load TransactionTypes List<TransactionType> ttypes = new ArrayList<TransactionType>(); // Always add an INVALID type for Carlo ttypes.add(TransactionType.INVALID); int txnIdOffset = lastTxnId; for (int i = 1; i < wrkld.getNumTxnTypes() + 1; i++) { String key = "transactiontypes" + pluginTest + "/transactiontype[" + i + "]"; String txnName = xmlConfig.getString(key + "/name"); int txnId = i + 1; if (xmlConfig.containsKey(key + "/id")) { txnId = xmlConfig.getInt(key + "/id"); } TransactionType tmpType = bench.initTransactionType(txnName, txnId + txnIdOffset); // Keep a reference for filtering activeTXTypes.add(tmpType); // Add a reference for the active TTypes in this benchmark ttypes.add(tmpType); lastTxnId = i; } // FOR TransactionTypes tt = new TransactionTypes(ttypes); wrkld.setTransTypes(tt); LOG.debug("Using the following transaction types: " + tt); benchList.add(bench); } // Export StatementDialects if (isBooleanOptionSet(argsLine, "dialects-export")) { BenchmarkModule bench = benchList.get(0); if (bench.getStatementDialects() != null) { LOG.info("Exporting StatementDialects for " + bench); String xml = bench.getStatementDialects().export(bench.getWorkloadConfiguration().getDBType(), bench.getProcedures().values()); System.out.println(xml); System.exit(0); } throw new RuntimeException("No StatementDialects is available for " + bench); } @Deprecated boolean verbose = argsLine.hasOption("v"); // Create the Benchmark's Database if (isBooleanOptionSet(argsLine, "create")) { for (BenchmarkModule benchmark : benchList) { CREATE_LOG.info("Creating new " + benchmark.getBenchmarkName().toUpperCase() + " database..."); runCreator(benchmark, verbose); CREATE_LOG.info("Finished!"); CREATE_LOG.info(SINGLE_LINE); } } else if (CREATE_LOG.isDebugEnabled()) { CREATE_LOG.debug("Skipping creating benchmark database tables"); CREATE_LOG.info(SINGLE_LINE); } // Clear the Benchmark's Database if (isBooleanOptionSet(argsLine, "clear")) { for (BenchmarkModule benchmark : benchList) { CREATE_LOG.info("Resetting " + benchmark.getBenchmarkName().toUpperCase() + " database..."); benchmark.clearDatabase(); CREATE_LOG.info("Finished!"); CREATE_LOG.info(SINGLE_LINE); } } else if (CREATE_LOG.isDebugEnabled()) { CREATE_LOG.debug("Skipping creating benchmark database tables"); CREATE_LOG.info(SINGLE_LINE); } // Execute Loader if (isBooleanOptionSet(argsLine, "load")) { for (BenchmarkModule benchmark : benchList) { LOAD_LOG.info("Loading data into " + benchmark.getBenchmarkName().toUpperCase() + " database..."); runLoader(benchmark, verbose); LOAD_LOG.info("Finished!"); LOAD_LOG.info(SINGLE_LINE); } } else if (LOAD_LOG.isDebugEnabled()) { LOAD_LOG.debug("Skipping loading benchmark database records"); LOAD_LOG.info(SINGLE_LINE); } // Execute a Script if (argsLine.hasOption("runscript")) { for (BenchmarkModule benchmark : benchList) { String script = argsLine.getOptionValue("runscript"); SCRIPT_LOG.info("Running a SQL script: " + script); runScript(benchmark, script); SCRIPT_LOG.info("Finished!"); SCRIPT_LOG.info(SINGLE_LINE); } } // Execute Workload if (isBooleanOptionSet(argsLine, "execute")) { // Bombs away! Results r = null; try { r = runWorkload(benchList, verbose); } catch (Throwable ex) { LOG.error("Unexpected error when running benchmarks.", ex); System.exit(1); } assert (r != null); PrintStream ps = System.out; PrintStream rs = System.out; ResultUploader ru = new ResultUploader(r, xmlConfig, argsLine); if (argsLine.hasOption("o")) { // Check if directory needs to be created if (outputDirectory.length() > 0) { FileUtil.makeDirIfNotExists(outputDirectory.split("/")); } // Build the complex path String baseFile = timestampValue + argsLine.getOptionValue("o"); // Increment the filename for new results String nextName = FileUtil.getNextFilename(FileUtil.joinPath(outputDirectory, baseFile + ".res")); ps = new PrintStream(new File(nextName)); EXEC_LOG.info("Output into file: " + nextName); nextName = FileUtil.getNextFilename(FileUtil.joinPath(outputDirectory, baseFile + ".raw")); rs = new PrintStream(new File(nextName)); EXEC_LOG.info("Output Raw data into file: " + nextName); nextName = FileUtil.getNextFilename(FileUtil.joinPath(outputDirectory, baseFile + ".summary")); PrintStream ss = new PrintStream(new File(nextName)); EXEC_LOG.info("Output summary data into file: " + nextName); ru.writeSummary(ss); ss.close(); nextName = FileUtil.getNextFilename(FileUtil.joinPath(outputDirectory, baseFile + ".db.cnf")); ss = new PrintStream(new File(nextName)); EXEC_LOG.info("Output db config into file: " + nextName); ru.writeDBParameters(ss); ss.close(); nextName = FileUtil.getNextFilename(FileUtil.joinPath(outputDirectory, baseFile + ".ben.cnf")); ss = new PrintStream(new File(nextName)); EXEC_LOG.info("Output benchmark config into file: " + nextName); ru.writeBenchmarkConf(ss); ss.close(); } else if (EXEC_LOG.isDebugEnabled()) { EXEC_LOG.debug("No output file specified"); } if (argsLine.hasOption("s")) { int windowSize = Integer.parseInt(argsLine.getOptionValue("s")); EXEC_LOG.info("Grouped into Buckets of " + windowSize + " seconds"); r.writeCSV(windowSize, ps); if (isBooleanOptionSet(argsLine, "upload")) { ru.uploadResult(); } // Allow more detailed reporting by transaction to make it easier to check if (argsLine.hasOption("ss")) { for (TransactionType t : activeTXTypes) { PrintStream ts = ps; if (ts != System.out) { // Get the actual filename for the output String baseFile = timestampValue + argsLine.getOptionValue("o") + "_" + t.getName(); String prepended = outputDirectory + timestampValue; String nextName = FileUtil .getNextFilename(FileUtil.joinPath(outputDirectory, baseFile + ".res")); ts = new PrintStream(new File(nextName)); r.writeCSV(windowSize, ts, t); ts.close(); } } } } else if (EXEC_LOG.isDebugEnabled()) { EXEC_LOG.warn("No bucket size specified"); } if (argsLine.hasOption("histograms")) { EXEC_LOG.info(SINGLE_LINE); EXEC_LOG.info("Completed Transactions:\n" + r.getTransactionSuccessHistogram() + "\n"); EXEC_LOG.info("Aborted Transactions:\n" + r.getTransactionAbortHistogram() + "\n"); EXEC_LOG.info("Rejected Transactions:\n" + r.getTransactionRetryHistogram()); EXEC_LOG.info("Unexpected Errors:\n" + r.getTransactionErrorHistogram()); if (r.getTransactionAbortMessageHistogram().isEmpty() == false) EXEC_LOG.info( "User Aborts:\n" + StringUtil.formatMaps(r.getTransactionAbortMessageHistogram())); } else if (EXEC_LOG.isDebugEnabled()) { EXEC_LOG.warn("No bucket size specified"); } r.writeAllCSVAbsoluteTiming(rs); ps.close(); rs.close(); } else { EXEC_LOG.info("Skipping benchmark workload execution"); } }
From source file:edu.stolaf.cs.wmrserver.TransformProcessor.java
private File getLibraryDirectory(SubnodeConfiguration languageConf) { String libDirString = languageConf.getString("library", ""); if (libDirString.isEmpty()) return null; File libDir = new File(libDirString); if (!libDir.isAbsolute()) libDir = new File(m_languageConfDir, libDirString); return libDir; }
From source file:edu.stolaf.cs.wmrserver.TransformProcessor.java
private File compile(SubnodeConfiguration languageConf, String transformTypeString, File srcTransformFile, File jobTransformDir) throws CompilationException, IOException { // Determine correct compiler, returning if none specified String compiler = languageConf.getString("compiler-" + transformTypeString, ""); if (compiler.isEmpty()) compiler = languageConf.getString("compiler", ""); if (compiler.isEmpty()) return srcTransformFile; // Determine destination filename File compiledTransformFile = new File(jobTransformDir, "compiled-job-" + transformTypeString); // Create map to replace ${wmr:...} variables. // NOTE: Commons Configuration's built-in interpolator does not work here // for some reason. File jobTempDir = getJobTempDir(); Hashtable<String, String> variableMap = new Hashtable<String, String>(); File libDir = getLibraryDirectory(languageConf); variableMap.put("wmr:lib.dir", (libDir != null) ? libDir.toString() : ""); variableMap.put("wmr:src.dir", relativizeFile(srcTransformFile.getParentFile(), jobTempDir).toString()); variableMap.put("wmr:src.file", relativizeFile(srcTransformFile, jobTempDir).toString()); variableMap.put("wmr:dest.dir", relativizeFile(jobTransformDir, jobTempDir).toString()); variableMap.put("wmr:dest.file", relativizeFile(compiledTransformFile, jobTempDir).toString()); // Replace variables in compiler string compiler = StrSubstitutor.replace(compiler, variableMap); // Run the compiler CommandLine compilerCommand = CommandLine.parse(compiler); DefaultExecutor exec = new DefaultExecutor(); ExecuteWatchdog dog = new ExecuteWatchdog(60000); // 1 minute ByteArrayOutputStream output = new ByteArrayOutputStream(); PumpStreamHandler pump = new PumpStreamHandler(output); exec.setWorkingDirectory(jobTempDir); exec.setWatchdog(dog);// w w w.j av a 2s. c o m exec.setStreamHandler(pump); exec.setExitValues(null); // Can't get the exit code if it throws exception int exitStatus = -1; try { exitStatus = exec.execute(compilerCommand); } catch (IOException ex) { // NOTE: Exit status is still -1 in this case, since exception was thrown throw new CompilationException("Compiling failed for " + transformTypeString, exitStatus, new String(output.toByteArray())); } // Check for successful exit if (exitStatus != 0) throw new CompilationException("Compiling failed for " + transformTypeString, exitStatus, new String(output.toByteArray())); // Check that output exists and is readable, and make it executable if (!compiledTransformFile.isFile()) throw new CompilationException( "Compiler did not output a " + transformTypeString + " executable (or it was not a regular file).", exitStatus, new String(output.toByteArray())); if (!compiledTransformFile.canRead()) throw new IOException(StringUtils.capitalize(transformTypeString) + " executable output from compiler was not readable: " + compiledTransformFile.toString()); if (!compiledTransformFile.canExecute()) compiledTransformFile.setExecutable(true, false); return compiledTransformFile; }
From source file:edu.kit.dama.scheduler.servlet.JobSchedulerInitializerListener.java
/** * Load configuration from XML-File/* w w w . j a v a 2 s . c o m*/ */ private void loadConfiguration() { URL configURL; HierarchicalConfiguration hc; try { configURL = DataManagerSettings.getConfigurationURL(); LOGGER.debug("Loading configuration from {}", configURL); hc = new HierarchicalConfiguration(new XMLConfiguration(configURL)); LOGGER.debug("Configuration successfully loaded"); } catch (ConfigurationException ex) { // error in configuration // reason see debug log message: LOGGER.warn("Failed to load configuration, using default values.", ex); loadDefaultConfiguration(); return; } SubnodeConfiguration configurationAt = hc.configurationAt(CONFIG_ROOT); if (configurationAt != null) { String jobStoreDriverParam = null; try { jobStoreDriverParam = configurationAt.getString(JOB_STORE_DRIVER, null); } catch (ConversionException ex) { LOGGER.error("Failed to parse parameter '{}'.", JOB_STORE_DRIVER); } if (jobStoreDriverParam != null) { jobStoreDriver = jobStoreDriverParam; } else { LOGGER.info("No parameter '{}' defined, defaulting to {}.", JOB_STORE_DRIVER, DEFAULT_JOB_STORE_DRIVER); jobStoreDriver = DEFAULT_JOB_STORE_DRIVER; } String jobStoreConnectionStringParam = null; try { jobStoreConnectionStringParam = configurationAt.getString(JOB_STORE_CONNECTION_STRING, null); } catch (ConversionException ex) { LOGGER.error("Failed to parse parameter '{}'.", JOB_STORE_CONNECTION_STRING); } if (jobStoreConnectionStringParam != null) { jobStoreConnectionString = jobStoreConnectionStringParam; } else { LOGGER.info("No parameter '{}' defined, defaulting to {}.", JOB_STORE_CONNECTION_STRING, DEFAULT_JOB_STORE_CONNECTION_STRING); jobStoreConnectionString = DEFAULT_JOB_STORE_CONNECTION_STRING; } String jobStoreUserParam = null; try { jobStoreUserParam = configurationAt.getString(JOB_STORE_USER, null); } catch (ConversionException ex) { LOGGER.error("Failed to parse parameter '{}'.", JOB_STORE_USER); } if (jobStoreUserParam != null) { jobStoreUser = jobStoreUserParam; } else { LOGGER.info("No parameter '{}' defined, defaulting to {}.", JOB_STORE_USER, DEFAULT_JOB_STORE_USER); jobStoreUser = DEFAULT_JOB_STORE_USER; } String jobStorePasswordParam = null; try { jobStorePasswordParam = configurationAt.getString(JOB_STORE_PASSWORD, null); } catch (ConversionException ex) { LOGGER.error("Failed to parse parameter '{}'.", JOB_STORE_PASSWORD); } if (jobStoreUserParam != null) { jobStorePassword = jobStorePasswordParam; } else { LOGGER.info("No parameter '{}' defined, defaulting to {}.", JOB_STORE_PASSWORD, DEFAULT_JOB_STORE_PASSWORD); jobStorePassword = DEFAULT_JOB_STORE_PASSWORD; } Boolean waitOnShutdownParam = null; try { waitOnShutdownParam = configurationAt.getBoolean(CONFIG_WAIT_ON_SHUTDOWN, null); } catch (ConversionException ex) { LOGGER.error("Failed to parse parameter '{}'.", CONFIG_WAIT_ON_SHUTDOWN); } if (waitOnShutdownParam != null) { waitOnShutdown = waitOnShutdownParam; } else { LOGGER.info("No parameter '{}' defined, defaulting to {}.", CONFIG_WAIT_ON_SHUTDOWN, DEFAULT_WAIT_ON_SHUTDOWN); waitOnShutdown = DEFAULT_WAIT_ON_SHUTDOWN; } Boolean addDefaultSchedulesParam = null; try { addDefaultSchedulesParam = configurationAt.getBoolean(ADD_DEFAULT_SCHEDULES, null); } catch (ConversionException ex) { LOGGER.error("Failed to parse parameter '{}'.", ADD_DEFAULT_SCHEDULES); } if (addDefaultSchedulesParam != null) { addDefaultSchedules = addDefaultSchedulesParam; } else { LOGGER.info("No parameter '{}' defined, defaulting to {}.", ADD_DEFAULT_SCHEDULES, DEFAULT_ADD_DEFAULT_SCHEDULES); addDefaultSchedules = DEFAULT_ADD_DEFAULT_SCHEDULES; } Integer startDelaySecondsParam = null; try { startDelaySecondsParam = configurationAt.getInteger(CONFIG_START_DELAY_SECONDS, null); } catch (ConversionException ex) { LOGGER.error("Failed to parse parameter '{}'.", CONFIG_START_DELAY_SECONDS); } if (startDelaySecondsParam != null) { startDelaySeconds = startDelaySecondsParam; } else { LOGGER.info("No parameter '{}' defined, defaulting to {}.", CONFIG_START_DELAY_SECONDS, DEFAULT_START_DELAY_SECONDS); startDelaySeconds = DEFAULT_START_DELAY_SECONDS; } } else { LOGGER.info("No scheduler configuration node found in datamanager config. Using default values."); } }
From source file:edu.stolaf.cs.wmrserver.TransformProcessor.java
private void copyLibraryFiles(SubnodeConfiguration languageConf, File jobTransformDir) throws IOException { // Get the library directory, returning if not specified File libDir = getLibraryDirectory(languageConf); if (libDir == null) return;//from www. j av a 2 s .co m // Get the file extension appropriate for the language String scriptExtension = languageConf.getString("extension", ""); if (!scriptExtension.isEmpty()) scriptExtension = "." + scriptExtension; // Create a file filter to exclude prefix/suffix files String pattern = "^(mapper|reducer)-(prefix|suffix)" + Pattern.quote(scriptExtension) + "$"; FileFilter libFilter = new NotFileFilter(new RegexFileFilter(pattern)); // Copy filtered contents of library directory FileUtils.copyDirectory(libDir, jobTransformDir, libFilter); }
From source file:net.datenwerke.sandbox.util.SandboxParser.java
public void configureSandboxService(SandboxService sandboxService, Configuration config) { if (!(config instanceof HierarchicalConfiguration)) throw new IllegalArgumentException("Expected HierarchicalConfiguration format"); HierarchicalConfiguration conf = (HierarchicalConfiguration) config; /* general properties */ SubnodeConfiguration properties = null; try {/*from ww w. j a va2 s .c o m*/ properties = conf.configurationAt("security.properties"); } catch (IllegalArgumentException e) { } if (null != properties) { /* remote */ boolean remoteConfig = properties.getBoolean("remote.configureService", false); if (remoteConfig) { boolean remote = properties.getBoolean("remote[@enable]", false); if (remote) { Integer poolsize = properties.getInteger("remote.jvm[@poolsize]", 2); Integer freelancersize = properties.getInteger("remote.jvm[@freelancersize]", 2); String vmArgs = properties.getString("remote.jvm.vmargs", null); Integer rmiMinPort = properties.getInteger("remote.jvm.rmi[@minport]", 10000); Integer rmiMaxPort = properties.getInteger("remote.jvm.rmi[@maxport]", 10200); sandboxService.shutdownJvmPool(); JvmPoolImpl jvmPoolImpl = new JvmPoolImpl( new JvmPoolConfigImpl(poolsize, freelancersize, new JvmInstantiatorImpl(rmiMinPort, rmiMaxPort, null != vmArgs && "".equals(vmArgs.trim()) ? null : vmArgs))); sandboxService.initJvmPool(jvmPoolImpl); } else { sandboxService.shutdownJvmPool(); } } /* daemons */ Integer monitorCheckInterval = properties.getInteger("monitor[@checkinterval]", 10); sandboxService.setMonitorDaemonCheckInterval(monitorCheckInterval); Integer monitorWatchdogCheckInterval = properties.getInteger("monitor[@watchdogCheckinterval]", 10000); sandboxService.setMonitorWatchdogCheckInterval(monitorWatchdogCheckInterval); /* codesource */ boolean enableCodesource = properties.getBoolean("codesource[@enable]", false); sandboxService.setCodesourceSecurityChecks(enableCodesource); } /* sandboxes */ parse(config); for (Entry<String, SandboxContext> e : getRestrictionSets().entrySet()) sandboxService.registerContext(e.getKey(), e.getValue()); }
From source file:edu.stolaf.cs.wmrserver.TransformProcessor.java
/** * Internal implementation of #prepareMapperPackage() and * #prepareReducerPackage() that works for either, determined by the * transformType argument./* w ww . j a v a 2s . c o m*/ */ protected File prepareTransformPackage(TransformType transformType, String transformSource, String language) throws IOException, ValidationException, CompilationException { // Get the given language's configuration, checking whether the language // exists in the configuration in the first place SubnodeConfiguration languageConf; try { languageConf = m_wmrConfig.configurationAt(language); } catch (IllegalArgumentException ex) { throw new ValidationException("Specified source code language is not recognized.", language); } // Get the transform type in string form ("mapper" or "reducer") for use // in messages, etc. String transformTypeString = transformType.toString().toLowerCase(); // Wrap source in library code as appropriate transformSource = wrap(languageConf, transformTypeString, transformSource); // Create temp directory to store transform package File jobTempDir = getJobTempDir(); File jobTransformDir = new File(jobTempDir, transformTypeString); if (!jobTransformDir.mkdir()) { throw new IOException("Could not create " + transformTypeString + " temporary directory:\n" + jobTransformDir.toString()); } // Write processed transform source to disk String scriptExtension = languageConf.getString("extension", ""); if (!scriptExtension.isEmpty()) scriptExtension = "." + scriptExtension; File transformFile = writeTransformSource(transformTypeString, jobTransformDir, scriptExtension, transformSource); // Copy extra library files unless otherwise specified boolean copyLibraryFiles = languageConf.getBoolean("copyLibraryFiles", true); if (copyLibraryFiles) { copyLibraryFiles(languageConf, jobTransformDir); } // Compile if specified File srcTransformFile = transformFile; transformFile = compile(languageConf, transformTypeString, transformFile, jobTransformDir); if (!transformFile.equals(srcTransformFile)) srcTransformFile.delete(); // Return the path to the final transform file, relativized to the temp dir return relativizeFile(transformFile, getJobTempDir()); }
From source file:org.loggo.server.KafkaConsumer.java
public void initialize(HierarchicalINIConfiguration props, LinkedBlockingDeque<LogEntry> queue) { SubnodeConfiguration kafkaConsumer = props.getSection("KafkaConsumer"); this.consumerConfig = new ConsumerConfig(asProperties(kafkaConsumer)); SubnodeConfiguration kafka = props.getSection("kafka"); topic = kafka.getString("topic", Defaults.TOPIC); this.queue = queue; }
From source file:org.midonet.config.providers.HierarchicalConfigurationProvider.java
@Override public String getValue(String group, String key, String defaultValue) { try {//from www . ja v a2s . com SubnodeConfiguration subConfig = config.configurationAt(group); return subConfig.getString(key, defaultValue); } catch (Throwable ex) { // fail properly if the configuration is missing this group return defaultValue; } }