List of usage examples for org.apache.commons.collections15.map ListOrderedMap ListOrderedMap
public ListOrderedMap()
ListOrderedMap that decorates a HashMap. From source file:edu.brown.benchmark.seats.SEATSProfile.java
@Override public String toString() { Map<String, Object> m = new ListOrderedMap<String, Object>(); m.put("Scale Factor", this.scale_factor); m.put("# of Reservations", this.num_reservations); m.put("Flight Start Date", this.flight_start_date); m.put("Flight Upcoming Date", this.flight_upcoming_date); m.put("Flight Past Days", this.flight_past_days); m.put("Flight Future Days", this.flight_future_days); m.put("Num Flights", this.num_flights); m.put("Num Customers", this.num_customers); m.put("Num Reservations", this.num_reservations); return (StringUtil.formatMaps(m)); }
From source file:edu.brown.hstore.dtxn.AbstractTransaction.java
protected Map<String, Object> getDebugMap() { Map<String, Object> m = new ListOrderedMap<String, Object>(); m.put("Transaction #", this.txn_id); m.put("Hash Code", this.hashCode()); m.put("SysProc", this.sysproc); m.put("Current Round State", Arrays.toString(this.round_state)); m.put("Read-Only", Arrays.toString(this.exec_readOnly)); m.put("Last UndoToken", Arrays.toString(this.last_undo_token)); m.put("# of Rounds", Arrays.toString(this.round_ctr)); if (this.pending_error != null) m.put("Pending Error", this.pending_error); return (m);/*from w ww .j a v a 2s . c om*/ }
From source file:edu.brown.hstore.BatchPlanner.java
/** * @param txn_id/*from w w w . j a v a 2 s .c o m*/ * @param client_handle * @param base_partition * @param predict_partitions * @param touched_partitions * @param batchArgs * @return */ public BatchPlan plan(Long txn_id, long client_handle, Integer base_partition, Collection<Integer> predict_partitions, boolean predict_singlepartitioned, Histogram<Integer> touched_partitions, ParameterSet[] batchArgs) { if (this.enable_profiling) time_plan.start(); if (d) LOG.debug(String.format("Constructing a new %s BatchPlan for %s txn #%d", this.catalog_proc.getName(), (predict_singlepartitioned ? "single-partition" : "distributed"), txn_id)); boolean cache_isSinglePartition[] = null; // OPTIMIZATION: Check whether we can use a cached single-partition BatchPlan if (this.force_singlePartition || this.enable_caching) { boolean is_allSinglePartition = true; cache_isSinglePartition = new boolean[this.batchSize]; // OPTIMIZATION: Skip all of this if we know that we're always // suppose to be single-partitioned if (this.force_singlePartition == false) { for (int stmt_index = 0; stmt_index < this.batchSize; stmt_index++) { if (cache_fastLookups[stmt_index] == null) { if (d) LOG.debug(String.format( "[#%d-%02d] No fast look-ups for %s. Cache is marked as not single-partitioned", txn_id, stmt_index, this.catalog_stmts[stmt_index].fullName())); cache_isSinglePartition[stmt_index] = false; } else { if (d) LOG.debug(String.format("[#%d-%02d] Using fast-lookup caching for %s: %s", txn_id, stmt_index, this.catalog_stmts[stmt_index].fullName(), Arrays.toString(cache_fastLookups[stmt_index]))); Object params[] = batchArgs[stmt_index].toArray(); cache_isSinglePartition[stmt_index] = true; for (int idx : cache_fastLookups[stmt_index]) { if (hasher.hash(params[idx]) != base_partition.intValue()) { cache_isSinglePartition[stmt_index] = false; break; } } // FOR } if (d) LOG.debug(String.format("[#%d-%02d] cache_isSinglePartition[%s] = %s", txn_id, stmt_index, this.catalog_stmts[stmt_index].fullName(), cache_isSinglePartition[stmt_index])); is_allSinglePartition = is_allSinglePartition && cache_isSinglePartition[stmt_index]; } // FOR (Statement) } if (t) LOG.trace(String.format("[#%d] is_allSinglePartition=%s", txn_id, is_allSinglePartition)); // If all of the Statements are single-partition, then we can use // the cached BatchPlan if we already have one. // This saves a lot of trouble if (is_allSinglePartition && cache_singlePartitionPlans[base_partition.intValue()] != null) { if (d) LOG.debug(String.format("[#%d] Using cached BatchPlan at partition #%02d: %s", txn_id, base_partition, Arrays.toString(this.catalog_stmts))); if (this.enable_profiling) time_plan.stop(); return (cache_singlePartitionPlans[base_partition.intValue()]); } } // Otherwise we have to construct a new BatchPlan plan.init(client_handle, base_partition); // ---------------------- // DEBUG DUMP // ---------------------- if (t) { Map<String, Object> m = new ListOrderedMap<String, Object>(); m.put("Batch Size", this.batchSize); for (int i = 0; i < this.batchSize; i++) { m.put(String.format("[%02d] %s", i, this.catalog_stmts[i].getName()), Arrays.toString(batchArgs[i].toArray())); } LOG.trace("\n" + StringUtil.formatMapsBoxed(m)); } // Only maintain the histogram of what partitions were touched if we // know that we're going to throw a MispredictionException Histogram<Integer> mispredict_h = null; boolean mispredict = false; for (int stmt_index = 0; stmt_index < this.batchSize; stmt_index++) { final Statement catalog_stmt = this.catalog_stmts[stmt_index]; assert (catalog_stmt != null) : "The Statement at index " + stmt_index + " is null for " + this.catalog_proc; final Object params[] = batchArgs[stmt_index].toArray(); if (t) LOG.trace(String.format("[#%d-%02d] Calculating touched partitions plans for %s", txn_id, stmt_index, catalog_stmt.fullName())); Map<PlanFragment, Set<Integer>> frag_partitions = plan.frag_partitions[stmt_index]; Set<Integer> stmt_all_partitions = plan.stmt_partitions[stmt_index]; boolean has_singlepartition_plan = catalog_stmt.getHas_singlesited(); boolean is_replicated_only = this.stmt_is_replicatedonly[stmt_index]; boolean is_read_only = this.stmt_is_readonly[stmt_index]; // boolean stmt_localFragsAreNonTransactional = // plan.localFragsAreNonTransactional; boolean is_singlepartition = has_singlepartition_plan; boolean is_local = true; CatalogMap<PlanFragment> fragments = null; // AbstractPlanNode node = // PlanNodeUtil.getRootPlanNodeForStatement(catalog_stmt, false); // LOG.info(PlanNodeUtil.debug(node)); // OPTIMIZATION: Fast partition look-up caching // OPTIMIZATION: Read-only queries on replicated tables always just // go to the local partition // OPTIMIZATION: If we're force to be single-partitioned, pretend // that the table is replicated if (cache_isSinglePartition[stmt_index] || (is_replicated_only && is_read_only) || this.force_singlePartition) { if (t) { if (cache_isSinglePartition[stmt_index]) { LOG.trace(String.format("[#%d-%02d] Using fast-lookup for %s. Skipping PartitionEstimator", txn_id, stmt_index, catalog_stmt.fullName())); } else { LOG.trace(String.format( "[#%d-%02d] %s is read-only and replicate-only. Skipping PartitionEstimator", txn_id, stmt_index, catalog_stmt.fullName())); } } assert (has_singlepartition_plan); if (this.cache_singlePartitionFragmentPartitions == null) { this.cache_singlePartitionFragmentPartitions = CACHED_FRAGMENT_PARTITION_MAPS[base_partition .intValue()]; } Map<PlanFragment, Set<Integer>> cached_frag_partitions = this.cache_singlePartitionFragmentPartitions .get(catalog_stmt); if (cached_frag_partitions == null) { cached_frag_partitions = new HashMap<PlanFragment, Set<Integer>>(); Set<Integer> p = CACHED_SINGLE_PARTITION_SETS[base_partition.intValue()]; for (PlanFragment catalog_frag : catalog_stmt.getFragments().values()) { cached_frag_partitions.put(catalog_frag, p); } // FOR this.cache_singlePartitionFragmentPartitions.put(catalog_stmt, cached_frag_partitions); } if (plan.stmt_partitions_swap[stmt_index] == null) { plan.stmt_partitions_swap[stmt_index] = plan.stmt_partitions[stmt_index]; plan.frag_partitions_swap[stmt_index] = plan.frag_partitions[stmt_index]; } stmt_all_partitions = plan.stmt_partitions[stmt_index] = CACHED_SINGLE_PARTITION_SETS[base_partition .intValue()]; frag_partitions = plan.frag_partitions[stmt_index] = cached_frag_partitions; } // Otherwise figure out whether the query can execute as // single-partitioned or not else { if (t) LOG.trace(String.format( "[#%d-%02d] Computing touched partitions %s in txn #%d with the PartitionEstimator", txn_id, stmt_index, catalog_stmt.fullName(), txn_id)); if (plan.stmt_partitions_swap[stmt_index] != null) { stmt_all_partitions = plan.stmt_partitions[stmt_index] = plan.stmt_partitions_swap[stmt_index]; plan.stmt_partitions_swap[stmt_index] = null; stmt_all_partitions.clear(); frag_partitions = plan.frag_partitions[stmt_index] = plan.frag_partitions_swap[stmt_index]; plan.frag_partitions_swap[stmt_index] = null; } try { // OPTIMIZATION: If we were told that the transaction is suppose to be // single-partitioned, then we will throw the single-partitioned PlanFragments // at the PartitionEstimator to get back what partitions each PlanFragment // will need to go to. If we get multiple partitions, then we know that we // mispredicted and we should throw a MispredictionException // If we originally didn't predict that it was single-partitioned, then we // actually still need to check whether the query should be single-partitioned or not. // This is because a query may actually just want to execute on just one // partition (note that it could be a local partition or the remote partition). // We'll assume that it's single-partition <<--- Can we cache that?? while (true) { if (is_singlepartition == false) stmt_all_partitions.clear(); fragments = (is_singlepartition ? catalog_stmt.getFragments() : catalog_stmt.getMs_fragments()); // PARTITION ESTIMATOR if (this.enable_profiling) ProfileMeasurement.swap(this.time_plan, this.time_partitionEstimator); this.p_estimator.getAllFragmentPartitions(frag_partitions, stmt_all_partitions, fragments.values(), params, base_partition); if (this.enable_profiling) ProfileMeasurement.swap(this.time_partitionEstimator, this.time_plan); int stmt_all_partitions_size = stmt_all_partitions.size(); if (is_singlepartition && stmt_all_partitions_size > 1) { // If this was suppose to be multi-partitioned, then // we want to stop right here!! if (predict_singlepartitioned) { if (t) LOG.trace(String.format("Mispredicted txn #%d - Multiple Partitions")); mispredict = true; break; } // Otherwise we can let it wrap back around and // construct the fragment mapping for the // multi-partition PlanFragments is_singlepartition = false; continue; } is_local = (stmt_all_partitions_size == 1 && stmt_all_partitions.contains(base_partition)); if (is_local == false && predict_singlepartitioned) { // Again, this is not what was suppose to happen! if (t) LOG.trace(String.format("Mispredicted txn #%d - Remote Partitions %s", txn_id, stmt_all_partitions)); mispredict = true; break; } else if (predict_partitions.containsAll(stmt_all_partitions) == false) { // Again, this is not what was suppose to happen! if (t) LOG.trace(String.format("Mispredicted txn #%d - Unallocated Partitions %s / %s", txn_id, stmt_all_partitions, predict_partitions)); mispredict = true; break; } // Score! We have a plan that works! break; } // WHILE // Bad Mojo! } catch (Exception ex) { String msg = ""; for (int i = 0; i < this.batchSize; i++) { msg += String.format("[#%d-%02d] %s %s\n%5s\n", txn_id, i, catalog_stmt.fullName(), catalog_stmt.getSqltext(), Arrays.toString(batchArgs[i].toArray())); } // FOR LOG.fatal("\n" + msg); throw new RuntimeException("Unexpected error when planning " + catalog_stmt.fullName(), ex); } } if (d) LOG.debug(String.format("[#%d-%02d] is_singlepartition=%s, partitions=%s", txn_id, stmt_index, is_singlepartition, stmt_all_partitions)); // Get a sorted list of the PlanFragments that we need to execute // for this query if (is_singlepartition) { if (this.sorted_singlep_fragments[stmt_index] == null) { this.sorted_singlep_fragments[stmt_index] = PlanNodeUtil.getSortedPlanFragments(catalog_stmt, true); } plan.frag_list[stmt_index] = this.sorted_singlep_fragments[stmt_index]; // Only mark that we touched these partitions if the Statement // is not on a replicated table if (is_replicated_only == false) { touched_partitions.putAll(stmt_all_partitions); } } else { if (this.sorted_multip_fragments[stmt_index] == null) { this.sorted_multip_fragments[stmt_index] = PlanNodeUtil.getSortedPlanFragments(catalog_stmt, false); } plan.frag_list[stmt_index] = this.sorted_multip_fragments[stmt_index]; // Always mark that we are touching these partitions touched_partitions.putAll(stmt_all_partitions); } plan.readonly = plan.readonly && catalog_stmt.getReadonly(); // plan.localFragsAreNonTransactional = // plan.localFragsAreNonTransactional || // stmt_localFragsAreNonTransactional; plan.all_singlepartitioned = plan.all_singlepartitioned && is_singlepartition; plan.all_local = plan.all_local && is_local; // Keep track of whether the current query in the batch was // single-partitioned or not plan.singlepartition_bitmap[stmt_index] = is_singlepartition; // Misprediction!! if (mispredict) { // If this is the first Statement in the batch that hits the mispredict, // then we need to create the histogram and populate it with the // partitions from the previous queries int start_idx = stmt_index; if (mispredict_h == null) { mispredict_h = new Histogram<Integer>(); start_idx = 0; } for (int i = start_idx; i <= stmt_index; i++) { if (d) LOG.debug(String.format( "Pending mispredict for txn #%d. Checking whether to add partitions for batch statement %02d", txn_id, i)); // Make sure that we don't count the local partition if it // was reading a replicated table. if (this.stmt_is_replicatedonly[i] == false || (this.stmt_is_replicatedonly[i] && this.stmt_is_readonly[i] == false)) { if (t) LOG.trace(String.format( "%s touches non-replicated table. Including %d partitions in mispredict histogram for txn #%d", this.catalog_stmts[i].fullName(), plan.stmt_partitions[i].size(), txn_id)); mispredict_h.putAll(plan.stmt_partitions[i]); } } // FOR continue; } // ---------------------- // DEBUG DUMP // ---------------------- if (d) { Map<?, ?> maps[] = new Map[fragments.size() + 1]; int ii = 0; for (PlanFragment catalog_frag : fragments) { Map<String, Object> m = new ListOrderedMap<String, Object>(); Set<Integer> p = plan.frag_partitions[stmt_index].get(catalog_frag); boolean frag_local = (p.size() == 1 && p.contains(base_partition)); m.put(String.format("[%02d] Fragment", ii), catalog_frag.fullName()); m.put(String.format(" Partitions"), p); m.put(String.format(" IsLocal"), frag_local); ii++; maps[ii] = m; } // FOR Map<String, Object> header = new ListOrderedMap<String, Object>(); header.put("Batch Statement#", String.format("%02d / %02d", stmt_index, this.batchSize)); header.put("Catalog Statement", catalog_stmt.fullName()); header.put("Statement SQL", catalog_stmt.getSqltext()); header.put("All Partitions", plan.stmt_partitions[stmt_index]); header.put("Local Partition", base_partition); header.put("IsSingledSited", is_singlepartition); header.put("IsStmtLocal", is_local); header.put("IsReplicatedOnly", is_replicated_only); header.put("IsBatchLocal", plan.all_local); header.put("Fragments", fragments.size()); maps[0] = header; LOG.debug("\n" + StringUtil.formatMapsBoxed(maps)); } } // FOR (Statement) // Check whether we have an existing graph exists for this batch // configuration // This is the only place where we need to synchronize int bitmap_hash = Arrays.hashCode(plan.singlepartition_bitmap); PlanGraph graph = this.plan_graphs.get(bitmap_hash); if (graph == null) { // assume fast case graph = this.buildPlanGraph(plan); this.plan_graphs.put(bitmap_hash, graph); } plan.graph = graph; plan.rounds_length = graph.num_rounds; if (this.enable_profiling) time_plan.stop(); // Create the MispredictException if any Statement in the loop above hit // it. We don't want to throw it because whoever called us may want to look // at the plan first if (mispredict_h != null) { plan.mispredict = new MispredictionException(txn_id, mispredict_h); } // If this a single-partition plan and we have caching enabled, we'll // add this to our cached listing. We'll mark it as cached so that it is never // returned back to the BatchPlan object pool else if (this.enable_caching && cache_singlePartitionPlans[base_partition.intValue()] == null && plan.isSingledPartitionedAndLocal()) { cache_singlePartitionPlans[base_partition.intValue()] = plan; plan.cached = true; plan = new BatchPlan(this.maxRoundSize); return cache_singlePartitionPlans[base_partition.intValue()]; } if (d) LOG.debug("Created BatchPlan:\n" + plan.toString()); return (plan); }
From source file:com.oltpbenchmark.benchmarks.seats.SEATSProfile.java
@Override public String toString() { Map<String, Object> m = new ListOrderedMap<String, Object>(); m.put("Scale Factor", this.scale_factor); m.put("Data Directory", this.airline_data_dir); m.put("# of Reservations", this.num_reservations); m.put("Flight Start Date", this.flight_start_date); m.put("Flight Upcoming Date", this.flight_upcoming_date); m.put("Flight Past Days", this.flight_past_days); m.put("Flight Future Days", this.flight_future_days); m.put("Flight Upcoming Offset", this.flight_upcoming_offset); m.put("Reservation Upcoming Offset", this.reservation_upcoming_offset); return (StringUtil.formatMaps(m)); }
From source file:edu.brown.benchmark.BenchmarkComponent.java
/** * Constructor that initializes the framework portions of the client. * Creates a Volt client and connects it to all the hosts provided on the * command line with the specified username and password * * @param args/*from w w w . j a va 2 s.c om*/ */ public BenchmarkComponent(String args[]) { // Initialize HStoreConf String hstore_conf_path = null; for (int i = 0; i < args.length; i++) { final String arg = args[i]; final String[] parts = arg.split("=", 2); if (parts.length > 1 && parts[1].startsWith("${") == false && parts[0].equalsIgnoreCase("CONF")) { hstore_conf_path = parts[1]; break; } } // FOR if (HStoreConf.isInitialized() == false) { assert (hstore_conf_path != null) : "Missing HStoreConf file"; File f = new File(hstore_conf_path); if (debug.get()) LOG.debug("Initializing HStoreConf from '" + f.getName() + "' along with input parameters"); HStoreConf.init(f, args); } else { if (debug.get()) LOG.debug("Initializing HStoreConf only with input parameters"); HStoreConf.singleton().loadFromArgs(args); } m_hstoreConf = HStoreConf.singleton(); if (trace.get()) LOG.trace("HStore Conf\n" + m_hstoreConf.toString(true)); int transactionRate = m_hstoreConf.client.txnrate; boolean blocking = m_hstoreConf.client.blocking; boolean tableStats = m_hstoreConf.client.tablestats; String tableStatsDir = m_hstoreConf.client.tablestats_dir; int tickInterval = m_hstoreConf.client.tick_interval; // default values String username = "user"; String password = "password"; ControlState state = ControlState.PREPARING; // starting state String reason = ""; // and error string int id = 0; boolean isLoader = false; int num_clients = 0; int num_partitions = 0; boolean exitOnCompletion = true; float checkTransaction = 0; boolean checkTables = false; boolean noConnections = false; boolean noUploading = false; File catalogPath = null; String projectName = null; String partitionPlanPath = null; boolean partitionPlanIgnoreMissing = false; long startupWait = -1; String statsDatabaseURL = null; String statsDatabaseUser = null; String statsDatabasePass = null; String statsDatabaseJDBC = null; int statsPollInterval = 10000; // scan the inputs once to read everything but host names Map<String, Object> componentParams = new TreeMap<String, Object>(); for (int i = 0; i < args.length; i++) { final String arg = args[i]; final String[] parts = arg.split("=", 2); if (parts.length == 1) { state = ControlState.ERROR; reason = "Invalid parameter: " + arg; break; } else if (parts[1].startsWith("${")) { continue; } else if (parts[0].equalsIgnoreCase("CONF")) { continue; } if (debug.get()) componentParams.put(parts[0], parts[1]); if (parts[0].equalsIgnoreCase("CATALOG")) { catalogPath = new File(parts[1]); assert (catalogPath.exists()) : "The catalog file '" + catalogPath.getAbsolutePath() + " does not exist"; if (debug.get()) componentParams.put(parts[0], catalogPath); } else if (parts[0].equalsIgnoreCase("LOADER")) { isLoader = Boolean.parseBoolean(parts[1]); } else if (parts[0].equalsIgnoreCase("NAME")) { projectName = parts[1]; } else if (parts[0].equalsIgnoreCase("USER")) { username = parts[1]; } else if (parts[0].equalsIgnoreCase("PASSWORD")) { password = parts[1]; } else if (parts[0].equalsIgnoreCase("EXITONCOMPLETION")) { exitOnCompletion = Boolean.parseBoolean(parts[1]); } else if (parts[0].equalsIgnoreCase("ID")) { id = Integer.parseInt(parts[1]); } else if (parts[0].equalsIgnoreCase("NUMCLIENTS")) { num_clients = Integer.parseInt(parts[1]); } else if (parts[0].equalsIgnoreCase("NUMPARTITIONS")) { num_partitions = Integer.parseInt(parts[1]); } else if (parts[0].equalsIgnoreCase("CHECKTRANSACTION")) { checkTransaction = Float.parseFloat(parts[1]); } else if (parts[0].equalsIgnoreCase("CHECKTABLES")) { checkTables = Boolean.parseBoolean(parts[1]); } else if (parts[0].equalsIgnoreCase("NOCONNECTIONS")) { noConnections = Boolean.parseBoolean(parts[1]); } else if (parts[0].equalsIgnoreCase("NOUPLOADING")) { noUploading = Boolean.parseBoolean(parts[1]); } else if (parts[0].equalsIgnoreCase("WAIT")) { startupWait = Long.parseLong(parts[1]); } // Procedure Stats Uploading Parameters else if (parts[0].equalsIgnoreCase("STATSDATABASEURL")) { statsDatabaseURL = parts[1]; } else if (parts[0].equalsIgnoreCase("STATSDATABASEUSER")) { if (parts[1].isEmpty() == false) statsDatabaseUser = parts[1]; } else if (parts[0].equalsIgnoreCase("STATSDATABASEPASS")) { if (parts[1].isEmpty() == false) statsDatabasePass = parts[1]; } else if (parts[0].equalsIgnoreCase("STATSDATABASEJDBC")) { if (parts[1].isEmpty() == false) statsDatabaseJDBC = parts[1]; } else if (parts[0].equalsIgnoreCase("STATSPOLLINTERVAL")) { statsPollInterval = Integer.parseInt(parts[1]); } else if (parts[0].equalsIgnoreCase(ArgumentsParser.PARAM_PARTITION_PLAN)) { partitionPlanPath = parts[1]; } else if (parts[0].equalsIgnoreCase(ArgumentsParser.PARAM_PARTITION_PLAN_IGNORE_MISSING)) { partitionPlanIgnoreMissing = Boolean.parseBoolean(parts[1]); } // If it starts with "benchmark.", then it always goes to the implementing class else if (parts[0].toLowerCase().startsWith(HStoreConstants.BENCHMARK_PARAM_PREFIX)) { if (debug.get()) componentParams.remove(parts[0]); parts[0] = parts[0].substring(HStoreConstants.BENCHMARK_PARAM_PREFIX.length()); m_extraParams.put(parts[0].toUpperCase(), parts[1]); } } if (trace.get()) { Map<String, Object> m = new ListOrderedMap<String, Object>(); m.put("BenchmarkComponent", componentParams); m.put("Extra Client", m_extraParams); LOG.debug("Input Parameters:\n" + StringUtil.formatMaps(m)); } // Thread.currentThread().setName(String.format("client-%02d", id)); m_catalogPath = catalogPath; m_projectName = projectName; m_id = id; m_numClients = num_clients; m_numPartitions = num_partitions; m_exitOnCompletion = exitOnCompletion; m_username = username; m_password = password; m_txnRate = (isLoader ? -1 : transactionRate); m_txnsPerMillisecond = (isLoader ? -1 : transactionRate / 1000.0); m_blocking = blocking; m_tickInterval = tickInterval; m_noUploading = noUploading; m_noConnections = noConnections || (isLoader && m_noUploading); m_tableStats = tableStats; m_tableStatsDir = (tableStatsDir.isEmpty() ? null : new File(tableStatsDir)); // If we were told to sleep, do that here before we try to load in the catalog // This is an attempt to keep us from overloading a single node all at once if (startupWait > 0) { if (debug.get()) LOG.debug(String.format("Delaying client start-up by %.2f sec", startupWait / 1000d)); try { Thread.sleep(startupWait); } catch (InterruptedException ex) { throw new RuntimeException("Unexpected interruption", ex); } } // HACK: This will instantiate m_catalog for us... if (m_catalogPath != null) { this.getCatalog(); } // Parse workload transaction weights if (m_hstoreConf.client.weights != null) { for (String entry : m_hstoreConf.client.weights.split("(,|;)")) { String data[] = entry.split(":"); if (data.length != 2) { LOG.warn("Invalid transaction weight entry '" + entry + "'"); continue; } try { String txnName = data[0]; int txnWeight = Integer.parseInt(data[1]); assert (txnWeight >= 0); // '*' is the default value if (txnName.equals("*")) { this.m_txnWeightsDefault = txnWeight; if (debug.get()) LOG.debug(String.format("Default Transaction Weight: %d", txnWeight)); } else { if (debug.get()) LOG.debug(String.format("%s Transaction Weight: %d", txnName, txnWeight)); this.m_txnWeights.put(txnName.toUpperCase(), txnWeight); } } catch (Throwable ex) { LOG.warn("Invalid transaction weight entry '" + entry + "'", ex); continue; } } // FOR } if (partitionPlanPath != null) { boolean exists = FileUtil.exists(partitionPlanPath); if (partitionPlanIgnoreMissing == false) assert (exists) : "Invalid partition plan path '" + partitionPlanPath + "'"; if (exists) this.applyPartitionPlan(partitionPlanPath); } StatsUploaderSettings statsSettings = null; if (statsDatabaseURL != null && statsDatabaseURL.isEmpty() == false) { try { statsSettings = StatsUploaderSettings.singleton(statsDatabaseURL, statsDatabaseUser, statsDatabasePass, statsDatabaseJDBC, projectName, (isLoader ? "LOADER" : "CLIENT"), statsPollInterval, m_catalog); } catch (Throwable ex) { throw new RuntimeException("Failed to initialize StatsUploader", ex); } if (debug.get()) LOG.debug("StatsUploaderSettings:\n" + statsSettings); } Client new_client = BenchmarkComponent.getClient((m_hstoreConf.client.txn_hints ? this.getCatalog() : null), getExpectedOutgoingMessageSize(), useHeavyweightClient(), statsSettings); if (m_blocking) { // && isLoader == false) { if (debug.get()) LOG.debug(String.format("Using BlockingClient [concurrent=%d]", m_hstoreConf.client.blocking_concurrent)); m_voltClient = new BlockingClient(new_client, m_hstoreConf.client.blocking_concurrent); } else { m_voltClient = new_client; } // report any errors that occurred before the client was instantiated if (state != ControlState.PREPARING) setState(state, reason); // scan the inputs again looking for host connections if (m_noConnections == false) { synchronized (BenchmarkComponent.class) { if (globalHasConnections == false) { this.setupConnections(); globalHasConnections = true; } } // SYNCH } m_checkTransaction = checkTransaction; m_checkTables = checkTables; m_constraints = new LinkedHashMap<Pair<String, Integer>, Expression>(); m_countDisplayNames = getTransactionDisplayNames(); if (m_countDisplayNames != null) { for (String txnName : m_countDisplayNames) { m_txnStats.transactions.put(txnName, 0); } // FOR } // If we need to call tick more frequently that when POLL is called, // then we'll want to use a separate thread if (m_tickInterval > 0 && isLoader == false) { if (debug.get()) LOG.debug(String.format( "Creating local thread that will call BenchmarkComponent.tick() every %.1f seconds", (m_tickInterval / 1000.0))); Runnable r = new Runnable() { @Override public void run() { try { while (true) { BenchmarkComponent.this.invokeTickCallback(m_tickCounter++); Thread.sleep(m_tickInterval); } // WHILE } catch (InterruptedException ex) { LOG.warn("Tick thread was interrupted"); } } }; m_tickThread = new Thread(r); m_tickThread.setDaemon(true); } else { m_tickThread = null; } }
From source file:com.oltpbenchmark.benchmarks.auctionmark.AuctionMarkProfile.java
@Override public String toString() { Map<String, Object> m = new ListOrderedMap<String, Object>(); m.put("Scale Factor", this.scale_factor); m.put("Loader Start", this.loaderStartTime); m.put("Loader Stop", this.loaderStopTime); m.put("Last CloseAuctions", (this.lastCloseAuctionsTime.getTime() > 0 ? this.lastCloseAuctionsTime : null)); m.put("Client Start", this.clientStartTime); m.put("Current Virtual Time", this.currentTime); m.put("Pending ItemCommentResponses", this.pending_commentResponses.size()); // Item Queues Histogram<ItemStatus> itemCounts = new Histogram<ItemStatus>(true); for (ItemStatus status : ItemStatus.values()) { int cnt = 0; switch (status) { case OPEN: cnt = this.items_available.size(); break; case ENDING_SOON: cnt = this.items_endingSoon.size(); break; case WAITING_FOR_PURCHASE: cnt = this.items_waitingForPurchase.size(); break; case CLOSED: cnt = this.items_completed.size(); break; default:/*w w w .j a va 2 s .c o m*/ assert (false) : "Unexpected " + status; } // SWITCH itemCounts.put(status, cnt); } m.put("Item Queues", itemCounts); return (StringUtil.formatMaps(m)); }
From source file:edu.brown.utils.ArgumentsParser.java
@Override public String toString() { Map<String, Object> m0 = new ListOrderedMap<String, Object>(); m0.putAll(this.params); Map<String, Object> m1 = null; int opt_cnt = this.opt_params.size(); if (opt_cnt > 0) { Map<String, Object> opt_inner = new ListOrderedMap<String, Object>(); for (int i = 0; i < opt_cnt; i++) { opt_inner.put(String.format("#%02d", i), this.opt_params.get(i)); }/*from w ww . j a v a 2s. c o m*/ m1 = new ListOrderedMap<String, Object>(); m1.put(String.format("Optional Parameters [%d]", opt_cnt), StringUtil.formatMaps(opt_inner)); } return StringUtil.formatMaps(m0, m1); }
From source file:edu.brown.hstore.dtxn.LocalTransaction.java
/** * Store a VoltTable result that this transaction is waiting for. * @param partition The partition id that generated the result * @param dependency_id The dependency id that this result corresponds to * @param key The hackish partition+dependency key * @param force If false, then we will check to make sure the result isn't a duplicate * @param result The actual data for the result *//*from www . j a v a 2 s . co m*/ private void addResult(final int partition, final int dependency_id, final int key, final boolean force, VoltTable result) { final int base_offset = hstore_site.getLocalPartitionOffset(this.base_partition); assert (result != null); assert (this.round_state[base_offset] == RoundState.INITIALIZED || this.round_state[base_offset] == RoundState.STARTED) : String.format( "Invalid round state %s for %s at partition %d", this.round_state[base_offset], this, this.base_partition); if (d) LOG.debug(String.format("%s - Attemping to add new result for %s [numRows=%d]", this, debugPartDep(partition, dependency_id), result.getRowCount())); // If the txn is still in the INITIALIZED state, then we just want to queue up the results // for now. They will get released when we switch to STARTED // This is the only part that we need to synchonize on if (force == false) { if (this.predict_singlePartition == false) this.state.lock.lock(); try { if (this.round_state[base_offset] == RoundState.INITIALIZED) { assert (this.state.queued_results.containsKey(key) == false) : String.format( "%s - Duplicate result %s [key=%d]", this, debugPartDep(partition, dependency_id), key); this.state.queued_results.put(key, result); if (d) LOG.debug(String.format("%s - Queued result %s until the round is started [key=%s]", this, debugPartDep(partition, dependency_id), key)); return; } if (d) { LOG.debug(String.format("%s - Storing new result for key %d", this, key)); if (t) LOG.trace("Result stmt_ctr(key=" + key + "): " + this.state.results_dependency_stmt_ctr.get(key)); } } finally { if (this.predict_singlePartition == false) this.state.lock.unlock(); } // SYNCH } // Each partition+dependency_id should be unique within the Statement batch. // So as the results come back to us, we have to figure out which Statement it belongs to DependencyInfo dinfo = null; Queue<Integer> queue = this.state.results_dependency_stmt_ctr.get(key); assert (queue != null) : String.format("Unexpected %s in %s", debugPartDep(partition, dependency_id), this); assert (queue.isEmpty() == false) : String.format( "No more statements for %s in %s [key=%d]\nresults_dependency_stmt_ctr = %s", debugPartDep(partition, dependency_id), this, key, this.state.results_dependency_stmt_ctr); int stmt_index = queue.remove().intValue(); dinfo = this.getDependencyInfo(dependency_id); assert (dinfo != null) : String.format("Unexpected %s for %s [stmt_index=%d]\n%s", debugPartDep(partition, dependency_id), this, stmt_index, result); dinfo.addResult(partition, result); if (this.predict_singlePartition == false) this.state.lock.lock(); try { this.state.received_ctr++; // Check whether we need to start running stuff now // 2011-12-31: This needs to be synchronized because they might check // whether there are no more blocked tasks before we // can add to_unblock to the unblocked_tasks queue if (this.state.blocked_tasks.isEmpty() == false && dinfo.hasTasksReady()) { Collection<WorkFragment> to_unblock = dinfo.getAndReleaseBlockedWorkFragments(); assert (to_unblock != null); assert (to_unblock.isEmpty() == false); if (d) LOG.debug(String.format( "%s - Got %d WorkFragments to unblock that were waiting for DependencyId %d", this, to_unblock.size(), dinfo.getDependencyId())); this.state.blocked_tasks.removeAll(to_unblock); this.state.unblocked_tasks.addLast(to_unblock); } else if (d) { LOG.debug(String.format( "%s - No WorkFragments to unblock after storing %s [blockedTasks=%d, hasTasksReady=%s]", this, debugPartDep(partition, dependency_id), this.state.blocked_tasks.size(), dinfo.hasTasksReady())); } if (this.state.dependency_latch != null) { this.state.dependency_latch.countDown(); // HACK: If the latch is now zero, then push an EMPTY set into the unblocked queue // This will cause the blocked PartitionExecutor thread to wake up and realize that he's done if (this.state.dependency_latch.getCount() == 0) { if (d) LOG.debug(String.format( "%s - Pushing EMPTY_SET to PartitionExecutor because all the dependencies have arrived!", this)); this.state.unblocked_tasks.addLast(EMPTY_FRAGMENT_SET); } if (d) LOG.debug(String.format("%s - Setting CountDownLatch to %d", this, this.state.dependency_latch.getCount())); } this.state.still_has_tasks = this.state.blocked_tasks.isEmpty() == false || this.state.unblocked_tasks.isEmpty() == false; } finally { if (this.predict_singlePartition == false) this.state.lock.unlock(); } // SYNCH if (d) { Map<String, Object> m = new ListOrderedMap<String, Object>(); m.put("Blocked Tasks", (this.state != null ? this.state.blocked_tasks.size() : null)); m.put("DependencyInfo", dinfo.toString()); m.put("hasTasksReady", dinfo.hasTasksReady()); LOG.debug(this + " - Status Information\n" + StringUtil.formatMaps(m)); if (t) LOG.trace(this.debug()); } }
From source file:edu.brown.costmodel.SingleSitedCostModel.java
/** * MAIN!// w w w.j a v a 2s . com * * @param vargs * @throws Exception */ public static void main(String[] vargs) throws Exception { ArgumentsParser args = ArgumentsParser.load(vargs); args.require(ArgumentsParser.PARAM_CATALOG, ArgumentsParser.PARAM_WORKLOAD, ArgumentsParser.PARAM_PARTITION_PLAN); assert (args.workload.getTransactionCount() > 0) : "No transactions were loaded from " + args.workload_path; if (args.hasParam(ArgumentsParser.PARAM_CATALOG_HOSTS)) { ClusterConfiguration cc = new ClusterConfiguration(args.getParam(ArgumentsParser.PARAM_CATALOG_HOSTS)); args.updateCatalog(FixCatalog.addHostInfo(args.catalog, cc), null); } // Enable compact output final boolean table_output = (args.getOptParams().contains("table")); // If given a PartitionPlan, then update the catalog File pplan_path = new File(args.getParam(ArgumentsParser.PARAM_PARTITION_PLAN)); PartitionPlan pplan = new PartitionPlan(); pplan.load(pplan_path.getAbsolutePath(), args.catalog_db); if (args.getBooleanParam(ArgumentsParser.PARAM_PARTITION_PLAN_REMOVE_PROCS, false)) { for (Procedure catalog_proc : pplan.proc_entries.keySet()) { pplan.setNullProcParameter(catalog_proc); } // FOR } if (args.getBooleanParam(ArgumentsParser.PARAM_PARTITION_PLAN_RANDOM_PROCS, false)) { for (Procedure catalog_proc : pplan.proc_entries.keySet()) { pplan.setRandomProcParameter(catalog_proc); } // FOR } pplan.apply(args.catalog_db); System.out.println("Applied PartitionPlan '" + pplan_path + "' to catalog\n" + pplan); System.out.print(StringUtil.DOUBLE_LINE); // if (!table_output) { // // } // } else if (!table_output) { // System.err.println("PartitionPlan file '" + pplan_path + // "' does not exist. Ignoring..."); // } if (args.hasParam(ArgumentsParser.PARAM_PARTITION_PLAN_OUTPUT)) { String output = args.getParam(ArgumentsParser.PARAM_PARTITION_PLAN_OUTPUT); if (output.equals("-")) output = pplan_path.getAbsolutePath(); pplan.save(output); System.out.println("Saved PartitionPlan to '" + output + "'"); } System.out.flush(); // TODO: REMOVE STORED PROCEDURE ROUTING FOR SCHISM long singlepartition = 0; long multipartition = 0; long total = 0; SingleSitedCostModel costmodel = new SingleSitedCostModel(args.catalog_db); Collection<Integer> all_partitions = CatalogUtil.getAllPartitionIds(args.catalog_db); // costmodel.setEntropyWeight(4.0); // costmodel.setJavaExecutionWeightEnabled(true); // costmodel.setJavaExecutionWeight(100); // XXX: 2011-10-28 costmodel.setCachingEnabled(true); Histogram<String> hist = new Histogram<String>(); for (int i = 0; i < 2; i++) { ProfileMeasurement time = new ProfileMeasurement("costmodel").start(); hist.clear(); for (AbstractTraceElement<? extends CatalogType> element : args.workload) { if (element instanceof TransactionTrace) { total++; TransactionTrace xact = (TransactionTrace) element; boolean is_singlesited = costmodel.processTransaction(args.catalog_db, xact, null).singlesited; if (is_singlesited) { singlepartition++; hist.put(xact.getCatalogItemName()); } else { multipartition++; if (!hist.contains(xact.getCatalogItemName())) hist.put(xact.getCatalogItemName(), 0); } } } // FOR System.err.println("ESTIMATE TIME: " + time.stop().getTotalThinkTimeSeconds()); break; // XXX } // FOR // long total_partitions_touched_txns = // costmodel.getTxnPartitionAccessHistogram().getSampleCount(); // long total_partitions_touched_queries = // costmodel.getQueryPartitionAccessHistogram().getSampleCount(); Histogram<Integer> h = null; if (!table_output) { System.out.println("Workload Procedure Histogram:"); System.out.println(StringUtil.addSpacers(args.workload.getProcedureHistogram().toString())); System.out.print(StringUtil.DOUBLE_LINE); System.out.println("SinglePartition Procedure Histogram:"); System.out.println(StringUtil.addSpacers(hist.toString())); System.out.print(StringUtil.DOUBLE_LINE); System.out.println("Java Execution Histogram:"); h = costmodel.getJavaExecutionHistogram(); h.setKeepZeroEntries(true); h.putAll(all_partitions, 0); System.out.println(StringUtil.addSpacers(h.toString())); System.out.print(StringUtil.DOUBLE_LINE); System.out.println("Transaction Partition Histogram:"); h = costmodel.getTxnPartitionAccessHistogram(); h.setKeepZeroEntries(true); h.putAll(all_partitions, 0); System.out.println(StringUtil.addSpacers(h.toString())); System.out.print(StringUtil.DOUBLE_LINE); System.out.println("Query Partition Touch Histogram:"); h = costmodel.getQueryPartitionAccessHistogram(); h.setKeepZeroEntries(true); h.putAll(all_partitions, 0); System.out.println(StringUtil.addSpacers(h.toString())); System.out.print(StringUtil.DOUBLE_LINE); } Map<String, Object> maps[] = new Map[2]; int idx = 0; ListOrderedMap<String, Object> m = null; // Execution Cost m = new ListOrderedMap<String, Object>(); m.put("SINGLE-PARTITION", singlepartition); m.put("MULTI-PARTITION", multipartition); m.put("TOTAL", total + " [" + singlepartition / (double) total + "]"); m.put("PARTITIONS TOUCHED (TXNS)", costmodel.getTxnPartitionAccessHistogram().getSampleCount()); m.put("PARTITIONS TOUCHED (QUERIES)", costmodel.getQueryPartitionAccessHistogram().getSampleCount()); maps[idx++] = m; // Utilization m = new ListOrderedMap<String, Object>(); costmodel.getJavaExecutionHistogram().setKeepZeroEntries(false); int active_partitions = costmodel.getJavaExecutionHistogram().getValueCount(); m.put("ACTIVE PARTITIONS", active_partitions); m.put("IDLE PARTITIONS", (all_partitions.size() - active_partitions)); // System.out.println("Partitions Touched By Queries: " + // total_partitions_touched_queries); Histogram<Integer> entropy_h = costmodel.getJavaExecutionHistogram(); m.put("JAVA SKEW", SkewFactorUtil.calculateSkew(all_partitions.size(), entropy_h.getSampleCount(), entropy_h)); entropy_h = costmodel.getTxnPartitionAccessHistogram(); m.put("TRANSACTION SKEW", SkewFactorUtil.calculateSkew(all_partitions.size(), entropy_h.getSampleCount(), entropy_h)); // TimeIntervalCostModel<SingleSitedCostModel> timecostmodel = new // TimeIntervalCostModel<SingleSitedCostModel>(args.catalog_db, // SingleSitedCostModel.class, 1); // timecostmodel.estimateCost(args.catalog_db, args.workload); // double entropy = timecostmodel.getLastEntropyCost() m.put("UTILIZATION", (costmodel.getJavaExecutionHistogram().getValueCount() / (double) all_partitions.size())); maps[idx++] = m; System.out.println(StringUtil.formatMaps(maps)); }
From source file:edu.brown.hstore.dtxn.LocalTransaction.java
@Override public String debug() { List<Map<String, Object>> maps = new ArrayList<Map<String, Object>>(); Map<String, Object> m; // Basic Info m = super.getDebugMap(); m.put("Procedure", this.getProcedureName()); maps.add(m);/*from w ww.j a va 2 s . c o m*/ // Predictions m = new ListOrderedMap<String, Object>(); m.put("Predict Single-Partitioned", (this.predict_touchedPartitions != null ? this.isPredictSinglePartition() : "???")); m.put("Predict Touched Partitions", this.getPredictTouchedPartitions()); m.put("Predict Read Only", this.isPredictReadOnly()); m.put("Predict Abortable", this.isPredictAbortable()); m.put("Restart Counter", this.restart_ctr); m.put("Deletable", this.deletable); m.put("Not Deletable", this.not_deletable); m.put("Needs Restart", this.needs_restart); m.put("Log Flushed", this.log_flushed); m.put("Estimator State", this.estimator_state); maps.add(m); m = new ListOrderedMap<String, Object>(); m.put("Exec Read Only", Arrays.toString(this.exec_readOnly)); m.put("Exec Touched Partitions", this.exec_touchedPartitions); // Actual Execution if (this.state != null) { m.put("Exec Single-Partitioned", this.isExecSinglePartition()); m.put("Speculative Execution", this.exec_speculative); m.put("Dependency Ctr", this.state.dependency_ctr); m.put("Received Ctr", this.state.received_ctr); m.put("CountdownLatch", this.state.dependency_latch); m.put("# of Blocked Tasks", this.state.blocked_tasks.size()); m.put("# of Statements", this.state.batch_size); m.put("Expected Results", this.state.results_dependency_stmt_ctr.keySet()); } maps.add(m); // Additional Info m = new ListOrderedMap<String, Object>(); m.put("Client Callback", this.client_callback); if (this.dtxnState != null) { m.put("Init Callback", this.dtxnState.init_callback); m.put("Prepare Callback", this.dtxnState.prepare_callback); m.put("Finish Callback", this.dtxnState.finish_callback); } maps.add(m); // Profile Times if (this.profiler != null) maps.add(this.profiler.debugMap()); StringBuilder sb = new StringBuilder(); sb.append(StringUtil.formatMaps(maps.toArray(new Map<?, ?>[maps.size()]))); if (this.state != null) { sb.append(StringUtil.SINGLE_LINE); String stmt_debug[] = new String[this.state.batch_size]; VoltProcedure voltProc = state.executor.getVoltProcedure(catalog_proc.getName()); assert (voltProc != null); SQLStmt stmts[] = voltProc.voltLastQueriesExecuted(); // This won't work in test cases // assert(stmt_debug.length == stmts.length) : // String.format("Expected %d SQLStmts but we only got %d", stmt_debug.length, stmts.length); for (int stmt_index = 0; stmt_index < stmt_debug.length; stmt_index++) { Map<Integer, DependencyInfo> s_dependencies = new HashMap<Integer, DependencyInfo>(); for (DependencyInfo dinfo : this.state.dependencies.values()) { if (dinfo.getStatementIndex() == stmt_index) s_dependencies.put(dinfo.getDependencyId(), dinfo); } // FOR String inner = " Statement #" + stmt_index; if (stmts != null && stmt_index < stmts.length) { inner += " - " + stmts[stmt_index].getStatement().getName(); } inner += "\n"; // inner += " Output Dependency Id: " + (this.state.output_order.contains(stmt_index) ? this.state.output_order.get(stmt_index) : "<NOT STARTED>") + "\n"; inner += " Dependency Partitions:\n"; for (Integer dependency_id : s_dependencies.keySet()) { inner += " [" + dependency_id + "] => " + s_dependencies.get(dependency_id).getPartitions() + "\n"; } // FOR inner += " Dependency Results:\n"; for (Integer dependency_id : s_dependencies.keySet()) { inner += " [" + dependency_id + "] => ["; String add = ""; for (VoltTable vt : s_dependencies.get(dependency_id).getResults()) { inner += add + (vt == null ? vt : "{" + vt.getRowCount() + " tuples}"); add = ","; } inner += "]\n"; } // FOR inner += " Blocked WorkFragments:\n"; boolean none = true; for (Integer dependency_id : s_dependencies.keySet()) { DependencyInfo d = s_dependencies.get(dependency_id); for (WorkFragment task : d.getBlockedWorkFragments()) { if (task == null) continue; inner += " [" + dependency_id + "] => ["; String add = ""; for (int frag_id : task.getFragmentIdList()) { inner += add + frag_id; add = ", "; } // FOR inner += "] "; if (d.hasTasksReady()) { inner += "*READY*"; } else if (d.hasTasksReleased()) { inner += "*RELEASED*"; } else { inner += String.format("%d / %d", d.getResults().size(), d.getPartitions().size()); } inner += "\n"; none = false; } } // FOR if (none) inner += " <none>\n"; stmt_debug[stmt_index] = inner; } // (dependencies) sb.append(StringUtil.columns(stmt_debug)); } return (sb.toString()); }