List of usage examples for com.google.common.collect Lists newArrayList
@GwtCompatible(serializable = true) public static <E> ArrayList<E> newArrayList()
From source file:com.yahoo.pulsar.testclient.PerformanceConsumer.java
public static void main(String[] args) throws Exception { final Arguments arguments = new Arguments(); JCommander jc = new JCommander(arguments); jc.setProgramName("pulsar-perf-consumer"); try {//from ww w . j av a 2s . c o m jc.parse(args); } catch (ParameterException e) { System.out.println(e.getMessage()); jc.usage(); System.exit(-1); } if (arguments.help) { jc.usage(); System.exit(-1); } if (arguments.topic.size() != 1) { System.out.println("Only one destination name is allowed"); jc.usage(); System.exit(-1); } if (arguments.confFile != null) { Properties prop = new Properties(System.getProperties()); prop.load(new FileInputStream(arguments.confFile)); if (arguments.serviceURL == null) { arguments.serviceURL = prop.getProperty("brokerServiceUrl"); } if (arguments.serviceURL == null) { arguments.serviceURL = prop.getProperty("webServiceUrl"); } // fallback to previous-version serviceUrl property to maintain backward-compatibility if (arguments.serviceURL == null) { arguments.serviceURL = prop.getProperty("serviceUrl", "http://localhost:8080/"); } if (arguments.authPluginClassName == null) { arguments.authPluginClassName = prop.getProperty("authPlugin", null); } if (arguments.authParams == null) { arguments.authParams = prop.getProperty("authParams", null); } } // Dump config variables ObjectMapper m = new ObjectMapper(); ObjectWriter w = m.writerWithDefaultPrettyPrinter(); log.info("Starting Pulsar performance consumer with config: {}", w.writeValueAsString(arguments)); final DestinationName prefixDestinationName = DestinationName.get(arguments.topic.get(0)); final RateLimiter limiter = arguments.rate > 0 ? RateLimiter.create(arguments.rate) : null; MessageListener listener = new MessageListener() { public void received(Consumer consumer, Message msg) { messagesReceived.increment(); bytesReceived.add(msg.getData().length); if (limiter != null) { limiter.acquire(); } consumer.acknowledgeAsync(msg); } }; EventLoopGroup eventLoopGroup; if (SystemUtils.IS_OS_LINUX) { eventLoopGroup = new EpollEventLoopGroup(Runtime.getRuntime().availableProcessors() * 2, new DefaultThreadFactory("pulsar-perf-consumer")); } else { eventLoopGroup = new NioEventLoopGroup(Runtime.getRuntime().availableProcessors(), new DefaultThreadFactory("pulsar-perf-consumer")); } ClientConfiguration clientConf = new ClientConfiguration(); clientConf.setConnectionsPerBroker(arguments.maxConnections); clientConf.setStatsInterval(arguments.statsIntervalSeconds, TimeUnit.SECONDS); if (isNotBlank(arguments.authPluginClassName)) { clientConf.setAuthentication(arguments.authPluginClassName, arguments.authParams); } PulsarClient pulsarClient = new PulsarClientImpl(arguments.serviceURL, clientConf, eventLoopGroup); List<Future<Consumer>> futures = Lists.newArrayList(); ConsumerConfiguration consumerConfig = new ConsumerConfiguration(); consumerConfig.setMessageListener(listener); consumerConfig.setReceiverQueueSize(arguments.receiverQueueSize); for (int i = 0; i < arguments.numDestinations; i++) { final DestinationName destinationName = (arguments.numDestinations == 1) ? prefixDestinationName : DestinationName.get(String.format("%s-%d", prefixDestinationName, i)); log.info("Adding {} consumers on destination {}", arguments.numConsumers, destinationName); for (int j = 0; j < arguments.numConsumers; j++) { String subscriberName; if (arguments.numConsumers > 1) { subscriberName = String.format("%s-%d", arguments.subscriberName, j); } else { subscriberName = arguments.subscriberName; } futures.add( pulsarClient.subscribeAsync(destinationName.toString(), subscriberName, consumerConfig)); } } for (Future<Consumer> future : futures) { future.get(); } log.info("Start receiving from {} consumers on {} destinations", arguments.numConsumers, arguments.numDestinations); long oldTime = System.nanoTime(); while (true) { try { Thread.sleep(10000); } catch (InterruptedException e) { break; } long now = System.nanoTime(); double elapsed = (now - oldTime) / 1e9; double rate = messagesReceived.sumThenReset() / elapsed; double throughput = bytesReceived.sumThenReset() / elapsed * 8 / 1024 / 1024; log.info("Throughput received: {} msg/s -- {} Mbit/s", dec.format(rate), dec.format(throughput)); oldTime = now; } pulsarClient.close(); }
From source file:com.android.idegen.IntellijProject.java
public static void main(String[] args) { logger.info("Args: " + Arrays.toString(args)); if (args.length < 3) { logger.severe("Not enough input arguments. Aborting"); return;//from w ww . ja v a 2s . co m } boolean buildFramework = true; int argIndex = 0; String arg = args[argIndex]; while (arg.startsWith("--")) { if (arg.equals("--no-framework")) { buildFramework = false; } argIndex++; arg = args[argIndex]; } String indexFile = args[argIndex++]; String projectPath = args[argIndex++]; // Remaining args are module directories ArrayList<String> moduleDirs = Lists.newArrayList(); for (int i = argIndex; i < args.length; i++) { moduleDirs.add(args[i]); } IntellijProject intellij = new IntellijProject(indexFile, projectPath, moduleDirs, buildFramework); try { intellij.build(); } catch (IOException e) { e.printStackTrace(); } }
From source file:org.bigmouth.nvwa.utils.xml.Dom4jEncoder.java
public static void main(String[] args) { List<O> l = Lists.newArrayList(); l.add(new O("200", "100", new C("A08032"))); l.add(new O("2300", "1200", new C("A08051"))); String xml = encode(l, "/items", "item"); System.out.println(xml);//ww w .j a v a2 s .co m }
From source file:com.maxpowered.amazon.advertising.api.app.App.java
public static void main(final String... args) throws FileNotFoundException, IOException, JAXBException, XMLStreamException, InterruptedException { try (ClassPathXmlApplicationContext ctx = new ClassPathXmlApplicationContext("application-context.xml")) { /*//from ww w . j ava 2 s . co m * Get default options based on spring configs */ final String inputDefault = getOptionDefaultBasedOnSpringProperty(ctx, PROPERTY_APP_INPUT, STD_IN_STR); final String processedDefault = inputDefault.equals(STD_IN_STR) ? DEFAULT_PROCESSED_FILE_BASE : inputDefault + PROCESSED_EXT; final String outputDefault = getOptionDefaultBasedOnSpringProperty(ctx, PROPERTY_APP_OUTPUT, STD_OUT_STR); int throttleDefault = Integer.valueOf(getOptionDefaultBasedOnSpringProperty(ctx, PROPERTY_APP_THROTTLE, String.valueOf(DEFAULT_APP_THROTTLE))); // Maximum of 25000 requests per hour throttleDefault = Math.min(throttleDefault, MAX_APP_THROTTLE); /* * Get options from the CLI args */ final Options options = new Options(); options.addOption("h", false, "Display this help."); options.addOption("i", true, "Set the file to read ASINs from. " + DEFAULT_STR + inputDefault); options.addOption("p", true, "Set the file to store processed ASINs in. " + DEFAULT_STR + processedDefault + " or '" + PROCESSED_EXT + "' appended to the input file name."); // Add a note that the output depends on the configured processors. If none are configured, it defaults to a // std.out processor options.addOption("o", true, "Set the file to write fetched info xml to via FileProcessor. " + DEFAULT_STR + outputDefault); options.addOption("1", false, "Override output file and always output fetched info xml to std.out."); options.addOption("t", true, "Set the requests per hour throttle (max of " + MAX_APP_THROTTLE + "). " + DEFAULT_STR + throttleDefault); final CommandLineParser parser = new DefaultParser(); CommandLine cmd = null; boolean needsHelp = false; try { cmd = parser.parse(options, args); } catch (final ParseException e) { needsHelp = true; } if (cmd.hasOption("h") || needsHelp) { final HelpFormatter formatter = new HelpFormatter(); formatter.printHelp("App", options); return; } // Get throttle rate final int throttle = Math.min( cmd.hasOption("t") ? Integer.valueOf(cmd.getOptionValue("t")) : throttleDefault, MAX_APP_THROTTLE); LOG.debug("Throttle (default {}) is {} requests per hour", throttleDefault, throttle); // We don't want to hit our limit, just under an hour worth of milliseconds final int requestWait = 3540000 / throttle; // Get input stream String input; if (cmd.hasOption("i")) { input = cmd.getOptionValue("i"); } else { input = inputDefault; } LOG.debug("Input name (default {}) is {}", inputDefault, input); // Get processed file String processed; if (cmd.hasOption("p")) { processed = cmd.getOptionValue("p"); } else { processed = input + PROCESSED_EXT; } LOG.debug("Processed file name (default {}) is {}", processedDefault, processed); final File processedFile = new File(processed); processedFile.createNewFile(); try (final InputStream inputStream = getInputStream(input)) { // Get output stream String output; if (cmd.hasOption("o")) { output = cmd.getOptionValue("o"); } else { output = outputDefault; } if (cmd.hasOption("1")) { output = STD_OUT_STR; } LOG.debug("Output (default {}) name is {}", outputDefault, output); // Special logic to set the FileProcessor output if (output.equals(STD_OUT_STR)) { final FileProcessor fileProcessor = ctx.getBeanFactory().getBean(FileProcessor.class); fileProcessor.setOutputStream(System.out); } else if (!output.equals(outputDefault)) { final FileProcessor fileProcessor = ctx.getBeanFactory().getBean(FileProcessor.class); fileProcessor.setOutputFile(output); } // This could be easily configured through CLI or properties final List<String> responseGroups = Lists.newArrayList(); for (final ResponseGroup responseGroup : new ResponseGroup[] { ResponseGroup.IMAGES, ResponseGroup.ITEM_ATTRIBUTES }) { responseGroups.add(responseGroup.getResponseGroupName()); } final String responseGroupString = Joiner.on(",").join(responseGroups); // Search the list of remaining ASINs final ProductFetcher fetcher = ctx.getBeanFactory().getBean(ProductFetcher.class); fetcher.setProcessedFile(processedFile); fetcher.setRequestWait(requestWait); fetcher.setInputStream(inputStream); fetcher.setResponseGroups(responseGroupString); // This ensures that statistics of processed asins should almost always get printed at the end Runtime.getRuntime().addShutdownHook(new Thread() { @Override public void run() { fetcher.logStatistics(); } }); fetcher.fetchProductInformation(); } } }
From source file:com.metamx.druid.http.MasterMain.java
public static void main(String[] args) throws Exception { LogLevelAdjuster.register();//w w w . j a va2 s. c o m final ObjectMapper jsonMapper = new DefaultObjectMapper(); final Properties props = Initialization.loadProperties(); final ConfigurationObjectFactory configFactory = Config.createFactory(props); final Lifecycle lifecycle = new Lifecycle(); final HttpClientConfig.Builder httpClientConfigBuilder = HttpClientConfig.builder().withNumConnections(1); final String emitterTimeout = props.getProperty("druid.emitter.timeOut"); if (emitterTimeout != null) { httpClientConfigBuilder.withReadTimeout(new Duration(emitterTimeout)); } final HttpClient httpClient = HttpClientInit.createClient(httpClientConfigBuilder.build(), lifecycle); final ServiceEmitter emitter = new ServiceEmitter(PropUtils.getProperty(props, "druid.service"), PropUtils.getProperty(props, "druid.host"), Emitters.create(props, httpClient, jsonMapper, lifecycle)); EmittingLogger.registerEmitter(emitter); final ScheduledExecutorFactory scheduledExecutorFactory = ScheduledExecutors.createFactory(lifecycle); final ServiceDiscoveryConfig serviceDiscoveryConfig = configFactory.build(ServiceDiscoveryConfig.class); CuratorFramework serviceDiscoveryCuratorFramework = Initialization .makeCuratorFramework(serviceDiscoveryConfig, lifecycle); final CuratorConfig curatorConfig = configFactory.build(CuratorConfig.class); CuratorFramework curatorFramework = Initialization.makeCuratorFramework(curatorConfig, lifecycle); final ZkPathsConfig zkPaths = configFactory.build(ZkPathsConfig.class); final ExecutorService exec = Executors.newFixedThreadPool(1, new ThreadFactoryBuilder().setDaemon(true).setNameFormat("ServerInventoryView-%s").build()); final ServerInventoryViewConfig serverInventoryViewConfig = configFactory .build(ServerInventoryViewConfig.class); final String announcerType = serverInventoryViewConfig.getAnnouncerType(); final ServerInventoryView serverInventoryView; if ("legacy".equalsIgnoreCase(announcerType)) { serverInventoryView = new SingleServerInventoryView(serverInventoryViewConfig, zkPaths, curatorFramework, exec, jsonMapper); } else if ("batch".equalsIgnoreCase(announcerType)) { serverInventoryView = new BatchServerInventoryView(serverInventoryViewConfig, zkPaths, curatorFramework, exec, jsonMapper); } else { throw new IAE("Unknown type %s", announcerType); } lifecycle.addManagedInstance(serverInventoryView); final DbConnectorConfig dbConnectorConfig = configFactory.build(DbConnectorConfig.class); final DatabaseRuleManagerConfig databaseRuleManagerConfig = configFactory .build(DatabaseRuleManagerConfig.class); final DBI dbi = new DbConnector(dbConnectorConfig).getDBI(); DbConnector.createSegmentTable(dbi, PropUtils.getProperty(props, "druid.database.segmentTable")); DbConnector.createRuleTable(dbi, PropUtils.getProperty(props, "druid.database.ruleTable")); DatabaseRuleManager.createDefaultRule(dbi, databaseRuleManagerConfig.getRuleTable(), databaseRuleManagerConfig.getDefaultDatasource(), jsonMapper); final DatabaseSegmentManager databaseSegmentManager = new DatabaseSegmentManager(jsonMapper, scheduledExecutorFactory.create(1, "DatabaseSegmentManager-Exec--%d"), configFactory.build(DatabaseSegmentManagerConfig.class), dbi); final DatabaseRuleManager databaseRuleManager = new DatabaseRuleManager(jsonMapper, scheduledExecutorFactory.create(1, "DatabaseRuleManager-Exec--%d"), databaseRuleManagerConfig, dbi); final ScheduledExecutorService globalScheduledExec = scheduledExecutorFactory.create(1, "Global--%d"); final List<Monitor> monitors = Lists.newArrayList(); monitors.add(new JvmMonitor()); if (Boolean.parseBoolean(props.getProperty("druid.monitoring.monitorSystem", "false"))) { monitors.add(new SysMonitor()); } final MonitorScheduler healthMonitor = new MonitorScheduler( configFactory.build(MonitorSchedulerConfig.class), globalScheduledExec, emitter, monitors); lifecycle.addManagedInstance(healthMonitor); final DruidMasterConfig druidMasterConfig = configFactory.build(DruidMasterConfig.class); final ServiceDiscovery serviceDiscovery = Initialization .makeServiceDiscoveryClient(serviceDiscoveryCuratorFramework, serviceDiscoveryConfig, lifecycle); final ServiceAnnouncer serviceAnnouncer = Initialization.makeServiceAnnouncer(serviceDiscoveryConfig, serviceDiscovery); Initialization.announceDefaultService(serviceDiscoveryConfig, serviceAnnouncer, lifecycle); ServiceProvider serviceProvider = null; if (druidMasterConfig.getMergerServiceName() != null) { serviceProvider = Initialization.makeServiceProvider(druidMasterConfig.getMergerServiceName(), serviceDiscovery, lifecycle); } IndexingServiceClient indexingServiceClient = new IndexingServiceClient(httpClient, jsonMapper, serviceProvider); final ConfigManagerConfig configManagerConfig = configFactory.build(ConfigManagerConfig.class); DbConnector.createConfigTable(dbi, configManagerConfig.getConfigTable()); JacksonConfigManager configManager = new JacksonConfigManager(new ConfigManager(dbi, configManagerConfig), jsonMapper); final LoadQueueTaskMaster taskMaster = new LoadQueueTaskMaster(curatorFramework, jsonMapper, scheduledExecutorFactory.create(1, "Master-PeonExec--%d"), druidMasterConfig); final DruidMaster master = new DruidMaster(druidMasterConfig, zkPaths, configManager, databaseSegmentManager, serverInventoryView, databaseRuleManager, curatorFramework, emitter, scheduledExecutorFactory, indexingServiceClient, taskMaster); lifecycle.addManagedInstance(master); try { lifecycle.start(); } catch (Throwable t) { log.error(t, "Error when starting up. Failing."); System.exit(1); } Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() { @Override public void run() { log.info("Running shutdown hook"); lifecycle.stop(); } })); final Injector injector = Guice .createInjector(new MasterServletModule(serverInventoryView, databaseSegmentManager, databaseRuleManager, master, jsonMapper, indexingServiceClient, configManager)); final Server server = Initialization.makeJettyServer(configFactory.build(ServerConfig.class)); final RedirectInfo redirectInfo = new RedirectInfo() { @Override public boolean doLocal() { return master.isClusterMaster(); } @Override public URL getRedirectURL(String queryString, String requestURI) { try { final String currentMaster = master.getCurrentMaster(); if (currentMaster == null) { return null; } String location = String.format("http://%s%s", currentMaster, requestURI); if (queryString != null) { location = String.format("%s?%s", location, queryString); } return new URL(location); } catch (Exception e) { throw Throwables.propagate(e); } } }; final Context staticContext = new Context(server, "/static", Context.SESSIONS); staticContext.addServlet(new ServletHolder(new RedirectServlet(redirectInfo)), "/*"); staticContext.setResourceBase(ComputeMain.class.getClassLoader().getResource("static").toExternalForm()); final Context root = new Context(server, "/", Context.SESSIONS); root.addServlet(new ServletHolder(new StatusServlet()), "/status"); root.addServlet(new ServletHolder(new DefaultServlet()), "/*"); root.addEventListener(new GuiceServletConfig(injector)); root.addFilter(GzipFilter.class, "/*", 0); root.addFilter( new FilterHolder(new RedirectFilter(new ToStringResponseHandler(Charsets.UTF_8), redirectInfo)), "/*", 0); root.addFilter(GuiceFilter.class, "/info/*", 0); root.addFilter(GuiceFilter.class, "/master/*", 0); server.start(); server.join(); }
From source file:com.yahoo.pulsar.testclient.PerformanceProducer.java
public static void main(String[] args) throws Exception { final Arguments arguments = new Arguments(); JCommander jc = new JCommander(arguments); jc.setProgramName("pulsar-perf-producer"); try {/*from ww w . j a va 2 s .c o m*/ jc.parse(args); } catch (ParameterException e) { System.out.println(e.getMessage()); jc.usage(); System.exit(-1); } if (arguments.help) { jc.usage(); System.exit(-1); } if (arguments.destinations.size() != 1) { System.out.println("Only one topic name is allowed"); jc.usage(); System.exit(-1); } if (arguments.confFile != null) { Properties prop = new Properties(System.getProperties()); prop.load(new FileInputStream(arguments.confFile)); if (arguments.serviceURL == null) { arguments.serviceURL = prop.getProperty("brokerServiceUrl"); } if (arguments.serviceURL == null) { arguments.serviceURL = prop.getProperty("webServiceUrl"); } // fallback to previous-version serviceUrl property to maintain backward-compatibility if (arguments.serviceURL == null) { arguments.serviceURL = prop.getProperty("serviceUrl", "http://localhost:8080/"); } if (arguments.authPluginClassName == null) { arguments.authPluginClassName = prop.getProperty("authPlugin", null); } if (arguments.authParams == null) { arguments.authParams = prop.getProperty("authParams", null); } } arguments.testTime = TimeUnit.SECONDS.toMillis(arguments.testTime); // Dump config variables ObjectMapper m = new ObjectMapper(); ObjectWriter w = m.writerWithDefaultPrettyPrinter(); log.info("Starting Pulsar perf producer with config: {}", w.writeValueAsString(arguments)); // Read payload data from file if needed byte payloadData[]; if (arguments.payloadFilename != null) { payloadData = Files.readAllBytes(Paths.get(arguments.payloadFilename)); } else { payloadData = new byte[arguments.msgSize]; } // Now processing command line arguments String prefixTopicName = arguments.destinations.get(0); List<Future<Producer>> futures = Lists.newArrayList(); EventLoopGroup eventLoopGroup; if (SystemUtils.IS_OS_LINUX) { eventLoopGroup = new EpollEventLoopGroup(Runtime.getRuntime().availableProcessors(), new DefaultThreadFactory("pulsar-perf-producer")); } else { eventLoopGroup = new NioEventLoopGroup(Runtime.getRuntime().availableProcessors(), new DefaultThreadFactory("pulsar-perf-producer")); } ClientConfiguration clientConf = new ClientConfiguration(); clientConf.setConnectionsPerBroker(arguments.maxConnections); clientConf.setStatsInterval(arguments.statsIntervalSeconds, TimeUnit.SECONDS); if (isNotBlank(arguments.authPluginClassName)) { clientConf.setAuthentication(arguments.authPluginClassName, arguments.authParams); } PulsarClient client = new PulsarClientImpl(arguments.serviceURL, clientConf, eventLoopGroup); ProducerConfiguration producerConf = new ProducerConfiguration(); producerConf.setSendTimeout(0, TimeUnit.SECONDS); producerConf.setCompressionType(arguments.compression); // enable round robin message routing if it is a partitioned topic producerConf.setMessageRoutingMode(MessageRoutingMode.RoundRobinPartition); if (arguments.batchTime > 0) { producerConf.setBatchingMaxPublishDelay(arguments.batchTime, TimeUnit.MILLISECONDS); producerConf.setBatchingEnabled(true); producerConf.setMaxPendingMessages(arguments.msgRate); } for (int i = 0; i < arguments.numTopics; i++) { String topic = (arguments.numTopics == 1) ? prefixTopicName : String.format("%s-%d", prefixTopicName, i); log.info("Adding {} publishers on destination {}", arguments.numProducers, topic); for (int j = 0; j < arguments.numProducers; j++) { futures.add(client.createProducerAsync(topic, producerConf)); } } final List<Producer> producers = Lists.newArrayListWithCapacity(futures.size()); for (Future<Producer> future : futures) { producers.add(future.get()); } log.info("Created {} producers", producers.size()); Runtime.getRuntime().addShutdownHook(new Thread() { public void run() { printAggregatedStats(); } }); Collections.shuffle(producers); AtomicBoolean isDone = new AtomicBoolean(); executor.submit(() -> { try { RateLimiter rateLimiter = RateLimiter.create(arguments.msgRate); long startTime = System.currentTimeMillis(); // Send messages on all topics/producers long totalSent = 0; while (true) { for (Producer producer : producers) { if (arguments.testTime > 0) { if (System.currentTimeMillis() - startTime > arguments.testTime) { log.info("------------------- DONE -----------------------"); printAggregatedStats(); isDone.set(true); Thread.sleep(5000); System.exit(0); } } if (arguments.numMessages > 0) { if (totalSent++ >= arguments.numMessages) { log.info("------------------- DONE -----------------------"); printAggregatedStats(); isDone.set(true); Thread.sleep(5000); System.exit(0); } } rateLimiter.acquire(); final long sendTime = System.nanoTime(); producer.sendAsync(payloadData).thenRun(() -> { messagesSent.increment(); bytesSent.add(payloadData.length); long latencyMicros = NANOSECONDS.toMicros(System.nanoTime() - sendTime); recorder.recordValue(latencyMicros); cumulativeRecorder.recordValue(latencyMicros); }).exceptionally(ex -> { log.warn("Write error on message", ex); System.exit(-1); return null; }); } } } catch (Throwable t) { log.error("Got error", t); } }); // Print report stats long oldTime = System.nanoTime(); Histogram reportHistogram = null; String statsFileName = "perf-producer-" + System.currentTimeMillis() + ".hgrm"; log.info("Dumping latency stats to {}", statsFileName); PrintStream histogramLog = new PrintStream(new FileOutputStream(statsFileName), false); HistogramLogWriter histogramLogWriter = new HistogramLogWriter(histogramLog); // Some log header bits histogramLogWriter.outputLogFormatVersion(); histogramLogWriter.outputLegend(); while (true) { try { Thread.sleep(10000); } catch (InterruptedException e) { break; } if (isDone.get()) { break; } long now = System.nanoTime(); double elapsed = (now - oldTime) / 1e9; double rate = messagesSent.sumThenReset() / elapsed; double throughput = bytesSent.sumThenReset() / elapsed / 1024 / 1024 * 8; reportHistogram = recorder.getIntervalHistogram(reportHistogram); log.info( "Throughput produced: {} msg/s --- {} Mbit/s --- Latency: mean: {} ms - med: {} - 95pct: {} - 99pct: {} - 99.9pct: {} - 99.99pct: {} - Max: {}", throughputFormat.format(rate), throughputFormat.format(throughput), dec.format(reportHistogram.getMean() / 1000.0), dec.format(reportHistogram.getValueAtPercentile(50) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(95) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99.9) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99.99) / 1000.0), dec.format(reportHistogram.getMaxValue() / 1000.0)); histogramLogWriter.outputIntervalHistogram(reportHistogram); reportHistogram.reset(); oldTime = now; } client.close(); }
From source file:org.apache.pulsar.testclient.PerformanceProducer.java
public static void main(String[] args) throws Exception { final Arguments arguments = new Arguments(); JCommander jc = new JCommander(arguments); jc.setProgramName("pulsar-perf-producer"); try {/*from w w w .j av a 2 s.c om*/ jc.parse(args); } catch (ParameterException e) { System.out.println(e.getMessage()); jc.usage(); System.exit(-1); } if (arguments.help) { jc.usage(); System.exit(-1); } if (arguments.destinations.size() != 1) { System.out.println("Only one topic name is allowed"); jc.usage(); System.exit(-1); } if (arguments.confFile != null) { Properties prop = new Properties(System.getProperties()); prop.load(new FileInputStream(arguments.confFile)); if (arguments.serviceURL == null) { arguments.serviceURL = prop.getProperty("brokerServiceUrl"); } if (arguments.serviceURL == null) { arguments.serviceURL = prop.getProperty("webServiceUrl"); } // fallback to previous-version serviceUrl property to maintain backward-compatibility if (arguments.serviceURL == null) { arguments.serviceURL = prop.getProperty("serviceUrl", "http://localhost:8080/"); } if (arguments.authPluginClassName == null) { arguments.authPluginClassName = prop.getProperty("authPlugin", null); } if (arguments.authParams == null) { arguments.authParams = prop.getProperty("authParams", null); } } arguments.testTime = TimeUnit.SECONDS.toMillis(arguments.testTime); // Dump config variables ObjectMapper m = new ObjectMapper(); ObjectWriter w = m.writerWithDefaultPrettyPrinter(); log.info("Starting Pulsar perf producer with config: {}", w.writeValueAsString(arguments)); // Read payload data from file if needed byte payloadData[]; if (arguments.payloadFilename != null) { payloadData = Files.readAllBytes(Paths.get(arguments.payloadFilename)); } else { payloadData = new byte[arguments.msgSize]; } // Now processing command line arguments String prefixTopicName = arguments.destinations.get(0); List<Future<Producer>> futures = Lists.newArrayList(); EventLoopGroup eventLoopGroup; if (SystemUtils.IS_OS_LINUX) { eventLoopGroup = new EpollEventLoopGroup(Runtime.getRuntime().availableProcessors(), new DefaultThreadFactory("pulsar-perf-producer")); } else { eventLoopGroup = new NioEventLoopGroup(Runtime.getRuntime().availableProcessors(), new DefaultThreadFactory("pulsar-perf-producer")); } ClientConfiguration clientConf = new ClientConfiguration(); clientConf.setConnectionsPerBroker(arguments.maxConnections); clientConf.setStatsInterval(arguments.statsIntervalSeconds, TimeUnit.SECONDS); if (isNotBlank(arguments.authPluginClassName)) { clientConf.setAuthentication(arguments.authPluginClassName, arguments.authParams); } PulsarClient client = new PulsarClientImpl(arguments.serviceURL, clientConf, eventLoopGroup); ProducerConfiguration producerConf = new ProducerConfiguration(); producerConf.setSendTimeout(0, TimeUnit.SECONDS); producerConf.setCompressionType(arguments.compression); // enable round robin message routing if it is a partitioned topic producerConf.setMessageRoutingMode(MessageRoutingMode.RoundRobinPartition); if (arguments.batchTime > 0) { producerConf.setBatchingMaxPublishDelay(arguments.batchTime, TimeUnit.MILLISECONDS); producerConf.setBatchingEnabled(true); producerConf.setMaxPendingMessages(arguments.msgRate); } // Block if queue is full else we will start seeing errors in sendAsync producerConf.setBlockIfQueueFull(true); for (int i = 0; i < arguments.numTopics; i++) { String topic = (arguments.numTopics == 1) ? prefixTopicName : String.format("%s-%d", prefixTopicName, i); log.info("Adding {} publishers on destination {}", arguments.numProducers, topic); for (int j = 0; j < arguments.numProducers; j++) { futures.add(client.createProducerAsync(topic, producerConf)); } } final List<Producer> producers = Lists.newArrayListWithCapacity(futures.size()); for (Future<Producer> future : futures) { producers.add(future.get()); } log.info("Created {} producers", producers.size()); Runtime.getRuntime().addShutdownHook(new Thread() { public void run() { printAggregatedStats(); } }); Collections.shuffle(producers); AtomicBoolean isDone = new AtomicBoolean(); executor.submit(() -> { try { RateLimiter rateLimiter = RateLimiter.create(arguments.msgRate); long startTime = System.currentTimeMillis(); // Send messages on all topics/producers long totalSent = 0; while (true) { for (Producer producer : producers) { if (arguments.testTime > 0) { if (System.currentTimeMillis() - startTime > arguments.testTime) { log.info("------------------- DONE -----------------------"); printAggregatedStats(); isDone.set(true); Thread.sleep(5000); System.exit(0); } } if (arguments.numMessages > 0) { if (totalSent++ >= arguments.numMessages) { log.info("------------------- DONE -----------------------"); printAggregatedStats(); isDone.set(true); Thread.sleep(5000); System.exit(0); } } rateLimiter.acquire(); final long sendTime = System.nanoTime(); producer.sendAsync(payloadData).thenRun(() -> { messagesSent.increment(); bytesSent.add(payloadData.length); long latencyMicros = NANOSECONDS.toMicros(System.nanoTime() - sendTime); recorder.recordValue(latencyMicros); cumulativeRecorder.recordValue(latencyMicros); }).exceptionally(ex -> { log.warn("Write error on message", ex); System.exit(-1); return null; }); } } } catch (Throwable t) { log.error("Got error", t); } }); // Print report stats long oldTime = System.nanoTime(); Histogram reportHistogram = null; String statsFileName = "perf-producer-" + System.currentTimeMillis() + ".hgrm"; log.info("Dumping latency stats to {}", statsFileName); PrintStream histogramLog = new PrintStream(new FileOutputStream(statsFileName), false); HistogramLogWriter histogramLogWriter = new HistogramLogWriter(histogramLog); // Some log header bits histogramLogWriter.outputLogFormatVersion(); histogramLogWriter.outputLegend(); while (true) { try { Thread.sleep(10000); } catch (InterruptedException e) { break; } if (isDone.get()) { break; } long now = System.nanoTime(); double elapsed = (now - oldTime) / 1e9; double rate = messagesSent.sumThenReset() / elapsed; double throughput = bytesSent.sumThenReset() / elapsed / 1024 / 1024 * 8; reportHistogram = recorder.getIntervalHistogram(reportHistogram); log.info( "Throughput produced: {} msg/s --- {} Mbit/s --- Latency: mean: {} ms - med: {} - 95pct: {} - 99pct: {} - 99.9pct: {} - 99.99pct: {} - Max: {}", throughputFormat.format(rate), throughputFormat.format(throughput), dec.format(reportHistogram.getMean() / 1000.0), dec.format(reportHistogram.getValueAtPercentile(50) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(95) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99.9) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99.99) / 1000.0), dec.format(reportHistogram.getMaxValue() / 1000.0)); histogramLogWriter.outputIntervalHistogram(reportHistogram); reportHistogram.reset(); oldTime = now; } client.close(); }
From source file:org.apache.mahout.ga.watchmaker.cd.tool.CDInfosTool.java
public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException { GroupBuilder gbuilder = new GroupBuilder(); Option inputOpt = DefaultOptionCreator.inputOption().create(); Option helpOpt = DefaultOptionCreator.helpOption(); Group group = gbuilder.withName("Options").withOption(inputOpt).withOption(helpOpt).create(); Parser parser = new Parser(); parser.setGroup(group);//from www .j ava 2 s. com try { CommandLine cmdLine = parser.parse(args); if (cmdLine.hasOption(helpOpt)) { CommandLineUtil.printHelp(group); return; } Path input = new Path(cmdLine.getValue(inputOpt).toString()); Path output = new Path("output"); // TODO surely this should be configurable? FileSystem fs = FileSystem.get(input.toUri(), new Configuration()); log.info("Loading Descriptors..."); Descriptors descriptors = loadDescriptors(fs, input); log.info("Gathering informations..."); List<String> descriptions = Lists.newArrayList(); gatherInfos(descriptors, input, output, descriptions); log.info("Storing Descriptions..."); storeDescriptions(fs, input, descriptors, descriptions); } catch (OptionException e) { log.error("Error while parsing options", e); CommandLineUtil.printHelp(group); } }
From source file:org.apache.flume.channel.file.CheckpointRebuilder.java
public static void main(String[] args) throws Exception { Options options = new Options(); Option opt = new Option("c", true, "checkpoint directory"); opt.setRequired(true);/* w ww . j a v a 2 s . c o m*/ options.addOption(opt); opt = new Option("l", true, "comma-separated list of log directories"); opt.setRequired(true); options.addOption(opt); options.addOption(opt); opt = new Option("t", true, "capacity of the channel"); opt.setRequired(true); options.addOption(opt); CommandLineParser parser = new GnuParser(); CommandLine cli = parser.parse(options, args); File checkpointDir = new File(cli.getOptionValue("c")); String[] logDirs = cli.getOptionValue("l").split(","); List<File> logFiles = Lists.newArrayList(); for (String logDir : logDirs) { logFiles.addAll(LogUtils.getLogs(new File(logDir))); } int capacity = Integer.parseInt(cli.getOptionValue("t")); File checkpointFile = new File(checkpointDir, "checkpoint"); if (checkpointFile.exists()) { LOG.error("Cannot execute fast replay", new IllegalStateException("Checkpoint exists" + checkpointFile)); } else { EventQueueBackingStore backingStore = EventQueueBackingStoreFactory.get(checkpointFile, capacity, "channel"); FlumeEventQueue queue = new FlumeEventQueue(backingStore, new File(checkpointDir, "inflighttakes"), new File(checkpointDir, "inflightputs")); CheckpointRebuilder rebuilder = new CheckpointRebuilder(logFiles, queue); if (rebuilder.rebuild()) { rebuilder.writeCheckpoint(); } else { LOG.error("Could not rebuild the checkpoint due to errors."); } } }
From source file:org.apache.jackrabbit.oak.scalability.ScalabilityRunner.java
public static void main(String[] args) throws Exception { OptionParser parser = new OptionParser(); OptionSpec<File> base = parser.accepts("base", "Base directory").withRequiredArg().ofType(File.class) .defaultsTo(new File("target")); OptionSpec<String> host = parser.accepts("host", "MongoDB host").withRequiredArg().defaultsTo("localhost"); OptionSpec<Integer> port = parser.accepts("port", "MongoDB port").withRequiredArg().ofType(Integer.class) .defaultsTo(27017);//w w w .jav a 2s. c o m OptionSpec<String> dbName = parser.accepts("db", "MongoDB database").withRequiredArg(); OptionSpec<Boolean> dropDBAfterTest = parser .accepts("dropDBAfterTest", "Whether to drop the MongoDB database after the test").withOptionalArg() .ofType(Boolean.class).defaultsTo(true); OptionSpec<String> rdbjdbcuri = parser.accepts("rdbjdbcuri", "RDB JDBC URI").withOptionalArg() .defaultsTo("jdbc:h2:./target/benchmark"); OptionSpec<String> rdbjdbcuser = parser.accepts("rdbjdbcuser", "RDB JDBC user").withOptionalArg() .defaultsTo(""); OptionSpec<String> rdbjdbcpasswd = parser.accepts("rdbjdbcpasswd", "RDB JDBC password").withOptionalArg() .defaultsTo(""); OptionSpec<String> rdbjdbctableprefix = parser.accepts("rdbjdbctableprefix", "RDB JDBC table prefix") .withOptionalArg().defaultsTo(""); OptionSpec<Boolean> mmap = parser.accepts("mmap", "TarMK memory mapping").withOptionalArg() .ofType(Boolean.class).defaultsTo("64".equals(System.getProperty("sun.arch.data.model"))); OptionSpec<Integer> cache = parser.accepts("cache", "cache size (MB)").withRequiredArg() .ofType(Integer.class).defaultsTo(100); OptionSpec<Integer> fdsCache = parser.accepts("blobCache", "cache size (MB)").withRequiredArg() .ofType(Integer.class).defaultsTo(32); OptionSpec<Boolean> withStorage = parser.accepts("storage", "Index storage enabled").withOptionalArg() .ofType(Boolean.class); OptionSpec<File> csvFile = parser.accepts("csvFile", "File to write a CSV version of the benchmark data.") .withOptionalArg().ofType(File.class); OptionSpec help = parser.acceptsAll(asList("h", "?", "help"), "show help").forHelp(); OptionSpec<String> nonOption = parser.nonOptions(); OptionSet options = parser.parse(args); if (options.has(help)) { parser.printHelpOn(System.out); System.exit(0); } int cacheSize = cache.value(options); RepositoryFixture[] allFixtures = new RepositoryFixture[] { new JackrabbitRepositoryFixture(base.value(options), cacheSize), OakRepositoryFixture.getMemoryNS(cacheSize * MB), OakRepositoryFixture.getMongo(host.value(options), port.value(options), dbName.value(options), dropDBAfterTest.value(options), cacheSize * MB), OakRepositoryFixture.getMongoWithFDS(host.value(options), port.value(options), dbName.value(options), dropDBAfterTest.value(options), cacheSize * MB, base.value(options), fdsCache.value(options)), OakRepositoryFixture.getMongoNS(host.value(options), port.value(options), dbName.value(options), dropDBAfterTest.value(options), cacheSize * MB), OakRepositoryFixture.getTar(base.value(options), 256, cacheSize, mmap.value(options)), OakRepositoryFixture.getTarWithBlobStore(base.value(options), 256, cacheSize, mmap.value(options)), OakRepositoryFixture.getSegmentTar(base.value(options), 256, cacheSize, mmap.value(options)), OakRepositoryFixture.getSegmentTarWithBlobStore(base.value(options), 256, cacheSize, mmap.value(options)), OakRepositoryFixture.getRDB(rdbjdbcuri.value(options), rdbjdbcuser.value(options), rdbjdbcpasswd.value(options), rdbjdbctableprefix.value(options), dropDBAfterTest.value(options), cacheSize * MB), OakRepositoryFixture.getRDBWithFDS(rdbjdbcuri.value(options), rdbjdbcuser.value(options), rdbjdbcpasswd.value(options), rdbjdbctableprefix.value(options), dropDBAfterTest.value(options), cacheSize * MB, base.value(options), fdsCache.value(options)) }; ScalabilitySuite[] allSuites = new ScalabilitySuite[] { new ScalabilityBlobSearchSuite(withStorage.value(options)).addBenchmarks(new FullTextSearcher(), new NodeTypeSearcher(), new FormatSearcher(), new FacetSearcher(), new LastModifiedSearcher(Date.LAST_2_HRS), new LastModifiedSearcher(Date.LAST_24_HRS), new LastModifiedSearcher(Date.LAST_7_DAYS), new LastModifiedSearcher(Date.LAST_MONTH), new LastModifiedSearcher(Date.LAST_YEAR), new OrderByDate()), new ScalabilityNodeSuite(withStorage.value(options)).addBenchmarks(new OrderBySearcher(), new SplitOrderBySearcher(), new OrderByOffsetPageSearcher(), new SplitOrderByOffsetPageSearcher(), new OrderByKeysetPageSearcher(), new SplitOrderByKeysetPageSearcher(), new MultiFilterOrderBySearcher(), new MultiFilterSplitOrderBySearcher(), new MultiFilterOrderByOffsetPageSearcher(), new MultiFilterSplitOrderByOffsetPageSearcher(), new MultiFilterOrderByKeysetPageSearcher(), new MultiFilterSplitOrderByKeysetPageSearcher(), new ConcurrentReader(), new ConcurrentWriter()), new ScalabilityNodeRelationshipSuite(withStorage.value(options)) .addBenchmarks(new AggregateNodeSearcher()) }; Set<String> argset = Sets.newHashSet(nonOption.values(options)); List<RepositoryFixture> fixtures = Lists.newArrayList(); for (RepositoryFixture fixture : allFixtures) { if (argset.remove(fixture.toString())) { fixtures.add(fixture); } } Map<String, List<String>> argmap = Maps.newHashMap(); // Split the args to get suites and benchmarks (i.e. suite:benchmark1,benchmark2) for (String arg : argset) { List<String> tokens = Splitter.on(":").limit(2).splitToList(arg); if (tokens.size() > 1) { argmap.put(tokens.get(0), Splitter.on(",").trimResults().splitToList(tokens.get(1))); } else { argmap.put(tokens.get(0), null); } argset.remove(arg); } if (argmap.isEmpty()) { System.err.println( "Warning: no scalability suites specified, " + "supported are: " + Arrays.asList(allSuites)); } List<ScalabilitySuite> suites = Lists.newArrayList(); for (ScalabilitySuite suite : allSuites) { if (argmap.containsKey(suite.toString())) { List<String> benchmarks = argmap.get(suite.toString()); // Only keep requested benchmarks if (benchmarks != null) { Iterator<String> iter = suite.getBenchmarks().keySet().iterator(); for (; iter.hasNext();) { String availBenchmark = iter.next(); if (!benchmarks.contains(availBenchmark)) { iter.remove(); } } } suites.add(suite); argmap.remove(suite.toString()); } } if (argmap.isEmpty()) { PrintStream out = null; if (options.has(csvFile)) { out = new PrintStream(FileUtils.openOutputStream(csvFile.value(options), true), false, Charsets.UTF_8.name()); } for (ScalabilitySuite suite : suites) { if (suite instanceof CSVResultGenerator) { ((CSVResultGenerator) suite).setPrintStream(out); } suite.run(fixtures); } if (out != null) { out.close(); } } else { System.err.println("Unknown arguments: " + argset); } }