List of usage examples for com.fasterxml.jackson.databind ObjectMapper writerWithDefaultPrettyPrinter
public ObjectWriter writerWithDefaultPrettyPrinter()
From source file:de.oth.keycloak.CreateConfigSchema.java
public static void main(String[] args) { ObjectMapper mapper = new ObjectMapper(); try {/*w ww .j a v a2 s . c o m*/ JsonSchema schema = mapper.generateJsonSchema(RealmsConfig.class); String s = mapper.writerWithDefaultPrettyPrinter().writeValueAsString(schema); System.out.println(s); } catch (Exception e) { e.printStackTrace(); } }
From source file:org.niord.core.ChartConverter.java
public static void main(String[] args) throws IOException { String csvPath = "/Users/carolus/Downloads/charts.csv"; String resultPath = "/Users/carolus/Downloads/charts.json"; List<SystemChartVo> charts = new ArrayList<>(); Files.readAllLines(Paths.get(csvPath), Charset.forName("UTF-8")).forEach(line -> { String[] fields = line.split(";"); SystemChartVo chart = new SystemChartVo(); chart.setChartNumber(fields[0].split(" ")[1].trim()); if (StringUtils.isNotBlank(fields[1]) && StringUtils.isNumeric(fields[1])) { chart.setInternationalNumber(Integer.valueOf(fields[1])); }//from w w w . ja v a 2s.c o m chart.setName(StringUtils.defaultIfBlank(fields[3], "")); if (StringUtils.isNotBlank(fields[4]) && StringUtils.isNumeric(fields[4])) { chart.setScale(Integer.valueOf(fields[4])); } if (!"Ukendt / Unknown".equals(fields[5])) { chart.setHorizontalDatum(StringUtils.defaultIfBlank(fields[5], "")); } Double south = parsePos(fields[6]); Double west = -parsePos(fields[7]); Double north = parsePos(fields[8]); Double east = -parsePos(fields[9]); double[][] coords = new double[][] { { east, north }, { east, south }, { west, south }, { west, north }, { east, north }, }; PolygonVo geometry = new PolygonVo(); geometry.setCoordinates(new double[][][] { coords }); GeoJsonUtils.roundCoordinates(geometry, 8); chart.setGeometry(geometry); charts.add(chart); }); ObjectMapper mapper = new ObjectMapper(); System.out.println(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(charts)); try (BufferedWriter writer = Files.newBufferedWriter(Paths.get(resultPath), StandardCharsets.UTF_8)) { //writer.write("\uFEFF"); writer.write(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(charts)); } }
From source file:la.alsocan.jsonshapeshifter.Main.java
/** * An entry point, to test the library in action. * @param args Console args (nothing expected here so far) * @throws IOException if the payload cannot be read *//*from ww w . j av a 2s. c o m*/ public static void main(String[] args) throws IOException { Schema source; Schema target; try { source = Schema.buildSchema(SOURCE_SCHEMA); target = Schema.buildSchema(TARGET_SCHEMA); } catch (IOException ex) { System.err.println("Oups: " + ex); return; } // build the transformation incrementally Transformation t = new Transformation(source, target); Iterator<SchemaNode> it = t.toBind(); t.bind(it.next(), new ArrayNodeBinding(source.at("/rootSourceArray"))); t.bind(it.next(), new ArrayNodeBinding(source.at("/rootSourceArray/{i}"))); t.bind(it.next(), new StringConstantBinding("Constant value")); t.bind(it.next(), new StringNodeBinding(source.at("/rootSourceString"))); t.bind(it.next(), new ArrayNodeBinding(source.at("/rootSourceArray/{i}/{i}/someSourceArray"))); t.bind(it.next(), new StringNodeBinding(source.at("/rootSourceArray/{i}/{i}/someSourceArray/{i}"))); // produce something JsonNode payload = new ObjectMapper().readTree(new File(SOURCE_PAYLOAD)); JsonNode result = t.apply(payload); ObjectMapper om = new ObjectMapper(); try { System.out.println("Original payload:"); System.out.println(om.writerWithDefaultPrettyPrinter().writeValueAsString(payload)); System.out.println("\nResulting payload:"); System.out.println(om.writerWithDefaultPrettyPrinter().writeValueAsString(result)); } catch (JsonProcessingException ex) { System.err.println("Oups: " + ex); } }
From source file:org.force66.json.tools.JSonSchemaGenerator.java
public static void main(String[] args) throws Exception { Validate.isTrue(args != null && args.length >= 2, "Usage arguments: className fileTarget"); String className = args[0];/* ww w . j a va2 s . c o m*/ String outputFileName = args[1]; HyperSchemaFactoryWrapper schemaVisitor = new HyperSchemaFactoryWrapper(); ObjectMapper mapper = new ObjectMapper(); mapper.acceptJsonFormatVisitor(ClassUtils.getClass(className), schemaVisitor); JsonSchema jsonSchema = schemaVisitor.finalSchema(); File outputFile = new File(outputFileName); FileUtils.write(outputFile, mapper.writerWithDefaultPrettyPrinter().writeValueAsString(jsonSchema)); }
From source file:com.titan.main.Invoker.java
public static void main(String... args) { try {//from w ww .ja v a 2s . co m NotificationsFactory factory = new NotificationsFactory(); ObjectMapper mapper = new ObjectMapper(); mapper.addMixInAnnotations(Notifications.class, NotificationsMixin.class); mapper.addMixInAnnotations(NotificationsInfo.class, NotificationsInfoMixin.class); Notifications notifications = factory.getNotifications(); mapper.writerWithDefaultPrettyPrinter().writeValue( new File("D:/workspace/nbworkspace/jackson-sample/notifications.json"), notifications); System.out.println("Done!"); } catch (Exception e) { e.printStackTrace(); } }
From source file:io.druid.server.sql.SQLRunner.java
public static void main(String[] args) throws Exception { Options options = new Options(); options.addOption("h", "help", false, "help"); options.addOption("v", false, "verbose"); options.addOption("e", "host", true, "endpoint [hostname:port]"); CommandLine cmd = new GnuParser().parse(options, args); if (cmd.hasOption("h")) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp("SQLRunner", options); System.exit(2);/* w w w . j a v a 2s . co m*/ } String hostname = cmd.getOptionValue("e", "localhost:8080"); String sql = cmd.getArgs().length > 0 ? cmd.getArgs()[0] : STATEMENT; ObjectMapper objectMapper = new DefaultObjectMapper(); ObjectWriter jsonWriter = objectMapper.writerWithDefaultPrettyPrinter(); CharStream stream = new ANTLRInputStream(sql); DruidSQLLexer lexer = new DruidSQLLexer(stream); TokenStream tokenStream = new CommonTokenStream(lexer); DruidSQLParser parser = new DruidSQLParser(tokenStream); lexer.removeErrorListeners(); parser.removeErrorListeners(); lexer.addErrorListener(ConsoleErrorListener.INSTANCE); parser.addErrorListener(ConsoleErrorListener.INSTANCE); try { DruidSQLParser.QueryContext queryContext = parser.query(); if (parser.getNumberOfSyntaxErrors() > 0) throw new IllegalStateException(); // parser.setBuildParseTree(true); // System.err.println(q.toStringTree(parser)); } catch (Exception e) { String msg = e.getMessage(); if (msg != null) System.err.println(e); System.exit(1); } final Query query; final TypeReference typeRef; boolean groupBy = false; if (parser.groupByDimensions.isEmpty()) { query = Druids.newTimeseriesQueryBuilder().dataSource(parser.getDataSource()) .aggregators(new ArrayList<AggregatorFactory>(parser.aggregators.values())) .postAggregators(parser.postAggregators).intervals(parser.intervals) .granularity(parser.granularity).filters(parser.filter).build(); typeRef = new TypeReference<List<Result<TimeseriesResultValue>>>() { }; } else { query = GroupByQuery.builder().setDataSource(parser.getDataSource()) .setAggregatorSpecs(new ArrayList<AggregatorFactory>(parser.aggregators.values())) .setPostAggregatorSpecs(parser.postAggregators).setInterval(parser.intervals) .setGranularity(parser.granularity).setDimFilter(parser.filter) .setDimensions(new ArrayList<DimensionSpec>(parser.groupByDimensions.values())).build(); typeRef = new TypeReference<List<Row>>() { }; groupBy = true; } String queryStr = jsonWriter.writeValueAsString(query); if (cmd.hasOption("v")) System.err.println(queryStr); URL url = new URL(String.format("http://%s/druid/v2/?pretty", hostname)); final URLConnection urlConnection = url.openConnection(); urlConnection.addRequestProperty("content-type", MediaType.APPLICATION_JSON); urlConnection.getOutputStream().write(StringUtils.toUtf8(queryStr)); BufferedReader stdInput = new BufferedReader( new InputStreamReader(urlConnection.getInputStream(), Charsets.UTF_8)); Object res = objectMapper.readValue(stdInput, typeRef); Joiner tabJoiner = Joiner.on("\t"); if (groupBy) { List<Row> rows = (List<Row>) res; Iterable<String> dimensions = Iterables.transform(parser.groupByDimensions.values(), new Function<DimensionSpec, String>() { @Override public String apply(@Nullable DimensionSpec input) { return input.getOutputName(); } }); System.out.println( tabJoiner.join(Iterables.concat(Lists.newArrayList("timestamp"), dimensions, parser.fields))); for (final Row r : rows) { System.out.println(tabJoiner.join(Iterables.concat( Lists.newArrayList(parser.granularity.toDateTime(r.getTimestampFromEpoch())), Iterables.transform(parser.groupByDimensions.values(), new Function<DimensionSpec, String>() { @Override public String apply(@Nullable DimensionSpec input) { return Joiner.on(",").join(r.getDimension(input.getOutputName())); } }), Iterables.transform(parser.fields, new Function<String, Object>() { @Override public Object apply(@Nullable String input) { return r.getFloatMetric(input); } })))); } } else { List<Result<TimeseriesResultValue>> rows = (List<Result<TimeseriesResultValue>>) res; System.out.println(tabJoiner.join(Iterables.concat(Lists.newArrayList("timestamp"), parser.fields))); for (final Result<TimeseriesResultValue> r : rows) { System.out.println(tabJoiner.join(Iterables.concat(Lists.newArrayList(r.getTimestamp()), Lists.transform(parser.fields, new Function<String, Object>() { @Override public Object apply(@Nullable String input) { return r.getValue().getMetric(input); } })))); } } CloseQuietly.close(stdInput); }
From source file:com.yahoo.pulsar.testclient.PerformanceReader.java
public static void main(String[] args) throws Exception { final Arguments arguments = new Arguments(); JCommander jc = new JCommander(arguments); jc.setProgramName("pulsar-perf-reader"); try {//from w w w .j a v a 2 s . c o m jc.parse(args); } catch (ParameterException e) { System.out.println(e.getMessage()); jc.usage(); System.exit(-1); } if (arguments.help) { jc.usage(); System.exit(-1); } if (arguments.topic.size() != 1) { System.out.println("Only one topic name is allowed"); jc.usage(); System.exit(-1); } if (arguments.confFile != null) { Properties prop = new Properties(System.getProperties()); prop.load(new FileInputStream(arguments.confFile)); if (arguments.serviceURL == null) { arguments.serviceURL = prop.getProperty("brokerServiceUrl"); } if (arguments.serviceURL == null) { arguments.serviceURL = prop.getProperty("webServiceUrl"); } // fallback to previous-version serviceUrl property to maintain backward-compatibility if (arguments.serviceURL == null) { arguments.serviceURL = prop.getProperty("serviceUrl", "http://localhost:8080/"); } if (arguments.authPluginClassName == null) { arguments.authPluginClassName = prop.getProperty("authPlugin", null); } if (arguments.authParams == null) { arguments.authParams = prop.getProperty("authParams", null); } } // Dump config variables ObjectMapper m = new ObjectMapper(); ObjectWriter w = m.writerWithDefaultPrettyPrinter(); log.info("Starting Pulsar performance reader with config: {}", w.writeValueAsString(arguments)); final DestinationName prefixTopicName = DestinationName.get(arguments.topic.get(0)); final RateLimiter limiter = arguments.rate > 0 ? RateLimiter.create(arguments.rate) : null; ReaderListener listener = (reader, msg) -> { messagesReceived.increment(); bytesReceived.add(msg.getData().length); if (limiter != null) { limiter.acquire(); } }; EventLoopGroup eventLoopGroup; if (SystemUtils.IS_OS_LINUX) { eventLoopGroup = new EpollEventLoopGroup(Runtime.getRuntime().availableProcessors() * 2, new DefaultThreadFactory("pulsar-perf-reader")); } else { eventLoopGroup = new NioEventLoopGroup(Runtime.getRuntime().availableProcessors(), new DefaultThreadFactory("pulsar-perf-reader")); } ClientConfiguration clientConf = new ClientConfiguration(); clientConf.setConnectionsPerBroker(arguments.maxConnections); clientConf.setStatsInterval(arguments.statsIntervalSeconds, TimeUnit.SECONDS); if (isNotBlank(arguments.authPluginClassName)) { clientConf.setAuthentication(arguments.authPluginClassName, arguments.authParams); } PulsarClient pulsarClient = new PulsarClientImpl(arguments.serviceURL, clientConf, eventLoopGroup); List<CompletableFuture<Reader>> futures = Lists.newArrayList(); ReaderConfiguration readerConfig = new ReaderConfiguration(); readerConfig.setReaderListener(listener); readerConfig.setReceiverQueueSize(arguments.receiverQueueSize); MessageId startMessageId; if ("earliest".equals(arguments.startMessageId)) { startMessageId = MessageId.earliest; } else if ("latest".equals(arguments.startMessageId)) { startMessageId = MessageId.latest; } else { String[] parts = arguments.startMessageId.split(":"); startMessageId = new MessageIdImpl(Long.parseLong(parts[0]), Long.parseLong(parts[1]), -1); } for (int i = 0; i < arguments.numDestinations; i++) { final DestinationName destinationName = (arguments.numDestinations == 1) ? prefixTopicName : DestinationName.get(String.format("%s-%d", prefixTopicName, i)); futures.add(pulsarClient.createReaderAsync(destinationName.toString(), startMessageId, readerConfig)); } FutureUtil.waitForAll(futures).get(); log.info("Start reading from {} topics", arguments.numDestinations); long oldTime = System.nanoTime(); while (true) { try { Thread.sleep(10000); } catch (InterruptedException e) { break; } long now = System.nanoTime(); double elapsed = (now - oldTime) / 1e9; double rate = messagesReceived.sumThenReset() / elapsed; double throughput = bytesReceived.sumThenReset() / elapsed * 8 / 1024 / 1024; log.info("Read throughput: {} msg/s -- {} Mbit/s", dec.format(rate), dec.format(throughput)); oldTime = now; } pulsarClient.close(); }
From source file:com.yahoo.pulsar.testclient.PerformanceConsumer.java
public static void main(String[] args) throws Exception { final Arguments arguments = new Arguments(); JCommander jc = new JCommander(arguments); jc.setProgramName("pulsar-perf-consumer"); try {/*from ww w . java 2 s . c o m*/ jc.parse(args); } catch (ParameterException e) { System.out.println(e.getMessage()); jc.usage(); System.exit(-1); } if (arguments.help) { jc.usage(); System.exit(-1); } if (arguments.topic.size() != 1) { System.out.println("Only one destination name is allowed"); jc.usage(); System.exit(-1); } if (arguments.confFile != null) { Properties prop = new Properties(System.getProperties()); prop.load(new FileInputStream(arguments.confFile)); if (arguments.serviceURL == null) { arguments.serviceURL = prop.getProperty("brokerServiceUrl"); } if (arguments.serviceURL == null) { arguments.serviceURL = prop.getProperty("webServiceUrl"); } // fallback to previous-version serviceUrl property to maintain backward-compatibility if (arguments.serviceURL == null) { arguments.serviceURL = prop.getProperty("serviceUrl", "http://localhost:8080/"); } if (arguments.authPluginClassName == null) { arguments.authPluginClassName = prop.getProperty("authPlugin", null); } if (arguments.authParams == null) { arguments.authParams = prop.getProperty("authParams", null); } } // Dump config variables ObjectMapper m = new ObjectMapper(); ObjectWriter w = m.writerWithDefaultPrettyPrinter(); log.info("Starting Pulsar performance consumer with config: {}", w.writeValueAsString(arguments)); final DestinationName prefixDestinationName = DestinationName.get(arguments.topic.get(0)); final RateLimiter limiter = arguments.rate > 0 ? RateLimiter.create(arguments.rate) : null; MessageListener listener = new MessageListener() { public void received(Consumer consumer, Message msg) { messagesReceived.increment(); bytesReceived.add(msg.getData().length); if (limiter != null) { limiter.acquire(); } consumer.acknowledgeAsync(msg); } }; EventLoopGroup eventLoopGroup; if (SystemUtils.IS_OS_LINUX) { eventLoopGroup = new EpollEventLoopGroup(Runtime.getRuntime().availableProcessors() * 2, new DefaultThreadFactory("pulsar-perf-consumer")); } else { eventLoopGroup = new NioEventLoopGroup(Runtime.getRuntime().availableProcessors(), new DefaultThreadFactory("pulsar-perf-consumer")); } ClientConfiguration clientConf = new ClientConfiguration(); clientConf.setConnectionsPerBroker(arguments.maxConnections); clientConf.setStatsInterval(arguments.statsIntervalSeconds, TimeUnit.SECONDS); if (isNotBlank(arguments.authPluginClassName)) { clientConf.setAuthentication(arguments.authPluginClassName, arguments.authParams); } PulsarClient pulsarClient = new PulsarClientImpl(arguments.serviceURL, clientConf, eventLoopGroup); List<Future<Consumer>> futures = Lists.newArrayList(); ConsumerConfiguration consumerConfig = new ConsumerConfiguration(); consumerConfig.setMessageListener(listener); consumerConfig.setReceiverQueueSize(arguments.receiverQueueSize); for (int i = 0; i < arguments.numDestinations; i++) { final DestinationName destinationName = (arguments.numDestinations == 1) ? prefixDestinationName : DestinationName.get(String.format("%s-%d", prefixDestinationName, i)); log.info("Adding {} consumers on destination {}", arguments.numConsumers, destinationName); for (int j = 0; j < arguments.numConsumers; j++) { String subscriberName; if (arguments.numConsumers > 1) { subscriberName = String.format("%s-%d", arguments.subscriberName, j); } else { subscriberName = arguments.subscriberName; } futures.add( pulsarClient.subscribeAsync(destinationName.toString(), subscriberName, consumerConfig)); } } for (Future<Consumer> future : futures) { future.get(); } log.info("Start receiving from {} consumers on {} destinations", arguments.numConsumers, arguments.numDestinations); long oldTime = System.nanoTime(); while (true) { try { Thread.sleep(10000); } catch (InterruptedException e) { break; } long now = System.nanoTime(); double elapsed = (now - oldTime) / 1e9; double rate = messagesReceived.sumThenReset() / elapsed; double throughput = bytesReceived.sumThenReset() / elapsed * 8 / 1024 / 1024; log.info("Throughput received: {} msg/s -- {} Mbit/s", dec.format(rate), dec.format(throughput)); oldTime = now; } pulsarClient.close(); }
From source file:org.apache.pulsar.testclient.ManagedLedgerWriter.java
public static void main(String[] args) throws Exception { final Arguments arguments = new Arguments(); JCommander jc = new JCommander(arguments); jc.setProgramName("pulsar-perf-producer"); try {// www . j a va 2s . c o m jc.parse(args); } catch (ParameterException e) { System.out.println(e.getMessage()); jc.usage(); System.exit(-1); } if (arguments.help) { jc.usage(); System.exit(-1); } arguments.testTime = TimeUnit.SECONDS.toMillis(arguments.testTime); // Dump config variables ObjectMapper m = new ObjectMapper(); ObjectWriter w = m.writerWithDefaultPrettyPrinter(); log.info("Starting Pulsar managed-ledger perf writer with config: {}", w.writeValueAsString(arguments)); byte[] payloadData = new byte[arguments.msgSize]; ByteBuf payloadBuffer = PooledByteBufAllocator.DEFAULT.directBuffer(arguments.msgSize); payloadBuffer.writerIndex(arguments.msgSize); // Now processing command line arguments String managedLedgerPrefix = "test-" + DigestUtils.sha1Hex(UUID.randomUUID().toString()).substring(0, 5); ClientConfiguration bkConf = new ClientConfiguration(); bkConf.setUseV2WireProtocol(true); bkConf.setAddEntryTimeout(30); bkConf.setReadEntryTimeout(30); bkConf.setThrottleValue(0); bkConf.setNumChannelsPerBookie(arguments.maxConnections); bkConf.setZkServers(arguments.zookeeperServers); ManagedLedgerFactoryConfig mlFactoryConf = new ManagedLedgerFactoryConfig(); mlFactoryConf.setMaxCacheSize(0); ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkConf, mlFactoryConf); ManagedLedgerConfig mlConf = new ManagedLedgerConfig(); mlConf.setEnsembleSize(arguments.ensembleSize); mlConf.setWriteQuorumSize(arguments.writeQuorum); mlConf.setAckQuorumSize(arguments.ackQuorum); mlConf.setMinimumRolloverTime(10, TimeUnit.MINUTES); mlConf.setMetadataEnsembleSize(arguments.ensembleSize); mlConf.setMetadataWriteQuorumSize(arguments.writeQuorum); mlConf.setMetadataAckQuorumSize(arguments.ackQuorum); mlConf.setDigestType(arguments.digestType); mlConf.setMaxSizePerLedgerMb(2048); List<CompletableFuture<ManagedLedger>> futures = new ArrayList<>(); for (int i = 0; i < arguments.numManagedLedgers; i++) { String name = String.format("%s-%03d", managedLedgerPrefix, i); CompletableFuture<ManagedLedger> future = new CompletableFuture<>(); futures.add(future); factory.asyncOpen(name, mlConf, new OpenLedgerCallback() { @Override public void openLedgerComplete(ManagedLedger ledger, Object ctx) { future.complete(ledger); } @Override public void openLedgerFailed(ManagedLedgerException exception, Object ctx) { future.completeExceptionally(exception); } }, null); } List<ManagedLedger> managedLedgers = futures.stream().map(CompletableFuture::join) .collect(Collectors.toList()); log.info("Created {} managed ledgers", managedLedgers.size()); Runtime.getRuntime().addShutdownHook(new Thread() { public void run() { printAggregatedStats(); } }); Collections.shuffle(managedLedgers); AtomicBoolean isDone = new AtomicBoolean(); List<List<ManagedLedger>> managedLedgersPerThread = Lists.partition(managedLedgers, Math.max(1, managedLedgers.size() / arguments.numThreads)); for (int i = 0; i < arguments.numThreads; i++) { List<ManagedLedger> managedLedgersForThisThread = managedLedgersPerThread.get(i); int nunManagedLedgersForThisThread = managedLedgersForThisThread.size(); long numMessagesForThisThread = arguments.numMessages / arguments.numThreads; int maxOutstandingForThisThread = arguments.maxOutstanding; executor.submit(() -> { try { final double msgRate = arguments.msgRate / (double) arguments.numThreads; final RateLimiter rateLimiter = RateLimiter.create(msgRate); // Acquire 1 sec worth of messages to have a slower ramp-up rateLimiter.acquire((int) msgRate); final long startTime = System.currentTimeMillis(); final Semaphore semaphore = new Semaphore(maxOutstandingForThisThread); final AddEntryCallback addEntryCallback = new AddEntryCallback() { @Override public void addComplete(Position position, Object ctx) { long sendTime = (Long) (ctx); messagesSent.increment(); bytesSent.add(payloadData.length); long latencyMicros = NANOSECONDS.toMicros(System.nanoTime() - sendTime); recorder.recordValue(latencyMicros); cumulativeRecorder.recordValue(latencyMicros); semaphore.release(); } @Override public void addFailed(ManagedLedgerException exception, Object ctx) { log.warn("Write error on message", exception); System.exit(-1); } }; // Send messages on all topics/producers long totalSent = 0; while (true) { for (int j = 0; j < nunManagedLedgersForThisThread; j++) { if (arguments.testTime > 0) { if (System.currentTimeMillis() - startTime > arguments.testTime) { log.info("------------------- DONE -----------------------"); printAggregatedStats(); isDone.set(true); Thread.sleep(5000); System.exit(0); } } if (numMessagesForThisThread > 0) { if (totalSent++ >= numMessagesForThisThread) { log.info("------------------- DONE -----------------------"); printAggregatedStats(); isDone.set(true); Thread.sleep(5000); System.exit(0); } } semaphore.acquire(); rateLimiter.acquire(); final long sendTime = System.nanoTime(); managedLedgersForThisThread.get(j).asyncAddEntry(payloadBuffer, addEntryCallback, sendTime); } } } catch (Throwable t) { log.error("Got error", t); } }); } // Print report stats long oldTime = System.nanoTime(); Histogram reportHistogram = null; while (true) { try { Thread.sleep(10000); } catch (InterruptedException e) { break; } if (isDone.get()) { break; } long now = System.nanoTime(); double elapsed = (now - oldTime) / 1e9; double rate = messagesSent.sumThenReset() / elapsed; double throughput = bytesSent.sumThenReset() / elapsed / 1024 / 1024 * 8; reportHistogram = recorder.getIntervalHistogram(reportHistogram); log.info( "Throughput produced: {} msg/s --- {} Mbit/s --- Latency: mean: {} ms - med: {} - 95pct: {} - 99pct: {} - 99.9pct: {} - 99.99pct: {} - Max: {}", throughputFormat.format(rate), throughputFormat.format(throughput), dec.format(reportHistogram.getMean() / 1000.0), dec.format(reportHistogram.getValueAtPercentile(50) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(95) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99.9) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99.99) / 1000.0), dec.format(reportHistogram.getMaxValue() / 1000.0)); reportHistogram.reset(); oldTime = now; } factory.shutdown(); }
From source file:com.yahoo.pulsar.testclient.PerformanceProducer.java
public static void main(String[] args) throws Exception { final Arguments arguments = new Arguments(); JCommander jc = new JCommander(arguments); jc.setProgramName("pulsar-perf-producer"); try {/*from w w w . ja v a 2s .c om*/ jc.parse(args); } catch (ParameterException e) { System.out.println(e.getMessage()); jc.usage(); System.exit(-1); } if (arguments.help) { jc.usage(); System.exit(-1); } if (arguments.destinations.size() != 1) { System.out.println("Only one topic name is allowed"); jc.usage(); System.exit(-1); } if (arguments.confFile != null) { Properties prop = new Properties(System.getProperties()); prop.load(new FileInputStream(arguments.confFile)); if (arguments.serviceURL == null) { arguments.serviceURL = prop.getProperty("brokerServiceUrl"); } if (arguments.serviceURL == null) { arguments.serviceURL = prop.getProperty("webServiceUrl"); } // fallback to previous-version serviceUrl property to maintain backward-compatibility if (arguments.serviceURL == null) { arguments.serviceURL = prop.getProperty("serviceUrl", "http://localhost:8080/"); } if (arguments.authPluginClassName == null) { arguments.authPluginClassName = prop.getProperty("authPlugin", null); } if (arguments.authParams == null) { arguments.authParams = prop.getProperty("authParams", null); } } arguments.testTime = TimeUnit.SECONDS.toMillis(arguments.testTime); // Dump config variables ObjectMapper m = new ObjectMapper(); ObjectWriter w = m.writerWithDefaultPrettyPrinter(); log.info("Starting Pulsar perf producer with config: {}", w.writeValueAsString(arguments)); // Read payload data from file if needed byte payloadData[]; if (arguments.payloadFilename != null) { payloadData = Files.readAllBytes(Paths.get(arguments.payloadFilename)); } else { payloadData = new byte[arguments.msgSize]; } // Now processing command line arguments String prefixTopicName = arguments.destinations.get(0); List<Future<Producer>> futures = Lists.newArrayList(); EventLoopGroup eventLoopGroup; if (SystemUtils.IS_OS_LINUX) { eventLoopGroup = new EpollEventLoopGroup(Runtime.getRuntime().availableProcessors(), new DefaultThreadFactory("pulsar-perf-producer")); } else { eventLoopGroup = new NioEventLoopGroup(Runtime.getRuntime().availableProcessors(), new DefaultThreadFactory("pulsar-perf-producer")); } ClientConfiguration clientConf = new ClientConfiguration(); clientConf.setConnectionsPerBroker(arguments.maxConnections); clientConf.setStatsInterval(arguments.statsIntervalSeconds, TimeUnit.SECONDS); if (isNotBlank(arguments.authPluginClassName)) { clientConf.setAuthentication(arguments.authPluginClassName, arguments.authParams); } PulsarClient client = new PulsarClientImpl(arguments.serviceURL, clientConf, eventLoopGroup); ProducerConfiguration producerConf = new ProducerConfiguration(); producerConf.setSendTimeout(0, TimeUnit.SECONDS); producerConf.setCompressionType(arguments.compression); // enable round robin message routing if it is a partitioned topic producerConf.setMessageRoutingMode(MessageRoutingMode.RoundRobinPartition); if (arguments.batchTime > 0) { producerConf.setBatchingMaxPublishDelay(arguments.batchTime, TimeUnit.MILLISECONDS); producerConf.setBatchingEnabled(true); producerConf.setMaxPendingMessages(arguments.msgRate); } for (int i = 0; i < arguments.numTopics; i++) { String topic = (arguments.numTopics == 1) ? prefixTopicName : String.format("%s-%d", prefixTopicName, i); log.info("Adding {} publishers on destination {}", arguments.numProducers, topic); for (int j = 0; j < arguments.numProducers; j++) { futures.add(client.createProducerAsync(topic, producerConf)); } } final List<Producer> producers = Lists.newArrayListWithCapacity(futures.size()); for (Future<Producer> future : futures) { producers.add(future.get()); } log.info("Created {} producers", producers.size()); Runtime.getRuntime().addShutdownHook(new Thread() { public void run() { printAggregatedStats(); } }); Collections.shuffle(producers); AtomicBoolean isDone = new AtomicBoolean(); executor.submit(() -> { try { RateLimiter rateLimiter = RateLimiter.create(arguments.msgRate); long startTime = System.currentTimeMillis(); // Send messages on all topics/producers long totalSent = 0; while (true) { for (Producer producer : producers) { if (arguments.testTime > 0) { if (System.currentTimeMillis() - startTime > arguments.testTime) { log.info("------------------- DONE -----------------------"); printAggregatedStats(); isDone.set(true); Thread.sleep(5000); System.exit(0); } } if (arguments.numMessages > 0) { if (totalSent++ >= arguments.numMessages) { log.info("------------------- DONE -----------------------"); printAggregatedStats(); isDone.set(true); Thread.sleep(5000); System.exit(0); } } rateLimiter.acquire(); final long sendTime = System.nanoTime(); producer.sendAsync(payloadData).thenRun(() -> { messagesSent.increment(); bytesSent.add(payloadData.length); long latencyMicros = NANOSECONDS.toMicros(System.nanoTime() - sendTime); recorder.recordValue(latencyMicros); cumulativeRecorder.recordValue(latencyMicros); }).exceptionally(ex -> { log.warn("Write error on message", ex); System.exit(-1); return null; }); } } } catch (Throwable t) { log.error("Got error", t); } }); // Print report stats long oldTime = System.nanoTime(); Histogram reportHistogram = null; String statsFileName = "perf-producer-" + System.currentTimeMillis() + ".hgrm"; log.info("Dumping latency stats to {}", statsFileName); PrintStream histogramLog = new PrintStream(new FileOutputStream(statsFileName), false); HistogramLogWriter histogramLogWriter = new HistogramLogWriter(histogramLog); // Some log header bits histogramLogWriter.outputLogFormatVersion(); histogramLogWriter.outputLegend(); while (true) { try { Thread.sleep(10000); } catch (InterruptedException e) { break; } if (isDone.get()) { break; } long now = System.nanoTime(); double elapsed = (now - oldTime) / 1e9; double rate = messagesSent.sumThenReset() / elapsed; double throughput = bytesSent.sumThenReset() / elapsed / 1024 / 1024 * 8; reportHistogram = recorder.getIntervalHistogram(reportHistogram); log.info( "Throughput produced: {} msg/s --- {} Mbit/s --- Latency: mean: {} ms - med: {} - 95pct: {} - 99pct: {} - 99.9pct: {} - 99.99pct: {} - Max: {}", throughputFormat.format(rate), throughputFormat.format(throughput), dec.format(reportHistogram.getMean() / 1000.0), dec.format(reportHistogram.getValueAtPercentile(50) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(95) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99.9) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99.99) / 1000.0), dec.format(reportHistogram.getMaxValue() / 1000.0)); histogramLogWriter.outputIntervalHistogram(reportHistogram); reportHistogram.reset(); oldTime = now; } client.close(); }