List of usage examples for java.lang Runtime getRuntime
public static Runtime getRuntime()
From source file:com.boonya.http.async.examples.nio.client.AsyncClientConfiguration.java
public final static void main(String[] args) throws Exception { // Use custom message parser / writer to customize the way HTTP // messages are parsed from and written out to the data stream. NHttpMessageParserFactory<HttpResponse> responseParserFactory = new DefaultHttpResponseParserFactory() { @Override//ww w .ja va 2s .c om public NHttpMessageParser<HttpResponse> create(final SessionInputBuffer buffer, final MessageConstraints constraints) { LineParser lineParser = new BasicLineParser() { @Override public Header parseHeader(final CharArrayBuffer buffer) { try { return super.parseHeader(buffer); } catch (ParseException ex) { return new BasicHeader(buffer.toString(), null); } } }; return new DefaultHttpResponseParser(buffer, lineParser, DefaultHttpResponseFactory.INSTANCE, constraints); } }; NHttpMessageWriterFactory<HttpRequest> requestWriterFactory = new DefaultHttpRequestWriterFactory(); // Use a custom connection factory to customize the process of // initialization of outgoing HTTP connections. Beside standard connection // configuration parameters HTTP connection factory can define message // parser / writer routines to be employed by individual connections. NHttpConnectionFactory<ManagedNHttpClientConnection> connFactory = new ManagedNHttpClientConnectionFactory( requestWriterFactory, responseParserFactory, HeapByteBufferAllocator.INSTANCE); // Client HTTP connection objects when fully initialized can be bound to // an arbitrary network socket. The process of network socket initialization, // its connection to a remote address and binding to a local one is controlled // by a connection socket factory. // SSL context for secure connections can be created either based on // system or application specific properties. SSLContext sslcontext = SSLContexts.createSystemDefault(); // Use custom hostname verifier to customize SSL hostname verification. HostnameVerifier hostnameVerifier = new DefaultHostnameVerifier(); // Create a registry of custom connection session strategies for supported // protocol schemes. Registry<SchemeIOSessionStrategy> sessionStrategyRegistry = RegistryBuilder .<SchemeIOSessionStrategy>create().register("http", NoopIOSessionStrategy.INSTANCE) .register("https", new SSLIOSessionStrategy(sslcontext, hostnameVerifier)).build(); // Use custom DNS resolver to override the system DNS resolution. DnsResolver dnsResolver = new SystemDefaultDnsResolver() { @Override public InetAddress[] resolve(final String host) throws UnknownHostException { if (host.equalsIgnoreCase("myhost")) { return new InetAddress[] { InetAddress.getByAddress(new byte[] { 127, 0, 0, 1 }) }; } else { return super.resolve(host); } } }; // Create I/O reactor configuration IOReactorConfig ioReactorConfig = IOReactorConfig.custom() .setIoThreadCount(Runtime.getRuntime().availableProcessors()).setConnectTimeout(30000) .setSoTimeout(30000).build(); // Create a custom I/O reactort ConnectingIOReactor ioReactor = new DefaultConnectingIOReactor(ioReactorConfig); // Create a connection manager with custom configuration. PoolingNHttpClientConnectionManager connManager = new PoolingNHttpClientConnectionManager(ioReactor, connFactory, sessionStrategyRegistry, dnsResolver); // Create message constraints MessageConstraints messageConstraints = MessageConstraints.custom().setMaxHeaderCount(200) .setMaxLineLength(2000).build(); // Create connection configuration ConnectionConfig connectionConfig = ConnectionConfig.custom() .setMalformedInputAction(CodingErrorAction.IGNORE) .setUnmappableInputAction(CodingErrorAction.IGNORE).setCharset(Consts.UTF_8) .setMessageConstraints(messageConstraints).build(); // Configure the connection manager to use connection configuration either // by default or for a specific host. connManager.setDefaultConnectionConfig(connectionConfig); connManager.setConnectionConfig(new HttpHost("somehost", 80), ConnectionConfig.DEFAULT); // Configure total max or per route limits for persistent connections // that can be kept in the pool or leased by the connection manager. connManager.setMaxTotal(100); connManager.setDefaultMaxPerRoute(10); connManager.setMaxPerRoute(new HttpRoute(new HttpHost("somehost", 80)), 20); // Use custom cookie store if necessary. CookieStore cookieStore = new BasicCookieStore(); // Use custom credentials provider if necessary. CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); // Create global request configuration RequestConfig defaultRequestConfig = RequestConfig.custom().setCookieSpec(CookieSpecs.DEFAULT) .setExpectContinueEnabled(true) .setTargetPreferredAuthSchemes(Arrays.asList(AuthSchemes.NTLM, AuthSchemes.DIGEST)) .setProxyPreferredAuthSchemes(Arrays.asList(AuthSchemes.BASIC)).build(); // Create an HttpClient with the given custom dependencies and configuration. CloseableHttpAsyncClient httpclient = HttpAsyncClients.custom().setConnectionManager(connManager) .setDefaultCookieStore(cookieStore).setDefaultCredentialsProvider(credentialsProvider) .setProxy(new HttpHost("myproxy", 8080)).setDefaultRequestConfig(defaultRequestConfig).build(); try { HttpGet httpget = new HttpGet("http://localhost/"); // Request configuration can be overridden at the request level. // They will take precedence over the one set at the client level. RequestConfig requestConfig = RequestConfig.copy(defaultRequestConfig).setSocketTimeout(5000) .setConnectTimeout(5000).setConnectionRequestTimeout(5000) .setProxy(new HttpHost("myotherproxy", 8080)).build(); httpget.setConfig(requestConfig); // Execution context can be customized locally. HttpClientContext localContext = HttpClientContext.create(); // Contextual attributes set the local context level will take // precedence over those set at the client level. localContext.setCookieStore(cookieStore); localContext.setCredentialsProvider(credentialsProvider); System.out.println("Executing request " + httpget.getRequestLine()); httpclient.start(); // Pass local context as a parameter Future<HttpResponse> future = httpclient.execute(httpget, localContext, null); // Please note that it may be unsafe to access HttpContext instance // while the request is still being executed HttpResponse response = future.get(); System.out.println("Response: " + response.getStatusLine()); // Once the request has been executed the local context can // be used to examine updated state and various objects affected // by the request execution. // Last executed request localContext.getRequest(); // Execution route localContext.getHttpRoute(); // Target auth state localContext.getTargetAuthState(); // Proxy auth state localContext.getTargetAuthState(); // Cookie origin localContext.getCookieOrigin(); // Cookie spec used localContext.getCookieSpec(); // User security token localContext.getUserToken(); } finally { httpclient.close(); } }
From source file:afred.javademo.httpclient.official.AsyncClientConfiguration.java
public final static void main(String[] args) throws Exception { // Use custom message parser / writer to customize the way HTTP // messages are parsed from and written out to the data stream. NHttpMessageParserFactory<HttpResponse> responseParserFactory = new DefaultHttpResponseParserFactory() { @Override/*from w w w . java 2 s.c om*/ public NHttpMessageParser<HttpResponse> create(final SessionInputBuffer buffer, final MessageConstraints constraints) { LineParser lineParser = new BasicLineParser() { @Override public Header parseHeader(final CharArrayBuffer buffer) { try { return super.parseHeader(buffer); } catch (ParseException ex) { return new BasicHeader(buffer.toString(), null); } } }; return new DefaultHttpResponseParser(buffer, lineParser, DefaultHttpResponseFactory.INSTANCE, constraints); } }; NHttpMessageWriterFactory<HttpRequest> requestWriterFactory = new DefaultHttpRequestWriterFactory(); // Use a custom connection factory to customize the process of // initialization of outgoing HTTP connections. Beside standard connection // configuration parameters HTTP connection factory can define message // parser / writer routines to be employed by individual connections. NHttpConnectionFactory<ManagedNHttpClientConnection> connFactory = new ManagedNHttpClientConnectionFactory( requestWriterFactory, responseParserFactory, HeapByteBufferAllocator.INSTANCE); // Client HTTP connection objects when fully initialized can be bound to // an arbitrary network socket. The process of network socket initialization, // its connection to a remote address and binding to a local one is controlled // by a connection socket factory. // SSL context for secure connections can be created either based on // system or application specific properties. SSLContext sslcontext = SSLContexts.createSystemDefault(); // Use custom hostname verifier to customize SSL hostname verification. X509HostnameVerifier hostnameVerifier = new BrowserCompatHostnameVerifier(); // Create a registry of custom connection session strategies for supported // protocol schemes. Registry<SchemeIOSessionStrategy> sessionStrategyRegistry = RegistryBuilder .<SchemeIOSessionStrategy>create().register("http", NoopIOSessionStrategy.INSTANCE) .register("https", new SSLIOSessionStrategy(sslcontext, hostnameVerifier)).build(); // Use custom DNS resolver to override the system DNS resolution. DnsResolver dnsResolver = new SystemDefaultDnsResolver() { @Override public InetAddress[] resolve(final String host) throws UnknownHostException { if (host.equalsIgnoreCase("myhost")) { return new InetAddress[] { InetAddress.getByAddress(new byte[] { 127, 0, 0, 1 }) }; } else { return super.resolve(host); } } }; // Create I/O reactor configuration IOReactorConfig ioReactorConfig = IOReactorConfig.custom() .setIoThreadCount(Runtime.getRuntime().availableProcessors()).setConnectTimeout(30000) .setSoTimeout(30000).build(); // Create a custom I/O reactort ConnectingIOReactor ioReactor = new DefaultConnectingIOReactor(ioReactorConfig); // Create a connection manager with custom configuration. PoolingNHttpClientConnectionManager connManager = new PoolingNHttpClientConnectionManager(ioReactor, connFactory, sessionStrategyRegistry, dnsResolver); // Create message constraints MessageConstraints messageConstraints = MessageConstraints.custom().setMaxHeaderCount(200) .setMaxLineLength(2000).build(); // Create connection configuration ConnectionConfig connectionConfig = ConnectionConfig.custom() .setMalformedInputAction(CodingErrorAction.IGNORE) .setUnmappableInputAction(CodingErrorAction.IGNORE).setCharset(Consts.UTF_8) .setMessageConstraints(messageConstraints).build(); // Configure the connection manager to use connection configuration either // by default or for a specific host. connManager.setDefaultConnectionConfig(connectionConfig); connManager.setConnectionConfig(new HttpHost("somehost", 80), ConnectionConfig.DEFAULT); // Configure total max or per route limits for persistent connections // that can be kept in the pool or leased by the connection manager. connManager.setMaxTotal(100); connManager.setDefaultMaxPerRoute(10); connManager.setMaxPerRoute(new HttpRoute(new HttpHost("somehost", 80)), 20); // Use custom cookie store if necessary. CookieStore cookieStore = new BasicCookieStore(); // Use custom credentials provider if necessary. CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); // Create global request configuration RequestConfig defaultRequestConfig = RequestConfig.custom().setCookieSpec(CookieSpecs.BEST_MATCH) .setExpectContinueEnabled(true).setStaleConnectionCheckEnabled(true) .setTargetPreferredAuthSchemes(Arrays.asList(AuthSchemes.NTLM, AuthSchemes.DIGEST)) .setProxyPreferredAuthSchemes(Arrays.asList(AuthSchemes.BASIC)).build(); // Create an HttpClient with the given custom dependencies and configuration. CloseableHttpAsyncClient httpclient = HttpAsyncClients.custom().setConnectionManager(connManager) .setDefaultCookieStore(cookieStore).setDefaultCredentialsProvider(credentialsProvider) .setProxy(new HttpHost("myproxy", 8080)).setDefaultRequestConfig(defaultRequestConfig).build(); try { HttpGet httpget = new HttpGet("http://localhost/"); // Request configuration can be overridden at the request level. // They will take precedence over the one set at the client level. RequestConfig requestConfig = RequestConfig.copy(defaultRequestConfig).setSocketTimeout(5000) .setConnectTimeout(5000).setConnectionRequestTimeout(5000) .setProxy(new HttpHost("myotherproxy", 8080)).build(); httpget.setConfig(requestConfig); // Execution context can be customized locally. HttpClientContext localContext = HttpClientContext.create(); // Contextual attributes set the local context level will take // precedence over those set at the client level. localContext.setCookieStore(cookieStore); localContext.setCredentialsProvider(credentialsProvider); System.out.println("Executing request " + httpget.getRequestLine()); httpclient.start(); // Pass local context as a parameter Future<HttpResponse> future = httpclient.execute(httpget, localContext, null); // Please note that it may be unsafe to access HttpContext instance // while the request is still being executed HttpResponse response = future.get(); System.out.println("Response: " + response.getStatusLine()); // Once the request has been executed the local context can // be used to examine updated state and various objects affected // by the request execution. // Last executed request localContext.getRequest(); // Execution route localContext.getHttpRoute(); // Target auth state localContext.getTargetAuthState(); // Proxy auth state localContext.getTargetAuthState(); // Cookie origin localContext.getCookieOrigin(); // Cookie spec used localContext.getCookieSpec(); // User security token localContext.getUserToken(); } finally { httpclient.close(); } }
From source file:MiniCluster.java
/** * Runs the {@link MiniAccumuloCluster} given a -p argument with a property file. Establishes a shutdown port for asynchronous operation. * /* ww w. ja va2s . c o m*/ * @param args * An optional -p argument can be specified with the path to a valid properties file. */ public static void main(String[] args) throws Exception { Opts opts = new Opts(); opts.parseArgs(MiniCluster.class.getName(), args); if (opts.printProps) { printProperties(); System.exit(0); } int shutdownPort = 4445; final File miniDir; if (opts.prop.containsKey(DIRECTORY_PROP)) miniDir = new File(opts.prop.getProperty(DIRECTORY_PROP)); else miniDir = Files.createTempDir(); String rootPass = opts.prop.containsKey(ROOT_PASSWORD_PROP) ? opts.prop.getProperty(ROOT_PASSWORD_PROP) : "secret"; String instanceName = opts.prop.containsKey(INSTANCE_NAME_PROP) ? opts.prop.getProperty(INSTANCE_NAME_PROP) : "accumulo"; MiniAccumuloConfig config = new MiniAccumuloConfig(miniDir, instanceName, rootPass); if (opts.prop.containsKey(NUM_T_SERVERS_PROP)) config.setNumTservers(Integer.parseInt(opts.prop.getProperty(NUM_T_SERVERS_PROP))); if (opts.prop.containsKey(ZOO_KEEPER_PORT_PROP)) config.setZooKeeperPort(Integer.parseInt(opts.prop.getProperty(ZOO_KEEPER_PORT_PROP))); // if (opts.prop.containsKey(JDWP_ENABLED_PROP)) // config.setJDWPEnabled(Boolean.parseBoolean(opts.prop.getProperty(JDWP_ENABLED_PROP))); // if (opts.prop.containsKey(ZOO_KEEPER_MEMORY_PROP)) // setMemoryOnConfig(config, opts.prop.getProperty(ZOO_KEEPER_MEMORY_PROP), ServerType.ZOOKEEPER); // if (opts.prop.containsKey(TSERVER_MEMORY_PROP)) // setMemoryOnConfig(config, opts.prop.getProperty(TSERVER_MEMORY_PROP), ServerType.TABLET_SERVER); // if (opts.prop.containsKey(MASTER_MEMORY_PROP)) // setMemoryOnConfig(config, opts.prop.getProperty(MASTER_MEMORY_PROP), ServerType.MASTER); // if (opts.prop.containsKey(DEFAULT_MEMORY_PROP)) // setMemoryOnConfig(config, opts.prop.getProperty(DEFAULT_MEMORY_PROP)); // if (opts.prop.containsKey(SHUTDOWN_PORT_PROP)) // shutdownPort = Integer.parseInt(opts.prop.getProperty(SHUTDOWN_PORT_PROP)); Map<String, String> siteConfig = new HashMap<String, String>(); for (Map.Entry<Object, Object> entry : opts.prop.entrySet()) { String key = (String) entry.getKey(); if (key.startsWith("site.")) siteConfig.put(key.replaceFirst("site.", ""), (String) entry.getValue()); } config.setSiteConfig(siteConfig); final MiniAccumuloCluster accumulo = new MiniAccumuloCluster(config); Runtime.getRuntime().addShutdownHook(new Thread() { @Override public void run() { try { accumulo.stop(); FileUtils.deleteDirectory(miniDir); System.out.println("\nShut down gracefully on " + new Date()); } catch (IOException e) { e.printStackTrace(); } catch (InterruptedException e) { e.printStackTrace(); } } }); accumulo.start(); printInfo(accumulo, shutdownPort); // start a socket on the shutdown port and block- anything connected to this port will activate the shutdown ServerSocket shutdownServer = new ServerSocket(shutdownPort); shutdownServer.accept(); System.exit(0); }
From source file:com.l2jserver.model.template.NPCTemplateConverter.java
public static void main(String[] args) throws SQLException, IOException, ClassNotFoundException, JAXBException { controllers.put("L2Teleporter", TeleporterController.class); controllers.put("L2CastleTeleporter", TeleporterController.class); controllers.put("L2Npc", BaseNPCController.class); controllers.put("L2Monster", MonsterController.class); controllers.put("L2FlyMonster", MonsterController.class); Class.forName("com.mysql.jdbc.Driver"); final File target = new File("generated/template/npc"); System.out.println("Scaning legacy HTML files..."); htmlScannedFiles = FileUtils.listFiles(L2J_HTML_FOLDER, new String[] { "html", "htm" }, true); final JAXBContext c = JAXBContext.newInstance(NPCTemplate.class, Teleports.class); final Connection conn = DriverManager.getConnection(JDBC_URL, JDBC_USERNAME, JDBC_PASSWORD); {//from www . ja v a 2 s. com System.out.println("Converting teleport templates..."); teleportation.teleport = CollectionFactory.newList(); final Marshaller m = c.createMarshaller(); m.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true); final PreparedStatement st = conn.prepareStatement("SELECT * FROM teleport"); st.execute(); final ResultSet rs = st.getResultSet(); while (rs.next()) { final TeleportationTemplate template = new TeleportationTemplate(); template.id = new TeleportationTemplateID(rs.getInt("id"), null); template.name = rs.getString("Description"); TemplateCoordinate coord = new TemplateCoordinate(); coord.x = rs.getInt("loc_x"); coord.y = rs.getInt("loc_y"); coord.z = rs.getInt("loc_z"); template.point = coord; template.price = rs.getInt("price"); template.item = rs.getInt("itemId"); if (rs.getBoolean("fornoble")) { template.restrictions = new Restrictions(); template.restrictions.restriction = Arrays.asList("NOBLE"); } teleportation.teleport.add(template); } m.marshal(teleportation, getXMLSerializer(new FileOutputStream(new File(target, "../teleports.xml")))); // System.exit(0); } System.out.println("Generating template XML files..."); // c.generateSchema(new SchemaOutputResolver() { // @Override // public Result createOutput(String namespaceUri, // String suggestedFileName) throws IOException { // // System.out.println(new File(target, suggestedFileName)); // // return null; // return new StreamResult(new File(target, suggestedFileName)); // } // }); try { final Marshaller m = c.createMarshaller(); m.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true); final PreparedStatement st = conn.prepareStatement( "SELECT npc.*, npcskills.level AS race " + "FROM npc " + "LEFT JOIN npcskills " + "ON(npc.idTemplate = npcskills.npcid AND npcskills.skillid = ?)"); st.setInt(1, 4416); st.execute(); final ResultSet rs = st.getResultSet(); while (rs.next()) { Object[] result = fillNPC(rs); NPCTemplate t = (NPCTemplate) result[0]; String type = (String) result[1]; String folder = createFolder(type); if (folder.isEmpty()) { m.setProperty(Marshaller.JAXB_SCHEMA_LOCATION, "npc ../npc.xsd"); } else { m.setProperty(Marshaller.JAXB_SCHEMA_LOCATION, "npc ../../npc.xsd"); } final File file = new File(target, "npc/" + folder + "/" + t.getID().getID() + (t.getInfo().getName() != null ? "-" + camelCase(t.getInfo().getName().getValue()) : "") + ".xml"); file.getParentFile().mkdirs(); templates.add(t); try { m.marshal(t, getXMLSerializer(new FileOutputStream(file))); } catch (MarshalException e) { System.err.println("Could not generate XML template file for " + t.getInfo().getName().getValue() + " - " + t.getID()); file.delete(); } } System.out.println("Generated " + templates.size() + " templates"); System.gc(); System.out.println("Free: " + FileUtils.byteCountToDisplaySize(Runtime.getRuntime().freeMemory())); System.out.println("Total: " + FileUtils.byteCountToDisplaySize(Runtime.getRuntime().totalMemory())); System.out.println("Used: " + FileUtils.byteCountToDisplaySize( Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory())); System.out.println("Max: " + FileUtils.byteCountToDisplaySize(Runtime.getRuntime().maxMemory())); } finally { conn.close(); } }
From source file:com.hygenics.parser.MainApp.java
/** * The entire programs main method.// w w w. j a v a2 s.c o m * * @param args */ public static void main(String[] args) { final Logger log = LoggerFactory.getLogger(MainApp.class); log.info("Starting Parse @ " + Calendar.getInstance().getTime().toString()); ClassPathXmlApplicationContext context = new ClassPathXmlApplicationContext( ("file:" + System.getProperty("beansfile").trim())); log.info("Found beans @ " + System.getProperty("beansfile").trim()); log.info("Starting"); ArrayList<String> results = null; String activity = null; log.info("Obtaining Activities"); ActivitiesStack stack = (ActivitiesStack) context.getBean("ActivitiesStack"); // integers keeping track of bean number to pull (e.g. DumpToText0 or // DumpToText1) // in keeping with the spirit of dumping out data int n = 0, srn = 0, mv = 0, cen = 0, pps = 0, sb = 0, kv = 0, cf = 0, zip = 0, bm = 0, dump = 0, kettle = 0, execute = 0, transfer = 0, sqls = 0, job = 0, parsepages = 0, getpages = 0, map = 0, js = 0, sdump = 0, transforms = 0, execs = 0, gim = 0, ems = 0, jd = 0; Pattern p = Pattern.compile("[A-Za-z]+[0-9]+"); Matcher m; // start stack init log.info("Stack Initialized with Size of " + stack.getSize() + " @ " + Calendar.getInstance().getTime().toString()); while (stack.getSize() > 0) { // pop activity form stack activity = stack.Pop(); log.info("Activities Remaining " + stack.getSize()); m = p.matcher(activity); log.info("\n\nACTIVITY: " + activity + "\n\n"); if (activity.toLowerCase().contains("manualrepconn")) { log.info("Manual Transformation Started @ " + Calendar.getInstance().getTime().toString()); ManualReplacement mrp = (ManualReplacement) context.getBean("ManualRepConn"); mrp.run(); log.info("Manual Transformation Finished @ " + Calendar.getInstance().getTime().toString()); } else if (activity.toLowerCase().contains("cleanfolder")) { log.info("Folder Cleanup Started @ " + Calendar.getInstance().getTime().toString()); CleanFolder c; if (cf == 0 || context.containsBean("CleanFolder")) { c = (CleanFolder) ((context.containsBean("CleanFolder")) ? context.getBean("CleanFolder") : context.getBean("CleanFolder" + cf)); } else { c = (CleanFolder) context.getBean("CleanFolder" + cf); } c.run(); cf++; log.info("File Cleanup Complete @ " + Calendar.getInstance().getTime().toString()); } else if (activity.toLowerCase().contains("zip")) { log.info("Zip File Creation Started @ " + Calendar.getInstance().getTime().toString()); Archiver a; if (zip == 0 || context.containsBean("Zip")) { a = (Archiver) ((context.containsBean("Zip")) ? context.getBean("Zip") : context.getBean("Zip" + zip)); } else { a = (Archiver) context.getBean("Zip" + zip); } a.run(); zip++; log.info("Zip File Creation Complete @ " + Calendar.getInstance().getTime().toString()); } else if (activity.toLowerCase().contains("transfer")) { log.info("File Transfer Started @ " + Calendar.getInstance().getTime().toString()); Upload u; if (transfer == 0 || context.containsBean("FileTransfer")) { u = (Upload) ((context.containsBean("FileTransfer")) ? context.getBean("FileTransfer") : context.getBean("FileTransfer" + transfer)); } else { u = (Upload) context.getBean("FileTransfer" + transfer); } u.run(); transfer++; log.info("File Transfer Complete @ " + Calendar.getInstance().getTime().toString()); } else if (activity.toLowerCase().contains("droptables")) { // drop tables log.info("Dropping Tables @ " + Calendar.getInstance().getTime().toString()); DropTables droptables = (DropTables) context.getBean("DropTables"); droptables.run(); droptables = null; log.info("Done Dropping Tables @ " + Calendar.getInstance().getTime().toString()); } else if (activity.toLowerCase().equals("createtables")) { // create tables log.info("Creating Tables @ " + Calendar.getInstance().getTime().toString()); CreateTable create = (CreateTable) context.getBean("CreateTables"); create.run(); create = null; log.info("Done Creating Tables @ " + Calendar.getInstance().getTime().toString()); } else if (activity.toLowerCase().contains("createtableswithreference")) { // create tables log.info("Creating Tables @ " + Calendar.getInstance().getTime().toString()); CreateTablesWithReference create = (CreateTablesWithReference) context .getBean("CreateTablesWithReference"); create.run(); create = null; log.info("Done Creating Tables @ " + Calendar.getInstance().getTime().toString()); } else if (activity.toLowerCase().contains("truncate")) { // truncate table log.info("Truncating @ " + Calendar.getInstance().getTime().toString()); Truncate truncate = (Truncate) context.getBean("Truncate"); truncate.truncate(); truncate = null; log.info("Truncated @ " + Calendar.getInstance().getTime().toString()); Runtime.getRuntime().gc(); log.info("Free Memory: " + Runtime.getRuntime().freeMemory()); } else if (activity.toLowerCase().equals("enforce")) { // enforce schema log.info("Enforcing Schema @" + Calendar.getInstance().getTime().toString()); ForceConformity ef = (ForceConformity) context.getBean("EnforceStandards"); ef.run(); log.info("Done Enforcing Schema @" + Calendar.getInstance().getTime().toString()); Runtime.getRuntime().gc(); log.info("Free Memory: " + Runtime.getRuntime().freeMemory()); } else if (activity.toLowerCase().contains("enforcewithreference")) { // enforce schema ForceConformityWithReference ef = (ForceConformityWithReference) context .getBean("EnforceStandardsWithReference"); log.info("Enforcing Schema By Reference @" + Calendar.getInstance().getTime().toString()); ef.run(); log.info("Done Enforcing Schema @" + Calendar.getInstance().getTime().toString()); Runtime.getRuntime().gc(); log.info("Free Memory: " + Runtime.getRuntime().freeMemory()); } else if (activity.toLowerCase().trim().equals("repconn")) { log.info("Replacing Transformation Connection Information @" + Calendar.getInstance().getTime().toString()); RepConn rep = (RepConn) context.getBean("repconn"); rep.run(); log.info("Finished Replacing Connection Information @" + Calendar.getInstance().getTime().toString()); } else if (activity.toLowerCase().contains("job")) { // run a Pentaho job as opposed to a Pentaho Transformation log.info("Run Job kjb file @" + Calendar.getInstance().getTime().toString()); RunJob kjb = null; if (m.find()) { kjb = (RunJob) context.getBean(activity); } else { kjb = (RunJob) context.getBean("Job" + job); } kjb.run(); kjb = null; log.info("Pentaho Job Complete @" + Calendar.getInstance().getTime().toString()); Runtime.getRuntime().gc(); } else if (activity.toLowerCase().compareTo("execute") == 0) { // Execute a process log.info("Executing Process @" + Calendar.getInstance().getTime().toString()); if (context.containsBean("Execute") && execs == 0) { ExecuteProcess proc = (ExecuteProcess) context.getBean(("Execute")); proc.Execute(); } else { ExecuteProcess proc = (ExecuteProcess) context.getBean(("Execute" + execute)); proc.Execute(); } execs++; log.info("Pages Obtained @" + Calendar.getInstance().getTime().toString()); Runtime.getRuntime().gc(); log.info("Free Memory: " + Runtime.getRuntime().freeMemory()); } else if (activity.toLowerCase().contains("parsepageswithscala") || activity.toLowerCase().contains("parsepagesscala")) { //parse pages with scala log.info("Parsing Pages with Scala @ " + Calendar.getInstance().getTime().toString()); ScalaParseDispatcher pds = null; if (context.containsBean("ParsePagesScala" + pps)) { pds = (ScalaParseDispatcher) context.getBean("ParsePagesScala" + pps); } else if (context.containsBean("ParsePagesScala") && pps > 0) { pds = (ScalaParseDispatcher) context.getBean("ParsePagesScala"); } pps++; pds.run(); Runtime.getRuntime().gc(); log.info("Finished Parsing Pages with Scala @ " + Calendar.getInstance().getTime().toString()); } else if (activity.toLowerCase().contains("parsepages")) { // Parse Pages with java log.info("Parsing Individual Pages @" + Calendar.getInstance().getTime().toString()); if (context.containsBean("ParsePages") && parsepages == 0) { ParseDispatcher pd = (ParseDispatcher) context.getBean("ParsePages"); pd.run(); pd = null; } else { ParseDispatcher pd = (ParseDispatcher) context.getBean("ParsePages" + parsepages); pd.run(); pd = null; } parsepages++; log.info("Finished Parsing @" + Calendar.getInstance().getTime().toString()); Runtime.getRuntime().gc(); log.info("Free Memory: " + Runtime.getRuntime().freeMemory()); } else if (activity.toLowerCase().contains("kvparser")) { // Parse Pages using the KV Parser log.info("Parsing Individual Pages with Key Value Pairs @" + Calendar.getInstance().getTime().toString()); if (context.containsBean("KVParser") && parsepages == 0) { KVParser pd = (KVParser) context.getBean("KVParser"); pd.run(); pd = null; } else { KVParser pd = (KVParser) context.getBean("KVParser" + kv); pd.run(); pd = null; } parsepages++; log.info("Finished Parsing @" + Calendar.getInstance().getTime().toString()); Runtime.getRuntime().gc(); log.info("Free Memory: " + Runtime.getRuntime().freeMemory()); } else if (activity.toLowerCase().contains("parsewithsoup")) { // Parse Pages with Jsoup log.info("Parsing Pages with JSoup @ " + Calendar.getInstance().getTime().toString()); if (context.containsBean("ParseJSoup") && js == 0) { ParseJSoup psj = (ParseJSoup) context.getBean("ParseJSoup"); psj.run(); } else { ParseJSoup psj = (ParseJSoup) context.getBean("ParseJSoup" + Integer.toString(js)); psj.run(); } js++; log.info("Finished Parsing @" + Calendar.getInstance().getTime().toString()); Runtime.getRuntime().gc(); log.info("Finished Parsing with JSoup @ " + Calendar.getInstance().getTime().toString()); } else if (activity.toLowerCase().contains("breakmultiplescala") || activity.toLowerCase().contains("breakmultiplewithscala")) { log.info("Breaking Records"); BreakMultipleScala bms = null; if (context.containsBean("BreakMultipleScala" + ems)) { bms = (BreakMultipleScala) context.getBean("BreakMultipleScala" + sb); } else { bms = (BreakMultipleScala) context.getBean("BreakMultipleScala"); } bms.run(); bms = null; sb++; Runtime.getRuntime().gc(); System.gc(); log.info("Free Memory: " + Runtime.getRuntime().freeMemory()); log.info("Completed Breaking Tasks"); } else if (activity.toLowerCase().contains("breakmultiple")) { // break apart multi-part records log.info("Breaking apart Records (BreakMultiple) @" + Calendar.getInstance().getTime().toString()); if (context.containsBean("BreakMultiple") && bm == 0) { BreakMultiple br = (BreakMultiple) context.getBean(("BreakMultiple")); br.run(); br = null; } else { BreakMultiple br = (BreakMultiple) context.getBean(("BreakMultiple" + Integer.toString(bm))); br.run(); br = null; } bm++; log.info("Finished Breaking Apart Records @" + Calendar.getInstance().getTime().toString()); log.info("Free Memory: " + Runtime.getRuntime().freeMemory()); } else if (activity.toLowerCase().compareTo("mapper") == 0) { // remap data log.info("Mapping Records @" + Calendar.getInstance().getTime().toString()); if (context.containsBean("Mapper") && map == 0) { Mapper mp = (Mapper) context.getBean("Mapper"); mp.run(); mp = null; } else { Mapper mp = (Mapper) context.getBean("Mapper" + Integer.toString(map)); mp.run(); mp = null; } map++; log.info("Completed Mapping Records @ " + Calendar.getInstance().getTime().toString()); } else if (activity.toLowerCase().contains("getimages")) { // Get Images in a Separate Step log.info("Beggining Image Pull @ " + Calendar.getInstance().getTime().toString()); if (context.containsBean("getImages") && gim == 0) { GetImages gi = (GetImages) context.getBean("getImages"); gi.run(); log.info("Image Pull Complete @ " + Calendar.getInstance().getTime().toString()); gi = null; } else { GetImages gi = (GetImages) context.getBean("getImages"); gi.run(); log.info("Image Pull Complete @ " + Calendar.getInstance().getTime().toString()); gi = null; } gim++; Runtime.getRuntime().gc(); System.gc(); log.info("Free Memory: " + Runtime.getRuntime().freeMemory()); } else if (activity.toLowerCase().compareTo("sql") == 0) { // execute a sql command log.info("Executing SQL Query @ " + Calendar.getInstance().getTime().toString()); if (context.containsBean("SQL") && sqls == 0) { ExecuteSQL sql; if (m.find()) { sql = (ExecuteSQL) context.getBean(activity); } else { sql = (ExecuteSQL) context.getBean("SQL"); } sql.execute(); sql = null; } else { ExecuteSQL sql; if (m.find()) { sql = (ExecuteSQL) context.getBean(activity); } else { sql = (ExecuteSQL) context.getBean("SQL" + sqls); } sql.execute(); sql = null; } sqls++; log.info("Finished SQL Query @ " + Calendar.getInstance().getTime().toString()); Runtime.getRuntime().gc(); System.gc(); log.info("Free Memory: " + Runtime.getRuntime().freeMemory()); } else if (activity.toLowerCase().compareTo("kettle") == 0) { // run one or more kettle transformation(s) log.info("Beginning Kettle Transformation @ " + Calendar.getInstance().getTime().toString()); RunTransformation rt = null; if (context.containsBean("kettle") && transforms == 0) { if (m.find()) { rt = (RunTransformation) context.getBean(activity); } else { rt = (RunTransformation) context.getBean(("kettle")); } rt.run(); rt = null; } else { if (m.find()) { rt = (RunTransformation) context.getBean(activity); } else { rt = (RunTransformation) context.getBean(("kettle" + kettle)); } rt.run(); rt = null; } transforms++; log.info("Ending Kettle Transformation @ " + Calendar.getInstance().getTime().toString()); Runtime.getRuntime().gc(); System.gc(); log.info("Free Memory: " + Runtime.getRuntime().freeMemory()); kettle++; } else if (activity.toLowerCase().contains("dumptotext")) { // dump to a text file via java log.info("Dumping to Text @ " + Calendar.getInstance().getTime().toString()); DumptoText dtt = null; if (m.find()) { dtt = (DumptoText) context.getBean(activity); } else { dtt = (DumptoText) context.getBean("DumpToText" + dump); } dtt.run(); dump++; log.info("Completed Dump @ " + Calendar.getInstance().getTime().toString()); dtt = null; Runtime.getRuntime().gc(); System.gc(); log.info("Free Memory: " + Runtime.getRuntime().freeMemory()); } else if (activity.toLowerCase().equals("jdump")) { log.info("Dumping via JDump @ " + Calendar.getInstance().getTime().toString()); if (jd == 0 && context.containsBean("JDump")) { JDump j = (JDump) context.getBean("JDump"); jd++; j.run(); } else { JDump j = (JDump) context.getBean("JDump" + jd); jd++; j.run(); } Runtime.getRuntime().gc(); System.gc(); log.info("Finished Dumping via JDump @ " + Calendar.getInstance().getTime().toString()); } else if (activity.toLowerCase().contains("jdumpwithreference")) { log.info("Dumping via JDump @ " + Calendar.getInstance().getTime().toString()); if (jd == 0 && context.containsBean("JDumpWithReference")) { JDumpWithReference j = (JDumpWithReference) context.getBean("JDumpWithReference"); jd++; j.run(); } else { JDumpWithReference j = (JDumpWithReference) context.getBean("JDumpWithReference" + jd); jd++; j.run(); } Runtime.getRuntime().gc(); System.gc(); log.info("Finished Dumping via JDump @ " + Calendar.getInstance().getTime().toString()); } else if (activity.toLowerCase().compareTo("commanddump") == 0) { // dump to text using a client side sql COPY TO command log.info("Dumping via SQL @ " + Calendar.getInstance().getTime().toString()); CommandDump d = (CommandDump) context.getBean("dump"); d.run(); d = null; log.info("Completed Dump @ " + Calendar.getInstance().getTime().toString()); // most likely not needed by satisfies my paranoia Runtime.getRuntime().gc(); System.gc(); } else if (activity.toLowerCase().equals("specdump")) { // Specified Dump log.info("Dumping via Specified Tables, Files, and Attributes @ " + Calendar.getInstance().getTime().toString()); if (context.containsBean("SpecDump") && sdump == 0) { sdump++; SpecifiedDump sd = (SpecifiedDump) context.getBean("SpecDump"); sd.run(); sd = null; } else if (context.containsBean("SpecDump" + Integer.toString(sdump))) { SpecifiedDump sd = (SpecifiedDump) context.getBean("SpecDump" + Integer.toString(sdump)); sd.run(); sd = null; } sdump++; log.info("Completed Dumping via Specified Tables, Files, and Attributes @ " + Calendar.getInstance().getTime().toString()); } else if (activity.toLowerCase().contains("specdumpwithreference")) { // Specified Dump log.info("Dumping via Specified Tables, Files, and Attributes by Reference @ " + Calendar.getInstance().getTime().toString()); if (context.containsBean("SpecDumpWithReference") && sdump == 0) { sdump++; SpecDumpWithReference sd = (SpecDumpWithReference) context.getBean("SpecDumpWithReference"); sd.run(); sd = null; } else if (context.containsBean("SpecDumpWithReference" + Integer.toString(sdump))) { SpecDumpWithReference sd = (SpecDumpWithReference) context .getBean("SpecDumpWithReference" + Integer.toString(sdump)); sd.run(); sd = null; } else { log.info("Bean Not Found For " + activity); } sdump++; log.info("Completed Dumping via Specified Tables, Files, and Attributes @ " + Calendar.getInstance().getTime().toString()); } else if (activity.toLowerCase().compareTo("email") == 0) { // email completion notice log.info("Sending Notice of Completion @ " + Calendar.getInstance().getTime().toString()); if (context.containsBean("Email") && ems == 0) { Send s = (Send) context.getBean("Email"); s.run(); s = null; } else { Send s = (Send) context.getBean("Email" + Integer.toString(ems)); s.run(); s = null; } ems++; Runtime.getRuntime().gc(); System.gc(); log.info("Completed Email @ " + Calendar.getInstance().getTime().toString()); log.info("Free Memory: " + Runtime.getRuntime().freeMemory()); } else if (activity.toLowerCase().equals("qa")) { // perform qa log.info("Performing Quality Assurance @ " + Calendar.getInstance().getTime().toString()); if (context.containsBean("QA")) { QualityAssurer qa = (QualityAssurer) context.getBean("QA"); qa.run(); qa = null; } // attempt to hint --> all tasks are really intense so anything // is nice Runtime.getRuntime().gc(); System.gc(); log.info("Completed QA @ " + Calendar.getInstance().getTime().toString()); log.info("Free Memory: " + Runtime.getRuntime().freeMemory()); } else if (activity.toLowerCase().equals("notify")) { log.info("Running Notification Tasks"); Notification nt = null; if (context.containsBean("Notify" + Integer.toString(n))) { nt = (Notification) context.getBean("Notify" + Integer.toString(n)); } else { nt = (Notification) context.getBean("Notify"); } nt.run(); nt = null; n++; if (context.containsBean("Email") && ems == 0) { Send s = (Send) context.getBean("Email"); s.run(); s = null; } else if (context.containsBean("Email" + ems)) { Send s = (Send) context.getBean("Email" + Integer.toString(ems)); s.run(); s = null; } ems++; Runtime.getRuntime().gc(); System.gc(); log.info("Free Memory: " + Runtime.getRuntime().freeMemory()); log.info("Completed Notification Tasks"); } else if (activity.toLowerCase().contains("move")) { log.info("Moving Files @ " + Calendar.getInstance().getTime().toString()); MoveFile mf = null; if (context.containsBean("Move" + Integer.toString(mv))) { mf = (MoveFile) context.getBean("Move" + Integer.toString(mv)); } else { mf = (MoveFile) context.getBean("Move"); } mf.run(); mv++; Runtime.getRuntime().gc(); log.info("Finished Moving Files @ " + Calendar.getInstance().getTime().toString()); } else if (activity.toLowerCase().contains("numericalcheck")) { log.info("Checking Counts"); NumericalChecker nc = (NumericalChecker) context.getBean("NumericalChecker"); nc.run(); Runtime.getRuntime().gc(); log.info("Finished Checking Counts @ " + Calendar.getInstance().getTime().toString()); } else if (activity.toLowerCase().contains("runscript")) { log.info("Running Script @ " + Calendar.getInstance().getTime().toString()); RunScript runner = null; if (context.containsBean("RunScript" + srn)) { runner = (RunScript) context.getBean("RunScript" + srn); } else { runner = (RunScript) context.getBean("RunScript"); } runner.run(); srn++; Runtime.getRuntime().gc(); log.info("Finished Running SCript @ " + Calendar.getInstance().getTime().toString()); } else if (activity.toLowerCase().contains("checkexistance")) { log.info("Checking Existance @ " + Calendar.getInstance().getTime().toString()); ExistanceChecker runner = null; if (context.containsBean("CheckExistance" + srn)) { runner = (ExistanceChecker) context.getBean("CheckExistance" + cen); } else { runner = (ExistanceChecker) context.getBean("CheckExistance"); } runner.run(); cen++; Runtime.getRuntime().gc(); log.info("Finished Checking Existance @ " + Calendar.getInstance().getTime().toString()); } else { log.info("Activity " + activity + " does not exist!"); } } log.info("Completed Parse @ " + Calendar.getInstance().getTime().toString()); context.destroy(); context.close(); }
From source file:fresto.datastore.titan.TitanEventWriter.java
public static void main(String[] args) throws Exception { if (args.length < 4) { LOGGER.error(//w w w . j a va 2 s . c o m "Argumests needed : <streamer url> <storage backend> <storage hostname> <max commit count>"); System.exit(1); } else { STREAMER_URL = args[0]; STORAGE_BACKEND = args[1]; STORAGE_HOSTNAME = args[2]; try { MAX_COMMIT_COUNT = Integer.parseInt(args[3]); } catch (NumberFormatException e) { LOGGER.error("Commit count should be an integer"); } } final ZMQ.Context context = ZMQ.context(1); final FrestoEventQueue frestoEventQueue = new FrestoEventQueue(); final Thread queueMonitorThread = new Thread() { //Logger _LOGGER = Logger.getLogger("writerThread"); @Override public void run() { while (work) { try { LOGGER.info("frestoEventQueue size = " + frestoEventQueue.size()); Thread.sleep(1000); } catch (InterruptedException ie) { } } } }; final Thread writerThread = new Thread() { //Logger _LOGGER = Logger.getLogger("writerThread"); @Override public void run() { //StopWatch _watch = new JavaLogStopWatch(_LOGGER); FrestoStopWatch _watch = new FrestoStopWatch(); FrestoStopWatch _durationWatch = new FrestoStopWatch(); TitanEventWriter eventWriter = new TitanEventWriter(); // Open database eventWriter.openTitanGraph(); ZMQ.Socket puller = context.socket(ZMQ.PULL); //puller.connect("tcp://" + frontHost + ":" + frontPort); puller.connect(STREAMER_URL); //Consume socket data frestoEventQueue.setPullerSocket(puller); frestoEventQueue.start(); int nextCommitCount = 0; int count = 0; long elapsedTime = 0; long duration = 0; _durationWatch.start(); while (work) { // To wait until at least one event in queue if (frestoEventQueue.isEmpty()) { try { Thread.sleep(SLEEP_TIME); continue; } catch (InterruptedException ie) { } } nextCommitCount = frestoEventQueue.size(); for (int i = 0; i < nextCommitCount; i++) { _watch.start(); count++; FrestoEvent frestoEvent = frestoEventQueue.poll(); try { eventWriter.writeEventData(frestoEvent.topic, frestoEvent.eventBytes); } catch (Exception e) { e.printStackTrace(); } finally { // } elapsedTime += _watch.stop(); duration += _durationWatch.stop(); if (count == MAX_COMMIT_COUNT) { eventWriter.commitGraph(); LOGGER.info(count + " events processed for " + elapsedTime + " ms. (total time " + duration + " ms.) Remaining events " + frestoEventQueue.size()); count = 0; elapsedTime = 0; duration = 0; // Stop FOR clause } } eventWriter.commitGraph(); LOGGER.info(count + " events processed for " + elapsedTime + " ms. (total time " + duration + " ms.) Remaining events " + frestoEventQueue.size()); count = 0; elapsedTime = 0; duration = 0; } LOGGER.info("Shutting down..."); if (g.isOpen()) { g.commit(); g.shutdown(); } puller.close(); context.term(); LOGGER.info("Good bye."); } }; Runtime.getRuntime().addShutdownHook(new Thread() { @Override public void run() { System.out.println("Interrupt received, killing server"); // To break while clause frestoEventQueue.stopWork(); work = false; try { writerThread.join(); frestoEventQueue.join(); //queueMonitorThread.join(); } catch (InterruptedException e) { } } }); //queueMonitorThread.start(); writerThread.start(); }
From source file:com.singhasdev.calamus.app.sentimentanalysis.SentimentAnalyzer.java
public static void main(String[] args) throws Exception { Properties props = new Properties(); props.put(StreamsConfig.APPLICATION_ID_CONFIG, "sentiment-analyzer"); props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092"); props.put(StreamsConfig.ZOOKEEPER_CONNECT_CONFIG, "localhost:2181"); props.put(StreamsConfig.KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass()); props.put(StreamsConfig.VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass()); // setting offset reset to earliest so that we can re-run the demo code with the same pre-loaded data props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); TopologyBuilder builder = new TopologyBuilder(); builder.addSource("Source", "test"); builder.addProcessor("Process", new CalculateSentiment(), "Source"); builder.addStateStore(/*w w w .jav a 2 s .com*/ Stores.create("SentimentAnalysis").withStringKeys().withStringValues().inMemory().build(), "Process"); builder.addSink("Sink", "test-output", "Process"); STREAMS = new KafkaStreams(builder, props); STREAMS.start(); Runtime.getRuntime().addShutdownHook(new Thread("MirrorMakerShutdownHook") { @Override public void run() { System.out.println("Closing Calamus sentiment-analyzer."); STREAMS.close(); } }); }
From source file:com.clustercontrol.HinemosManagerMain.java
/** * Hinemos Manager?main<br/>//ww w . j a v a 2 s.c o m * @param args * @throws Exception */ public static void main(String args[]) { try { try { if (System.getProperty("systime.iso") != null) { String oldVmName = System.getProperty("java.vm.name"); //System?mock?????HotSpot?????????? System.setProperty("java.vm.name", "HotSpot 64-Bit Server VM"); Class.forName("com.clustercontrol.util.SystemTimeShifter"); System.setProperty("java.vm.name", oldVmName); } } catch (Exception e) { log.error(e); } catch (Error e) { log.error(e); } long bootTime = System.currentTimeMillis(); log.info("Hinemos Manager is starting." + " (startupMode=" + _startupMode + ", clustered=" + _isClustered + ", locale=" + Locale.getDefault() + ")"); // Hinemos(??????)??????? // (???????????????????????) long offset = HinemosPropertyUtil.getHinemosPropertyNum("common.time.offset", Long.valueOf(0)); HinemosTime.setTimeOffsetMillis(offset); // Hinemos?(UTC??)??/(??) int timeZoneOffset = HinemosPropertyUtil .getHinemosPropertyNum("common.timezone", Long.valueOf(TimeZone.getDefault().getRawOffset())) .intValue(); HinemosTime.setTimeZoneOffset(timeZoneOffset); // ???HinemosPlugin??(create)? HinemosPluginService.getInstance().create(); // ???HinemosPlugin?(activate)? HinemosPluginService.getInstance().activate(); // Hinemos Manger???? Runtime.getRuntime().addShutdownHook(new Thread() { @Override public void run() { log.info("shutdown hook called."); synchronized (shutdownLock) { // Hinemos Manager??? String[] msgArgsShutdown = { _hostname }; AplLogger.put(PriorityConstant.TYPE_INFO, HinemosModuleConstant.HINEMOS_MANAGER_MONITOR, MessageConstant.MESSAGE_SYS_002_MNG, msgArgsShutdown); // ???HinemosPlugin??(deactivate)? HinemosPluginService.getInstance().deactivate(); // ???HinemosPlugin?(destroy)? HinemosPluginService.getInstance().destroy(); log.info("Hinemos Manager is stopped."); shutdown = true; shutdownLock.notify(); } } }); // ?? long startupTime = System.currentTimeMillis(); long initializeSec = (startupTime - bootTime) / 1000; long initializeMSec = (startupTime - bootTime) % 1000; log.info("Hinemos Manager Started in " + initializeSec + "s:" + initializeMSec + "ms"); // Hinemos Manager?? String[] msgArgsStart = { _hostname }; AplLogger.put(PriorityConstant.TYPE_INFO, HinemosModuleConstant.HINEMOS_MANAGER_MONITOR, MessageConstant.MESSAGE_SYS_001_MNG, msgArgsStart); // Hinemos Manager??????? synchronized (shutdownLock) { while (!shutdown) { try { shutdownLock.wait(); } catch (InterruptedException e) { log.warn("shutdown lock interrupted.", e); try { Thread.sleep(1000); } catch (InterruptedException sleepE) { } ; } } } System.exit(0); } catch (Throwable e) { log.error("unknown error occured.", e); } }
From source file:com.yahoo.pulsar.testclient.PerformanceProducer.java
public static void main(String[] args) throws Exception { final Arguments arguments = new Arguments(); JCommander jc = new JCommander(arguments); jc.setProgramName("pulsar-perf-producer"); try {/*from w w w . j a va2s . c o m*/ jc.parse(args); } catch (ParameterException e) { System.out.println(e.getMessage()); jc.usage(); System.exit(-1); } if (arguments.help) { jc.usage(); System.exit(-1); } if (arguments.destinations.size() != 1) { System.out.println("Only one topic name is allowed"); jc.usage(); System.exit(-1); } if (arguments.confFile != null) { Properties prop = new Properties(System.getProperties()); prop.load(new FileInputStream(arguments.confFile)); if (arguments.serviceURL == null) { arguments.serviceURL = prop.getProperty("brokerServiceUrl"); } if (arguments.serviceURL == null) { arguments.serviceURL = prop.getProperty("webServiceUrl"); } // fallback to previous-version serviceUrl property to maintain backward-compatibility if (arguments.serviceURL == null) { arguments.serviceURL = prop.getProperty("serviceUrl", "http://localhost:8080/"); } if (arguments.authPluginClassName == null) { arguments.authPluginClassName = prop.getProperty("authPlugin", null); } if (arguments.authParams == null) { arguments.authParams = prop.getProperty("authParams", null); } } arguments.testTime = TimeUnit.SECONDS.toMillis(arguments.testTime); // Dump config variables ObjectMapper m = new ObjectMapper(); ObjectWriter w = m.writerWithDefaultPrettyPrinter(); log.info("Starting Pulsar perf producer with config: {}", w.writeValueAsString(arguments)); // Read payload data from file if needed byte payloadData[]; if (arguments.payloadFilename != null) { payloadData = Files.readAllBytes(Paths.get(arguments.payloadFilename)); } else { payloadData = new byte[arguments.msgSize]; } // Now processing command line arguments String prefixTopicName = arguments.destinations.get(0); List<Future<Producer>> futures = Lists.newArrayList(); EventLoopGroup eventLoopGroup; if (SystemUtils.IS_OS_LINUX) { eventLoopGroup = new EpollEventLoopGroup(Runtime.getRuntime().availableProcessors(), new DefaultThreadFactory("pulsar-perf-producer")); } else { eventLoopGroup = new NioEventLoopGroup(Runtime.getRuntime().availableProcessors(), new DefaultThreadFactory("pulsar-perf-producer")); } ClientConfiguration clientConf = new ClientConfiguration(); clientConf.setConnectionsPerBroker(arguments.maxConnections); clientConf.setStatsInterval(arguments.statsIntervalSeconds, TimeUnit.SECONDS); if (isNotBlank(arguments.authPluginClassName)) { clientConf.setAuthentication(arguments.authPluginClassName, arguments.authParams); } PulsarClient client = new PulsarClientImpl(arguments.serviceURL, clientConf, eventLoopGroup); ProducerConfiguration producerConf = new ProducerConfiguration(); producerConf.setSendTimeout(0, TimeUnit.SECONDS); producerConf.setCompressionType(arguments.compression); // enable round robin message routing if it is a partitioned topic producerConf.setMessageRoutingMode(MessageRoutingMode.RoundRobinPartition); if (arguments.batchTime > 0) { producerConf.setBatchingMaxPublishDelay(arguments.batchTime, TimeUnit.MILLISECONDS); producerConf.setBatchingEnabled(true); producerConf.setMaxPendingMessages(arguments.msgRate); } for (int i = 0; i < arguments.numTopics; i++) { String topic = (arguments.numTopics == 1) ? prefixTopicName : String.format("%s-%d", prefixTopicName, i); log.info("Adding {} publishers on destination {}", arguments.numProducers, topic); for (int j = 0; j < arguments.numProducers; j++) { futures.add(client.createProducerAsync(topic, producerConf)); } } final List<Producer> producers = Lists.newArrayListWithCapacity(futures.size()); for (Future<Producer> future : futures) { producers.add(future.get()); } log.info("Created {} producers", producers.size()); Runtime.getRuntime().addShutdownHook(new Thread() { public void run() { printAggregatedStats(); } }); Collections.shuffle(producers); AtomicBoolean isDone = new AtomicBoolean(); executor.submit(() -> { try { RateLimiter rateLimiter = RateLimiter.create(arguments.msgRate); long startTime = System.currentTimeMillis(); // Send messages on all topics/producers long totalSent = 0; while (true) { for (Producer producer : producers) { if (arguments.testTime > 0) { if (System.currentTimeMillis() - startTime > arguments.testTime) { log.info("------------------- DONE -----------------------"); printAggregatedStats(); isDone.set(true); Thread.sleep(5000); System.exit(0); } } if (arguments.numMessages > 0) { if (totalSent++ >= arguments.numMessages) { log.info("------------------- DONE -----------------------"); printAggregatedStats(); isDone.set(true); Thread.sleep(5000); System.exit(0); } } rateLimiter.acquire(); final long sendTime = System.nanoTime(); producer.sendAsync(payloadData).thenRun(() -> { messagesSent.increment(); bytesSent.add(payloadData.length); long latencyMicros = NANOSECONDS.toMicros(System.nanoTime() - sendTime); recorder.recordValue(latencyMicros); cumulativeRecorder.recordValue(latencyMicros); }).exceptionally(ex -> { log.warn("Write error on message", ex); System.exit(-1); return null; }); } } } catch (Throwable t) { log.error("Got error", t); } }); // Print report stats long oldTime = System.nanoTime(); Histogram reportHistogram = null; String statsFileName = "perf-producer-" + System.currentTimeMillis() + ".hgrm"; log.info("Dumping latency stats to {}", statsFileName); PrintStream histogramLog = new PrintStream(new FileOutputStream(statsFileName), false); HistogramLogWriter histogramLogWriter = new HistogramLogWriter(histogramLog); // Some log header bits histogramLogWriter.outputLogFormatVersion(); histogramLogWriter.outputLegend(); while (true) { try { Thread.sleep(10000); } catch (InterruptedException e) { break; } if (isDone.get()) { break; } long now = System.nanoTime(); double elapsed = (now - oldTime) / 1e9; double rate = messagesSent.sumThenReset() / elapsed; double throughput = bytesSent.sumThenReset() / elapsed / 1024 / 1024 * 8; reportHistogram = recorder.getIntervalHistogram(reportHistogram); log.info( "Throughput produced: {} msg/s --- {} Mbit/s --- Latency: mean: {} ms - med: {} - 95pct: {} - 99pct: {} - 99.9pct: {} - 99.99pct: {} - Max: {}", throughputFormat.format(rate), throughputFormat.format(throughput), dec.format(reportHistogram.getMean() / 1000.0), dec.format(reportHistogram.getValueAtPercentile(50) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(95) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99.9) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99.99) / 1000.0), dec.format(reportHistogram.getMaxValue() / 1000.0)); histogramLogWriter.outputIntervalHistogram(reportHistogram); reportHistogram.reset(); oldTime = now; } client.close(); }
From source file:fresto.datastore.EventLogWriter.java
public static void main(String[] args) throws Exception { if (args.length != 2) { LOGGER.severe("Argumests needed : <frontHost> <frontPort>"); System.exit(1);//w w w.j a v a 2 s . c om } else { frontHost = args[0]; frontPort = args[1]; LOGGER.info("Connecting... " + frontHost + ":" + frontPort + " with SUB"); } final ZMQ.Context context = ZMQ.context(1); final FrestoEventQueue frestoEventQueue = new FrestoEventQueue(); final Thread queueMonitorThread = new Thread() { Logger _LOGGER = Logger.getLogger("logWriteThread"); @Override public void run() { while (work) { try { _LOGGER.info("frestoEventQueue size = " + frestoEventQueue.size()); Thread.sleep(1000); } catch (InterruptedException ie) { } } } }; final Thread logWriteThread = new Thread() { Logger _LOGGER = Logger.getLogger("logWriteThread"); @Override public void run() { //FrestoStopWatch _watch = new FrestoStopWatch(); //FrestoStopWatch _durationWatch = new FrestoStopWatch(); EventLogWriter eventLogWriter = new EventLogWriter(); // Open database //eventLogWriter.openTitanGraph(); ZMQ.Socket receiver = null; //if("pull".equalsIgnoreCase(subOrPull)) { // receiver = context.socket(ZMQ.PULL); // receiver.connect("tcp://" + frontHost + ":" + frontPort); //} else if("sub".equalsIgnoreCase(subOrPull)) { receiver = context.socket(ZMQ.SUB); receiver.connect("tcp://" + frontHost + ":" + frontPort); receiver.subscribe("".getBytes()); //} else { // LOGGER.severe(subOrPull + " is not supported."); // System.exit(1); //} //Consume socket data frestoEventQueue.setPullerSocket(receiver); frestoEventQueue.start(); int waitingEventCount = 0; //int count = 0; //long elapsedTime = 0; //long duration = 0; //_durationWatch.start(); while (work) { // To wait until at least one event in queue if (frestoEventQueue.isEmpty()) { try { //_LOGGER.info("FrestoEventQueue is empty. Waiting " + SLEEP_TIME + "ms..."); Thread.sleep(SLEEP_TIME); continue; } catch (InterruptedException ie) { } } waitingEventCount = frestoEventQueue.size(); for (int i = 0; i < waitingEventCount; i++) { //_watch.start(); //count++; FrestoEvent frestoEvent = frestoEventQueue.poll(); try { eventLogWriter.writeEventData(frestoEvent.topic, frestoEvent.eventBytes); } catch (Exception e) { e.printStackTrace(); } finally { // } //elapsedTime += _watch.stop(); //duration += _durationWatch.stop(); //if(count == maxCommitCount) { // eventLogWriter.commitGraph(); // _LOGGER.info(count + " events processed for " + elapsedTime + " ms. (total time " + duration + " ms.) Remaining events " + frestoEventQueue.size()); // // count = 0; // elapsedTime = 0; // duration = 0; // // Stop FOR clause //} } //eventLogWriter.commitGraph(); _LOGGER.info("Remaining events " + frestoEventQueue.size()); //count = 0; //elapsedTime = 0; //duration = 0; } _LOGGER.info("Shutting down..."); //if(g.isOpen()) { // g.commit(); // g.shutdown(); //} receiver.close(); context.term(); _LOGGER.info("Good bye."); } }; Runtime.getRuntime().addShutdownHook(new Thread() { @Override public void run() { System.out.println(" Interrupt received, killing logger"); // To break while clause frestoEventQueue.stopWork(); work = false; try { logWriteThread.join(); frestoEventQueue.join(); //queueMonitorThread.join(); } catch (InterruptedException e) { // } } }); //queueMonitorThread.start(); logWriteThread.start(); }