List of usage examples for com.google.common.collect Lists newArrayList
@GwtCompatible(serializable = true) public static <E> ArrayList<E> newArrayList()
From source file:com.mapr.franz.server.Server.java
public static void main(String[] args) throws IOException, InterruptedException, KeeperException { if (args.length < 3) { System.out.println("Usage: java -cp <classpath> " + "com.mapr.franz.server.Server " + "<server_write_path> <hostname> <port> [zkhost:port]"); System.exit(1);/*w w w .ja v a 2s. c o m*/ } Properties props = loadProperties(); /* * args */ log.info("ARGUMENTS:"); for (int i = 0; i < args.length; i++) { log.info(i + "-" + args[i]); } setBasePath(args[0]); int port = Integer.parseInt(args[2]); PeerInfo serverInfo = new PeerInfo(args[1], port); String zk_str = props.getProperty("zookeeper.connection.string", ZK_CONNECT_STRING); if (args.length == 4) { zk_str = args[3]; } //You need then to create a DuplexTcpServerBootstrap and provide it an RpcCallExecutor. DuplexTcpServerBootstrap bootstrap = new DuplexTcpServerBootstrap(serverInfo, new NioServerSocketChannelFactory(Executors.newCachedThreadPool(), Executors.newCachedThreadPool())); bootstrap.setRpcServerCallExecutor(new ThreadPoolCallExecutor(10, 10)); // set up request logging // final CategoryPerServiceLogger logPolicy = new CategoryPerServiceLogger(); // logPolicy.setLogRequestProto(true); // logPolicy.setLogResponseProto(true); // bootstrap.setLogger(logPolicy); //Finally binding the bootstrap to the TCP port will start off the socket accepting and clients can start to connect. long serverId = new SecureRandom().nextLong(); List<Client.HostPort> addresses = Lists.newArrayList(); Enumeration<NetworkInterface> networkInterfaces = NetworkInterface.getNetworkInterfaces(); while (networkInterfaces.hasMoreElements()) { NetworkInterface ifc = networkInterfaces.nextElement(); if (!ifc.isLoopback()) { for (InterfaceAddress address : ifc.getInterfaceAddresses()) { addresses.add(new Client.HostPort(address.getAddress().getHostAddress(), port)); } } } ClusterState zkState = new ClusterState(zk_str, FRANZ_BASE, new Info(serverId, addresses)); bootstrap.getRpcServiceRegistry().registerBlockingService(Catcher.CatcherService .newReflectiveBlockingService(new com.mapr.franz.server.CatcherServiceImpl(serverId, zkState))); //If you want to track the RPC peering events with clients, use a RpcClientConnectionRegistry or a TcpConnectionEventListener for TCP connection events. This is the mechanism you can use to "discover" RPC clients before they "call" any service. TcpConnectionEventListener listener = new TcpConnectionEventListener() { @Override public void connectionClosed(RpcClientChannel clientChannel) { log.debug("Disconnect from {}", clientChannel.getPeerInfo()); } @Override public void connectionOpened(RpcClientChannel clientChannel) { log.debug("Connect with {}", clientChannel.getPeerInfo()); } }; bootstrap.registerConnectionEventListener(listener); bootstrap.bind(); }
From source file:tv.icntv.recommend.algorithm.cf.ViewTimeReducer.java
public static void main(String[] args) { // System.out.println(MathExtend.divide(4,(double)5,5)); List<String> list = Lists.newArrayList(); Collections.sort(list);//from w w w . j a v a2 s . co m System.out.println(list); }
From source file:org.apache.hadoop.hive.ql.io.orc.FileDump.java
public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); List<Integer> rowIndexCols = null; Options opts = createOptions();/*from w ww .ja v a2 s. c om*/ CommandLine cli = new GnuParser().parse(opts, args); if (cli.hasOption('h')) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp("orcfiledump", opts); return; } boolean dumpData = cli.hasOption('d'); boolean recover = cli.hasOption("recover"); boolean skipDump = cli.hasOption("skip-dump"); String backupPath = DEFAULT_BACKUP_PATH; if (cli.hasOption("backup-path")) { backupPath = cli.getOptionValue("backup-path"); } if (cli.hasOption("r")) { String[] colStrs = cli.getOptionValue("r").split(","); rowIndexCols = new ArrayList<Integer>(colStrs.length); for (String colStr : colStrs) { rowIndexCols.add(Integer.parseInt(colStr)); } } boolean printTimeZone = cli.hasOption('t'); boolean jsonFormat = cli.hasOption('j'); String[] files = cli.getArgs(); if (files.length == 0) { System.err.println("Error : ORC files are not specified"); return; } // if the specified path is directory, iterate through all files and print the file dump List<String> filesInPath = Lists.newArrayList(); for (String filename : files) { Path path = new Path(filename); filesInPath.addAll(getAllFilesInPath(path, conf)); } if (dumpData) { printData(filesInPath, conf); } else if (recover && skipDump) { recoverFiles(filesInPath, conf, backupPath); } else { if (jsonFormat) { boolean prettyPrint = cli.hasOption('p'); JsonFileDump.printJsonMetaData(filesInPath, conf, rowIndexCols, prettyPrint, printTimeZone); } else { printMetaData(filesInPath, conf, rowIndexCols, printTimeZone, recover, backupPath); } } }
From source file:ca.dealsaccess.scout.driver.ScoutDriver.java
public static void main(String[] args) throws Throwable { ProgramDriver programDriver = new ProgramDriver(); Properties mainClasses = loadProperties("driver.classes.props"); if (mainClasses == null) { mainClasses = loadProperties("driver.classes.default.props"); }// ww w .j a v a 2 s. c om if (mainClasses == null) { throw new IOException("Can't load any properties file?"); } boolean foundShortName = false; for (Object key : mainClasses.keySet()) { String keyString = (String) key; if (args.length > 0 && shortName(mainClasses.getProperty(keyString)).equals(args[0])) { foundShortName = true; } if (args.length > 0 && keyString.equalsIgnoreCase(args[0]) && isDeprecated(mainClasses, keyString)) { log.error(desc(mainClasses.getProperty(keyString))); return; } if (isDeprecated(mainClasses, keyString)) { continue; } addClass(programDriver, keyString, mainClasses.getProperty(keyString)); } if (args.length < 1 || args[0] == null || "-h".equals(args[0]) || "--help".equals(args[0])) { programDriver.driver(args); } String progName = args[0]; if (!foundShortName) { addClass(programDriver, progName, progName); } shift(args); Properties mainProps = loadProperties(progName + ".props"); if (mainProps == null) { log.warn("No {}.props found on classpath, will use command-line arguments only", progName); mainProps = new Properties(); } Map<String, String[]> argMap = Maps.newHashMap(); int i = 0; while (i < args.length && args[i] != null) { List<String> argValues = Lists.newArrayList(); String arg = args[i]; i++; if (arg.startsWith("-D")) { // '-Dkey=value' or '-Dkey=value1,value2,etc' case String[] argSplit = arg.split("="); arg = argSplit[0]; if (argSplit.length == 2) { argValues.add(argSplit[1]); } } else { // '-key [values]' or '--key [values]' case. while (i < args.length && args[i] != null) { if (args[i].startsWith("-")) { break; } argValues.add(args[i]); i++; } } argMap.put(arg, argValues.toArray(new String[argValues.size()])); } // Add properties from the .props file that are not overridden on the command line for (String key : mainProps.stringPropertyNames()) { String[] argNamePair = key.split("\\|"); String shortArg = '-' + argNamePair[0].trim(); String longArg = argNamePair.length < 2 ? null : "--" + argNamePair[1].trim(); if (!argMap.containsKey(shortArg) && (longArg == null || !argMap.containsKey(longArg))) { argMap.put(longArg, new String[] { mainProps.getProperty(key) }); } } // Now add command-line args List<String> argsList = Lists.newArrayList(); argsList.add(progName); for (Map.Entry<String, String[]> entry : argMap.entrySet()) { String arg = entry.getKey(); if (arg.startsWith("-D")) { // arg is -Dkey - if value for this !isEmpty(), then arg -> -Dkey + "=" + value String[] argValues = entry.getValue(); if (argValues.length > 0 && !argValues[0].trim().isEmpty()) { arg += '=' + argValues[0].trim(); } argsList.add(1, arg); } else { argsList.add(arg); for (String argValue : Arrays.asList(argMap.get(arg))) { if (!argValue.isEmpty()) { argsList.add(argValue); } } } } long start = System.currentTimeMillis(); programDriver.driver(argsList.toArray(new String[argsList.size()])); if (log.isInfoEnabled()) { log.info("Program took {} ms (Minutes: {})", System.currentTimeMillis() - start, (System.currentTimeMillis() - start) / 60000.0); } }
From source file:org.apache.mahout.driver.MahoutDriver.java
public static void main(String[] args) throws Throwable { Properties mainClasses = loadProperties("driver.classes.props"); if (mainClasses == null) { mainClasses = loadProperties("driver.classes.default.props"); }// w ww . j a v a 2 s .c o m if (mainClasses == null) { throw new IOException("Can't load any properties file?"); } boolean foundShortName = false; ProgramDriver programDriver = new ProgramDriver(); for (Object key : mainClasses.keySet()) { String keyString = (String) key; if (args.length > 0 && shortName(mainClasses.getProperty(keyString)).equals(args[0])) { foundShortName = true; } if (args.length > 0 && keyString.equalsIgnoreCase(args[0]) && isDeprecated(mainClasses, keyString)) { log.error(desc(mainClasses.getProperty(keyString))); return; } if (isDeprecated(mainClasses, keyString)) { continue; } addClass(programDriver, keyString, mainClasses.getProperty(keyString)); } if (args.length < 1 || args[0] == null || "-h".equals(args[0]) || "--help".equals(args[0])) { programDriver.driver(args); return; } String progName = args[0]; if (!foundShortName) { addClass(programDriver, progName, progName); } shift(args); Properties mainProps = loadProperties(progName + ".props"); if (mainProps == null) { log.warn("No {}.props found on classpath, will use command-line arguments only", progName); mainProps = new Properties(); } Map<String, String[]> argMap = Maps.newHashMap(); int i = 0; while (i < args.length && args[i] != null) { List<String> argValues = Lists.newArrayList(); String arg = args[i]; i++; if (arg.startsWith("-D")) { // '-Dkey=value' or '-Dkey=value1,value2,etc' case String[] argSplit = arg.split("="); arg = argSplit[0]; if (argSplit.length == 2) { argValues.add(argSplit[1]); } } else { // '-key [values]' or '--key [values]' case. while (i < args.length && args[i] != null) { if (args[i].startsWith("-")) { break; } argValues.add(args[i]); i++; } } argMap.put(arg, argValues.toArray(new String[argValues.size()])); } // Add properties from the .props file that are not overridden on the command line for (String key : mainProps.stringPropertyNames()) { String[] argNamePair = key.split("\\|"); String shortArg = '-' + argNamePair[0].trim(); String longArg = argNamePair.length < 2 ? null : "--" + argNamePair[1].trim(); if (!argMap.containsKey(shortArg) && (longArg == null || !argMap.containsKey(longArg))) { argMap.put(longArg, new String[] { mainProps.getProperty(key) }); } } // Now add command-line args List<String> argsList = Lists.newArrayList(); argsList.add(progName); for (Map.Entry<String, String[]> entry : argMap.entrySet()) { String arg = entry.getKey(); if (arg.startsWith("-D")) { // arg is -Dkey - if value for this !isEmpty(), then arg -> -Dkey + "=" + value String[] argValues = entry.getValue(); if (argValues.length > 0 && !argValues[0].trim().isEmpty()) { arg += '=' + argValues[0].trim(); } argsList.add(1, arg); } else { argsList.add(arg); for (String argValue : Arrays.asList(argMap.get(arg))) { if (!argValue.isEmpty()) { argsList.add(argValue); } } } } long start = System.currentTimeMillis(); programDriver.driver(argsList.toArray(new String[argsList.size()])); if (log.isInfoEnabled()) { log.info("Program took {} ms (Minutes: {})", System.currentTimeMillis() - start, (System.currentTimeMillis() - start) / 60000.0); } }
From source file:org.apache.drill.exec.store.hive.HiveInputReader.java
public static void main(String args[]) throws Exception { /*/*from www . j a va 2 s . co m*/ String[] columnNames = {"n_nationkey", "n_name", "n_regionkey", "n_comment"}; String[] columnTypes = {"bigint", "string", "bigint", "string"}; List<FieldSchema> cols = Lists.newArrayList(); for (int i = 0; i < columnNames.length; i++) { cols.add(new FieldSchema(columnNames[i], columnTypes[i], null)); } String location = "file:///tmp/nation_s"; String inputFormat = TextInputFormat.class.getCanonicalName(); String serdeLib = LazySimpleSerDe.class.getCanonicalName(); // String inputFormat = HiveHBaseTableInputFormat.class.getCanonicalName(); // String serdeLib = HBaseSerDe.class.getCanonicalName(); Map<String, String> serdeParams = new HashMap(); // serdeParams.put("serialization.format", "1"); // serdeParams.put("hbase.columns.mapping", ":key,f:name,f:regionkey,f:comment"); serdeParams.put("serialization.format", "|"); serdeParams.put("field.delim", "|"); Map<String, String> tableParams = new HashMap(); tableParams.put("hbase.table.name", "nation"); SerDeInfo serDeInfo = new SerDeInfo(null, serdeLib, serdeParams); StorageDescriptor storageDescriptor = new StorageDescriptor(cols, location, inputFormat, null, false, -1, serDeInfo, null, null, null); Table table = new Table("table", "default", "sphillips", 0, 0, 0, storageDescriptor, new ArrayList<FieldSchema>(), tableParams, null, null, "MANAGED_TABLE"); Properties properties = MetaStoreUtils.getTableMetadata(table); */ HiveConf conf = new HiveConf(); conf.set("hive.metastore.uris", "thrift://10.10.31.51:9083"); HiveMetaStoreClient client = new HiveMetaStoreClient(conf); Table table = client.getTable("default", "nation"); Properties properties = MetaStoreUtils.getTableMetadata(table); Path path = new Path(table.getSd().getLocation()); JobConf job = new JobConf(); for (Object obj : properties.keySet()) { job.set((String) obj, (String) properties.get(obj)); } // job.set("hbase.zookeeper.quorum", "10.10.31.51"); // job.set("hbase.zookeeper.property.clientPort", "5181"); InputFormat f = (InputFormat) Class.forName(table.getSd().getInputFormat()).getConstructor().newInstance(); job.setInputFormat(f.getClass()); FileInputFormat.addInputPath(job, path); InputFormat format = job.getInputFormat(); SerDe serde = (SerDe) Class.forName(table.getSd().getSerdeInfo().getSerializationLib()).getConstructor() .newInstance(); serde.initialize(job, properties); ObjectInspector inspector = serde.getObjectInspector(); ObjectInspector.Category cat = inspector.getCategory(); TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromObjectInspector(inspector); List<String> columns = null; List<TypeInfo> colTypes = null; List<ObjectInspector> fieldObjectInspectors = Lists.newArrayList(); switch (typeInfo.getCategory()) { case STRUCT: columns = ((StructTypeInfo) typeInfo).getAllStructFieldNames(); colTypes = ((StructTypeInfo) typeInfo).getAllStructFieldTypeInfos(); for (int i = 0; i < columns.size(); i++) { System.out.print(columns.get(i)); System.out.print(" "); System.out.print(colTypes.get(i)); } System.out.println(""); for (StructField field : ((StructObjectInspector) inspector).getAllStructFieldRefs()) { fieldObjectInspectors.add(field.getFieldObjectInspector()); } } for (InputSplit split : format.getSplits(job, 1)) { String encoded = serializeInputSplit(split); System.out.println(encoded); InputSplit newSplit = deserializeInputSplit(encoded, split.getClass().getCanonicalName()); System.out.print("Length: " + newSplit.getLength() + " "); System.out.print("Locations: "); for (String loc : newSplit.getLocations()) System.out.print(loc + " "); System.out.println(); } for (InputSplit split : format.getSplits(job, 1)) { RecordReader reader = format.getRecordReader(split, job, Reporter.NULL); Object key = reader.createKey(); Object value = reader.createValue(); int count = 0; while (reader.next(key, value)) { List<Object> values = ((StructObjectInspector) inspector) .getStructFieldsDataAsList(serde.deserialize((Writable) value)); StructObjectInspector sInsp = (StructObjectInspector) inspector; Object obj = sInsp.getStructFieldData(serde.deserialize((Writable) value), sInsp.getStructFieldRef("n_name")); System.out.println(obj); /* for (Object obj : values) { PrimitiveObjectInspector.PrimitiveCategory pCat = ((PrimitiveObjectInspector)fieldObjectInspectors.get(count)).getPrimitiveCategory(); Object pObj = ((PrimitiveObjectInspector)fieldObjectInspectors.get(count)).getPrimitiveJavaObject(obj); System.out.print(pObj + " "); } */ System.out.println(""); } } }
From source file:org.spdx.tools.LicenseRDFAGenerator.java
/** * @param args Arg 0 is the input spreadsheet, arg 1 is the directory for the output html files *//*from w ww . j av a 2 s.c o m*/ public static void main(String[] args) { if (args == null || args.length < MIN_ARGS || args.length > MAX_ARGS) { System.out.println("Invalid arguments"); usage(); return; } File ssFile = new File(args[0]); if (!ssFile.exists()) { System.out.println("Spreadsheet file " + ssFile.getName() + " does not exist"); usage(); return; } File dir = new File(args[1]); if (!dir.exists()) { System.out.println("Output directory " + dir.getName() + " does not exist"); usage(); return; } if (!dir.isDirectory()) { System.out.println("Output directory " + dir.getName() + " is not a directory"); usage(); return; } String version = null; if (args.length > 2) { version = args[2]; } String releaseDate = null; if (args.length > 3) { releaseDate = args[3]; } List<String> warnings = Lists.newArrayList(); ISpdxListedLicenseProvider licenseProvider = null; try { if (ssFile.getName().toLowerCase().endsWith(".xls")) { SPDXLicenseSpreadsheet licenseSpreadsheet = new SPDXLicenseSpreadsheet(ssFile, false, true); licenseProvider = licenseSpreadsheet; if (version == null || version.trim().isEmpty()) { version = licenseSpreadsheet.getLicenseSheet().getVersion(); } if (releaseDate == null || releaseDate.trim().isEmpty()) { releaseDate = licenseSpreadsheet.getLicenseSheet().getReleaseDate(); } } else { System.out.println("Unsupported file format. Must be a .xls file"); System.exit(ERROR_STATUS); } File textFolder = new File(dir.getPath() + File.separator + TEXT_FOLDER_NAME); if (!textFolder.isDirectory() && !textFolder.mkdir()) { System.out.println("Error: text folder is not a directory"); return; } File templateFolder = new File(dir.getPath() + File.separator + TEMPLATE_FOLDER_NAME); if (!templateFolder.isDirectory() && !templateFolder.mkdir()) { System.out.println("Error: template folder is not a directory"); return; } File htmlFolder = new File(dir.getPath() + File.separator + HTML_FOLDER_NAME); if (!htmlFolder.isDirectory() && !htmlFolder.mkdir()) { System.out.println("Error: HTML folder is not a directory"); return; } File rdfaFolder = new File(dir.getPath() + File.separator + RDFA_FOLDER_NAME); if (!rdfaFolder.isDirectory() && !rdfaFolder.mkdir()) { System.out.println("Error: RDFa folder is not a directory"); return; } File jsonFolder = new File(dir.getPath() + File.separator + JSON_FOLDER_NAME); if (!jsonFolder.isDirectory() && !jsonFolder.mkdir()) { System.out.println("Error: JSON folder is not a directory"); return; } File jsonFolderDetails = new File( dir.getPath() + File.separator + JSON_FOLDER_NAME + File.separator + "details"); if (!jsonFolderDetails.isDirectory() && !jsonFolderDetails.mkdir()) { System.out.println("Error: JSON folder is not a directory"); return; } File jsonFolderExceptions = new File( dir.getPath() + File.separator + JSON_FOLDER_NAME + File.separator + "exceptions"); if (!jsonFolderExceptions.isDirectory() && !jsonFolderExceptions.mkdir()) { System.out.println("Error: JSON folder is not a directory"); return; } File website = new File(dir.getPath() + File.separator + WEBSITE_FOLDER_NAME); if (!website.isDirectory() && !website.mkdir()) { System.out.println("Error: Website folder is not a directory"); return; } System.out.print("Processing License List"); writeLicenseList(version, releaseDate, licenseProvider, warnings, website, textFolder, htmlFolder, templateFolder, rdfaFolder, jsonFolder); System.out.println(); System.out.print("Processing Exceptions"); writeExceptionList(version, releaseDate, licenseProvider, warnings, website, textFolder, htmlFolder, templateFolder, rdfaFolder, jsonFolder); writeCssFile(website); writeSortTableFile(website); System.out.println(); if (warnings.size() > 0) { System.out.println("The following warning(s) were identified:"); for (String warning : warnings) { System.out.println("\t" + warning); } } System.out.println("Completed processing licenses"); } catch (SpreadsheetException e) { System.out.println("\nInvalid spreadsheet: " + e.getMessage()); } catch (SpdxListedLicenseException e) { System.out.println("\nError reading standard licenses: " + e.getMessage()); } catch (InvalidLicenseTemplateException e) { System.out.println("\nInvalid template found on one of the licenses: " + e.getMessage()); } catch (Exception e) { System.out.println("\nUnhandled exception generating html:"); e.printStackTrace(); } finally { if (licenseProvider != null && (licenseProvider instanceof SPDXLicenseSpreadsheet)) { try { SPDXLicenseSpreadsheet spreadsheet = (SPDXLicenseSpreadsheet) licenseProvider; spreadsheet.close(); } catch (SpreadsheetException e) { System.out.println("Error closing spreadsheet file: " + e.getMessage()); } } } }
From source file:es.tid.fiware.fiwareconnectors.cygnus.nodes.CygnusApplication.java
/** * Main application to be run when this CygnusApplication is invoked. The only differences with the original one * are the CygnusApplication is used instead of the Application one, and the Management Interface port option in * the command line.//from w w w . j a v a2 s . co m * @param args */ public static void main(String[] args) { try { Options options = new Options(); Option option = new Option("n", "name", true, "the name of this agent"); option.setRequired(true); options.addOption(option); option = new Option("f", "conf-file", true, "specify a conf file"); option.setRequired(true); options.addOption(option); option = new Option(null, "no-reload-conf", false, "do not reload " + "conf file if changed"); options.addOption(option); option = new Option("h", "help", false, "display help text"); options.addOption(option); option = new Option("p", "mgmt-if-port", true, "the management interface port"); option.setRequired(false); options.addOption(option); CommandLineParser parser = new GnuParser(); CommandLine commandLine = parser.parse(options, args); File configurationFile = new File(commandLine.getOptionValue('f')); String agentName = commandLine.getOptionValue('n'); boolean reload = !commandLine.hasOption("no-reload-conf"); if (commandLine.hasOption('h')) { new HelpFormatter().printHelp("flume-ng agent", options, true); return; } // if int mgmtIfPort = 8081; // default value if (commandLine.hasOption('p')) { mgmtIfPort = new Integer(commandLine.getOptionValue('p')).intValue(); } // if // the following is to ensure that by default the agent will fail on startup if the file does not exist if (!configurationFile.exists()) { // if command line invocation, then need to fail fast if (System.getProperty(Constants.SYSPROP_CALLED_FROM_SERVICE) == null) { String path = configurationFile.getPath(); try { path = configurationFile.getCanonicalPath(); } catch (IOException ex) { logger.error("Failed to read canonical path for file: " + path, ex); } // try catch throw new ParseException("The specified configuration file does not exist: " + path); } // if } // if List<LifecycleAware> components = Lists.newArrayList(); CygnusApplication application; if (reload) { EventBus eventBus = new EventBus(agentName + "-event-bus"); PollingPropertiesFileConfigurationProvider configurationProvider = new PollingPropertiesFileConfigurationProvider( agentName, configurationFile, eventBus, 30); components.add(configurationProvider); application = new CygnusApplication(components, mgmtIfPort); eventBus.register(application); } else { PropertiesFileConfigurationProvider configurationProvider = new PropertiesFileConfigurationProvider( agentName, configurationFile); application = new CygnusApplication(mgmtIfPort); application.handleConfigurationEvent(configurationProvider.getConfiguration()); } // if else application.start(); final CygnusApplication appReference = application; Runtime.getRuntime().addShutdownHook(new Thread("agent-shutdown-hook") { @Override public void run() { appReference.stop(); } // run }); } catch (Exception e) { logger.error("A fatal error occurred while running. Exception follows.", e); } // try catch }
From source file:org.apache.orc.tools.FileDump.java
public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); List<Integer> rowIndexCols = new ArrayList<Integer>(0); Options opts = createOptions();/*from w w w . jav a2 s . co m*/ CommandLine cli = new GnuParser().parse(opts, args); if (cli.hasOption('h')) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp("orcfiledump", opts); return; } boolean dumpData = cli.hasOption('d'); boolean recover = cli.hasOption("recover"); boolean skipDump = cli.hasOption("skip-dump"); String backupPath = DEFAULT_BACKUP_PATH; if (cli.hasOption("backup-path")) { backupPath = cli.getOptionValue("backup-path"); } if (cli.hasOption("r")) { String val = cli.getOptionValue("r"); if (val != null && val.trim().equals("*")) { rowIndexCols = null; // All the columns } else { String[] colStrs = cli.getOptionValue("r").split(","); rowIndexCols = new ArrayList<Integer>(colStrs.length); for (String colStr : colStrs) { rowIndexCols.add(Integer.parseInt(colStr)); } } } boolean printTimeZone = cli.hasOption('t'); boolean jsonFormat = cli.hasOption('j'); String[] files = cli.getArgs(); if (files.length == 0) { System.err.println("Error : ORC files are not specified"); return; } // if the specified path is directory, iterate through all files and print the file dump List<String> filesInPath = Lists.newArrayList(); for (String filename : files) { Path path = new Path(filename); filesInPath.addAll(getAllFilesInPath(path, conf)); } if (dumpData) { printData(filesInPath, conf); } else if (recover && skipDump) { recoverFiles(filesInPath, conf, backupPath); } else { if (jsonFormat) { boolean prettyPrint = cli.hasOption('p'); JsonFileDump.printJsonMetaData(filesInPath, conf, rowIndexCols, prettyPrint, printTimeZone); } else { printMetaData(filesInPath, conf, rowIndexCols, printTimeZone, recover, backupPath); } } }
From source file:org.apache.ctakes.temporal.data.analysis.PrintInconsistentAnnotations.java
public static void main(String[] args) throws Exception { Options options = CliFactory.parseArguments(Options.class, args); int windowSize = 50; List<Integer> patientSets = options.getPatients().getList(); List<Integer> trainItems = THYMEData.getPatientSets(patientSets, THYMEData.TRAIN_REMAINDERS); List<File> files = THYMEData.getFilesFor(trainItems, options.getRawTextDirectory()); CollectionReader reader = UriCollectionReader.getCollectionReaderFromFiles(files); AggregateBuilder aggregateBuilder = new AggregateBuilder(); aggregateBuilder.add(UriToDocumentTextAnnotator.getDescription()); aggregateBuilder.add(AnalysisEngineFactory.createEngineDescription(XMIReader.class, XMIReader.PARAM_XMI_DIRECTORY, options.getXMIDirectory())); int totalDocTimeRels = 0; int totalInconsistentDocTimeRels = 0; for (Iterator<JCas> casIter = new JCasIterator(reader, aggregateBuilder.createAggregate()); casIter .hasNext();) {/*from ww w .j ava2 s .co m*/ JCas jCas = casIter.next(); String text = jCas.getDocumentText(); JCas goldView = jCas.getView("GoldView"); // group events by their narrative container Multimap<Annotation, EventMention> containers = HashMultimap.create(); for (TemporalTextRelation relation : JCasUtil.select(goldView, TemporalTextRelation.class)) { if (relation.getCategory().equals("CONTAINS")) { Annotation arg1 = relation.getArg1().getArgument(); Annotation arg2 = relation.getArg2().getArgument(); if (arg2 instanceof EventMention) { EventMention event = (EventMention) arg2; containers.put(arg1, event); } } } // check each container for inconsistent DocTimeRels for (Annotation container : containers.keySet()) { Set<String> docTimeRels = Sets.newHashSet(); for (EventMention event : containers.get(container)) { docTimeRels.add(event.getEvent().getProperties().getDocTimeRel()); } totalDocTimeRels += docTimeRels.size(); boolean inconsistentDocTimeRels; if (container instanceof EventMention) { EventMention mention = ((EventMention) container); String containerDocTimeRel = mention.getEvent().getProperties().getDocTimeRel(); inconsistentDocTimeRels = false; for (String docTimeRel : docTimeRels) { if (docTimeRel.equals(containerDocTimeRel)) { continue; } if (containerDocTimeRel.equals("BEFORE/OVERLAP") && (docTimeRel.equals("BEFORE") || docTimeRel.equals("OVERLAP"))) { continue; } inconsistentDocTimeRels = true; break; } } else { if (docTimeRels.size() == 1) { inconsistentDocTimeRels = false; } else if (docTimeRels.contains("BEFORE/OVERLAP")) { inconsistentDocTimeRels = docTimeRels.size() == 1 && (docTimeRels.contains("BEFORE") || docTimeRels.contains("OVERLAP")); } else { inconsistentDocTimeRels = true; } } // if inconsistent: print events, DocTimeRels and surrounding context if (inconsistentDocTimeRels) { totalInconsistentDocTimeRels += docTimeRels.size(); List<Integer> offsets = Lists.newArrayList(); offsets.add(container.getBegin()); offsets.add(container.getEnd()); for (EventMention event : containers.get(container)) { offsets.add(event.getBegin()); offsets.add(event.getEnd()); } Collections.sort(offsets); int begin = Math.max(offsets.get(0) - windowSize, 0); int end = Math.min(offsets.get(offsets.size() - 1) + windowSize, text.length()); System.err.printf("Inconsistent DocTimeRels in %s, ...%s...\n", new File(ViewUriUtil.getURI(jCas)).getName(), text.substring(begin, end).replaceAll("([\r\n])[\r\n]+", "$1")); if (container instanceof EventMention) { System.err.printf("Container: \"%s\" (docTimeRel=%s)\n", container.getCoveredText(), ((EventMention) container).getEvent().getProperties().getDocTimeRel()); } else { System.err.printf("Container: \"%s\"\n", container.getCoveredText()); } Ordering<EventMention> byBegin = Ordering.natural() .onResultOf(new Function<EventMention, Integer>() { @Override public Integer apply(@Nullable EventMention event) { return event.getBegin(); } }); for (EventMention event : byBegin.sortedCopy(containers.get(container))) { System.err.printf("* \"%s\" (docTimeRel=%s)\n", event.getCoveredText(), event.getEvent().getProperties().getDocTimeRel()); } System.err.println(); } } } System.err.printf("Inconsistent DocTimeRels: %.1f%% (%d/%d)\n", 100.0 * totalInconsistentDocTimeRels / totalDocTimeRels, totalInconsistentDocTimeRels, totalDocTimeRels); }