List of usage examples for com.google.common.collect Lists newArrayList
@GwtCompatible(serializable = true) public static <E> ArrayList<E> newArrayList()
From source file:org.apache.mahout.cf.taste.example.kddcup.track2.Track2Runner.java
public static void main(String[] args) throws Exception { File dataFileDirectory = new File(args[0]); if (!dataFileDirectory.exists() || !dataFileDirectory.isDirectory()) { throw new IllegalArgumentException("Bad data file directory: " + dataFileDirectory); }/*from ww w. j a v a 2 s .c o m*/ long start = System.currentTimeMillis(); KDDCupDataModel model = new KDDCupDataModel(KDDCupDataModel.getTrainingFile(dataFileDirectory)); Track2Recommender recommender = new Track2Recommender(model, dataFileDirectory); long end = System.currentTimeMillis(); log.info("Loaded model in {}s", (end - start) / 1000); start = end; Collection<Track2Callable> callables = Lists.newArrayList(); for (Pair<PreferenceArray, long[]> tests : new DataFileIterable( KDDCupDataModel.getTestFile(dataFileDirectory))) { PreferenceArray userTest = tests.getFirst(); callables.add(new Track2Callable(recommender, userTest)); } int cores = Runtime.getRuntime().availableProcessors(); log.info("Running on {} cores", cores); ExecutorService executor = Executors.newFixedThreadPool(cores); List<Future<UserResult>> futures = executor.invokeAll(callables); executor.shutdown(); end = System.currentTimeMillis(); log.info("Ran recommendations in {}s", (end - start) / 1000); start = end; OutputStream out = new BufferedOutputStream(new FileOutputStream(new File(args[1]))); try { long lastUserID = Long.MIN_VALUE; for (Future<UserResult> future : futures) { UserResult result = future.get(); long userID = result.getUserID(); if (userID <= lastUserID) { throw new IllegalStateException(); } lastUserID = userID; out.write(result.getResultBytes()); } } finally { Closeables.close(out, false); } end = System.currentTimeMillis(); log.info("Wrote output in {}s", (end - start) / 1000); }
From source file:com.google.cloud.dataflow.sdk.coders.PrintBase64Encodings.java
public static void main(String[] argv) throws Exception { @SuppressWarnings("unchecked") Coder<Object> testCoder = (Coder<Object>) getFullyQualifiedValue(argv[0]); @SuppressWarnings("unchecked") List<Object> testValues = (List<Object>) getFullyQualifiedValue(argv[1]); List<String> base64Encodings = Lists.newArrayList(); for (Object value : testValues) { base64Encodings.add(CoderUtils.encodeToBase64(testCoder, value)); }//from w ww . j a v a2 s . c om System.out.println(String.format("\"%s\"", Joiner.on("\",\n\"").join(base64Encodings))); }
From source file:org.apache.twill.example.yarn.HelloWorld.java
public static void main(String[] args) { if (args.length < 1) { System.err.println("Arguments format: <host:port of zookeeper server>"); System.exit(1);// ww w . j av a2 s.c o m } String zkStr = args[0]; YarnConfiguration yarnConfiguration = new YarnConfiguration(); final TwillRunnerService twillRunner = new YarnTwillRunnerService(yarnConfiguration, zkStr); twillRunner.start(); String yarnClasspath = yarnConfiguration.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH, Joiner.on(",").join(YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)); List<String> applicationClassPaths = Lists.newArrayList(); Iterables.addAll(applicationClassPaths, Splitter.on(",").split(yarnClasspath)); final TwillController controller = twillRunner.prepare(new HelloWorldRunnable()) .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true))) .withApplicationClassPaths(applicationClassPaths) .withBundlerClassAcceptor(new HadoopClassExcluder()).start(); Runtime.getRuntime().addShutdownHook(new Thread() { @Override public void run() { try { Futures.getUnchecked(controller.terminate()); } finally { twillRunner.stop(); } } }); try { controller.awaitTerminated(); } catch (ExecutionException e) { e.printStackTrace(); } }
From source file:co.cask.cdap.data.stream.StreamTailer.java
public static void main(String[] args) throws Exception { if (args.length < 1) { System.out.println(String.format("Usage: java %s [streamName]", StreamTailer.class.getName())); return;/* w ww. jav a 2 s. com*/ } String streamName = args[0]; CConfiguration cConf = CConfiguration.create(); Configuration hConf = new Configuration(); Injector injector = Guice.createInjector(new ConfigModule(cConf, hConf), new DataFabricModules().getDistributedModules(), new DataSetsModules().getDistributedModules(), new LocationRuntimeModule().getDistributedModules(), new StreamAdminModules().getDistributedModules(), new NotificationFeedClientModule()); StreamAdmin streamAdmin = injector.getInstance(StreamAdmin.class); //TODO: get namespace from commandline arguments Id.Stream streamId = Id.Stream.from(Constants.DEFAULT_NAMESPACE, streamName); StreamConfig streamConfig = streamAdmin.getConfig(streamId); Location streamLocation = streamConfig.getLocation(); List<Location> eventFiles = Lists.newArrayList(); for (Location partition : streamLocation.list()) { if (!partition.isDirectory()) { continue; } for (Location file : partition.list()) { if (StreamFileType.EVENT.isMatched(file.getName())) { eventFiles.add(file); } } } int generation = StreamUtils.getGeneration(streamConfig); MultiLiveStreamFileReader reader = new MultiLiveStreamFileReader(streamConfig, ImmutableList.copyOf(Iterables.transform(eventFiles, createOffsetConverter(generation)))); List<StreamEvent> events = Lists.newArrayList(); while (reader.read(events, 10, 100, TimeUnit.MILLISECONDS) >= 0) { for (StreamEvent event : events) { System.out.println(event.getTimestamp() + " " + Charsets.UTF_8.decode(event.getBody())); } events.clear(); } reader.close(); }
From source file:RunText.java
public static void main(String[] args) throws Exception { o = new Options(); JCommander jc = null;/* w w w .ja va 2 s .co m*/ try { jc = new JCommander(o, args); jc.setProgramName("./runText"); } catch (ParameterException e) { System.out.println(e.getMessage()); String[] valid = { "-p", "path", "-d", "delimiter", "v", "value", "-i", "index" }; new JCommander(o, valid).usage(); System.exit(-1); } if (o.help) { jc.usage(); System.exit(0); } path = new Path(o.path); delim = o.delimiter.getBytes()[0]; toFind = o.value; index = o.index; numThreads = o.threads; Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(conf); TextInputFormat format = new TextInputFormat(); long len = fs.getFileStatus(path).getLen() / numThreads; List<Thread> threads = Lists.newArrayList(); for (int i = 0; i < numThreads; i++) { FileSplit split = new FileSplit(path, i * len, len, new String[] { "" }); threads.add(new Thread(new RunText(split, format))); } runningThreads = new AtomicInteger(numThreads); for (Thread t : threads) { t.start(); } int prev = 0; int current; long t1 = System.nanoTime(); long t2; while (runningThreads.get() > 0) { Thread.sleep(5000); current = totalCount.get(); t2 = System.nanoTime(); System.out.println(String.format("%f records/sec", (current - prev) * 1e9 / (t2 - t1))); t1 = t2; prev = current; } for (Thread t : threads) { t.join(); } fs.close(); }
From source file:edu.umd.cloud9.example.bigram.AnalyzeBigramRelativeFrequency.java
@SuppressWarnings({ "static-access" })
public static void main(String[] args) {
Options options = new Options();
options.addOption(OptionBuilder.withArgName("path").hasArg().withDescription("input path").create(INPUT));
CommandLine cmdline = null;//from ww w .j a v a 2 s. c o m
CommandLineParser parser = new GnuParser();
try {
cmdline = parser.parse(options, args);
} catch (ParseException exp) {
System.err.println("Error parsing command line: " + exp.getMessage());
System.exit(-1);
}
if (!cmdline.hasOption(INPUT)) {
System.out.println("args: " + Arrays.toString(args));
HelpFormatter formatter = new HelpFormatter();
formatter.setWidth(120);
formatter.printHelp(AnalyzeBigramRelativeFrequency.class.getName(), options);
ToolRunner.printGenericCommandUsage(System.out);
System.exit(-1);
}
String inputPath = cmdline.getOptionValue(INPUT);
System.out.println("input path: " + inputPath);
List<PairOfWritables<PairOfStrings, FloatWritable>> pairs = SequenceFileUtils
.readDirectory(new Path(inputPath));
List<PairOfWritables<PairOfStrings, FloatWritable>> list1 = Lists.newArrayList();
List<PairOfWritables<PairOfStrings, FloatWritable>> list2 = Lists.newArrayList();
for (PairOfWritables<PairOfStrings, FloatWritable> p : pairs) {
PairOfStrings bigram = p.getLeftElement();
if (bigram.getLeftElement().equals("light")) {
list1.add(p);
}
if (bigram.getLeftElement().equals("contain")) {
list2.add(p);
}
}
Collections.sort(list1, new Comparator<PairOfWritables<PairOfStrings, FloatWritable>>() {
public int compare(PairOfWritables<PairOfStrings, FloatWritable> e1,
PairOfWritables<PairOfStrings, FloatWritable> e2) {
if (e1.getRightElement().compareTo(e2.getRightElement()) == 0) {
return e1.getLeftElement().compareTo(e2.getLeftElement());
}
return e2.getRightElement().compareTo(e1.getRightElement());
}
});
Iterator<PairOfWritables<PairOfStrings, FloatWritable>> iter1 = Iterators.limit(list1.iterator(), 10);
while (iter1.hasNext()) {
PairOfWritables<PairOfStrings, FloatWritable> p = iter1.next();
PairOfStrings bigram = p.getLeftElement();
System.out.println(bigram + "\t" + p.getRightElement());
}
Collections.sort(list2, new Comparator<PairOfWritables<PairOfStrings, FloatWritable>>() {
public int compare(PairOfWritables<PairOfStrings, FloatWritable> e1,
PairOfWritables<PairOfStrings, FloatWritable> e2) {
if (e1.getRightElement().compareTo(e2.getRightElement()) == 0) {
return e1.getLeftElement().compareTo(e2.getLeftElement());
}
return e2.getRightElement().compareTo(e1.getRightElement());
}
});
Iterator<PairOfWritables<PairOfStrings, FloatWritable>> iter2 = Iterators.limit(list2.iterator(), 10);
while (iter2.hasNext()) {
PairOfWritables<PairOfStrings, FloatWritable> p = iter2.next();
PairOfStrings bigram = p.getLeftElement();
System.out.println(bigram + "\t" + p.getRightElement());
}
}
From source file:edu.umd.jyothivinjumur.AnalyzeBigramRelativeFrequencyJson.java
@SuppressWarnings({ "static-access" })
public static void main(String[] args) {
Options options = new Options();
options.addOption(OptionBuilder.withArgName("path").hasArg().withDescription("input path").create(INPUT));
CommandLine cmdline = null;//from w ww. j a v a2 s.c o m
CommandLineParser parser = new GnuParser();
try {
cmdline = parser.parse(options, args);
} catch (ParseException exp) {
System.err.println("Error parsing command line: " + exp.getMessage());
System.exit(-1);
}
if (!cmdline.hasOption(INPUT)) {
System.out.println("args: " + Arrays.toString(args));
HelpFormatter formatter = new HelpFormatter();
formatter.setWidth(120);
formatter.printHelp(AnalyzeBigramRelativeFrequencyJson.class.getName(), options);
ToolRunner.printGenericCommandUsage(System.out);
System.exit(-1);
}
String inputPath = cmdline.getOptionValue(INPUT);
System.out.println("input path: " + inputPath);
List<PairOfWritables<BigramRelativeFrequencyJson.MyTuple, FloatWritable>> pairs = SequenceFileUtils
.readDirectory(new Path(inputPath));
List<PairOfWritables<BigramRelativeFrequencyJson.MyTuple, FloatWritable>> list1 = Lists.newArrayList();
List<PairOfWritables<BigramRelativeFrequencyJson.MyTuple, FloatWritable>> list2 = Lists.newArrayList();
for (PairOfWritables<BigramRelativeFrequencyJson.MyTuple, FloatWritable> p : pairs) {
BigramRelativeFrequencyJson.MyTuple bigram = p.getLeftElement();
if (bigram.getJsonObject().get("Left").getAsString().equals("light")) {
list1.add(p);
}
if (bigram.getJsonObject().get("Left").getAsString().equals("contain")) {
list2.add(p);
}
}
Collections.sort(list1,
new Comparator<PairOfWritables<BigramRelativeFrequencyJson.MyTuple, FloatWritable>>() {
public int compare(PairOfWritables<BigramRelativeFrequencyJson.MyTuple, FloatWritable> e1,
PairOfWritables<BigramRelativeFrequencyJson.MyTuple, FloatWritable> e2) {
if (e1.getRightElement().compareTo(e2.getRightElement()) == 0) {
return e1.getLeftElement().compareTo(e2.getLeftElement());
}
return e2.getRightElement().compareTo(e1.getRightElement());
}
});
Iterator<PairOfWritables<BigramRelativeFrequencyJson.MyTuple, FloatWritable>> iter1 = Iterators
.limit(list1.iterator(), 10);
while (iter1.hasNext()) {
PairOfWritables<BigramRelativeFrequencyJson.MyTuple, FloatWritable> p = iter1.next();
BigramRelativeFrequencyJson.MyTuple bigram = p.getLeftElement();
System.out.println(bigram + "\t" + p.getRightElement());
}
Collections.sort(list2,
new Comparator<PairOfWritables<BigramRelativeFrequencyJson.MyTuple, FloatWritable>>() {
public int compare(PairOfWritables<BigramRelativeFrequencyJson.MyTuple, FloatWritable> e1,
PairOfWritables<BigramRelativeFrequencyJson.MyTuple, FloatWritable> e2) {
if (e1.getRightElement().compareTo(e2.getRightElement()) == 0) {
return e1.getLeftElement().compareTo(e2.getLeftElement());
}
return e2.getRightElement().compareTo(e1.getRightElement());
}
});
Iterator<PairOfWritables<BigramRelativeFrequencyJson.MyTuple, FloatWritable>> iter2 = Iterators
.limit(list2.iterator(), 10);
while (iter2.hasNext()) {
PairOfWritables<BigramRelativeFrequencyJson.MyTuple, FloatWritable> p = iter2.next();
BigramRelativeFrequencyJson.MyTuple bigram = p.getLeftElement();
System.out.println(bigram + "\t" + p.getRightElement());
}
}
From source file:com.google.NithPoints.examples.PrintPeers.java
public static void main(String[] args) throws Exception { BriefLogFormatter.init();//from ww w . ja va2 s . c o m System.out.println("=== DNS ==="); printDNS(); System.out.println("=== Version/chain heights ==="); ArrayList<InetAddress> addrs = new ArrayList<InetAddress>(); for (InetSocketAddress peer : dnsPeers) addrs.add(peer.getAddress()); System.out.println("Scanning " + addrs.size() + " peers:"); final NetworkParameters params = NetworkParameters.prodNet(); final Object lock = new Object(); final long[] bestHeight = new long[1]; List<ListenableFuture<TCPNetworkConnection>> futures = Lists.newArrayList(); for (final InetAddress addr : addrs) { final ListenableFuture<TCPNetworkConnection> future = TCPNetworkConnection.connectTo(params, new InetSocketAddress(addr, params.port), 1000 /* timeout */); futures.add(future); // Once the connection has completed version handshaking ... Futures.addCallback(future, new FutureCallback<TCPNetworkConnection>() { public void onSuccess(TCPNetworkConnection conn) { // Check the chain height it claims to have. VersionMessage ver = conn.getVersionMessage(); long nodeHeight = ver.bestHeight; synchronized (lock) { long diff = bestHeight[0] - nodeHeight; if (diff > 0) { System.out.println("Node is behind by " + diff + " blocks: " + addr); } else if (diff == 0) { System.out.println("Node " + addr + " has " + nodeHeight + " blocks"); bestHeight[0] = nodeHeight; } else if (diff < 0) { System.out.println("Node is ahead by " + Math.abs(diff) + " blocks: " + addr); bestHeight[0] = nodeHeight; } } conn.close(); } public void onFailure(Throwable throwable) { System.out.println("Failed to talk to " + addr + ": " + throwable.getMessage()); } }); } // Wait for every tried connection to finish. Futures.successfulAsList(futures).get(); }
From source file:org.apache.ctakes.temporal.data.analysis.CompareFeatureStructures.java
public static void main(String[] args) throws Exception { Options options = CliFactory.parseArguments(Options.class, args); List<Class<?>> annotationClasses = Lists.newArrayList(); for (String annotationClassName : options.getAnnotationClassNames()) { annotationClasses.add(Class.forName(annotationClassName)); }/*w w w . jav a 2 s . c om*/ MyersDiff<String> stringDiff = new MyersDiff<String>(); MyersDiff<FeatureStructure> fsDiff = new MyersDiff<FeatureStructure>(new FeatureStructureEqualizer()); File originalDir = options.getDirectory1(); File revisedDir = options.getDirectory2(); Patch<String> dirPatch = stringDiff.diff(originalDir.list(), revisedDir.list()); if (!dirPatch.getDeltas().isEmpty()) { log("--- %s files\n", originalDir); log("+++ %s files\n", revisedDir); log(dirPatch); } else { for (String fileName : originalDir.list()) { File originalFile = new File(originalDir, fileName); File revisedFile = new File(revisedDir, fileName); JCas originalJCas = readXMI(originalFile); JCas revisedJCas = readXMI(revisedFile); List<String> originalViews = getViewNames(originalJCas); List<String> revisedViews = getViewNames(revisedJCas); Patch<String> viewsPatch = stringDiff.diff(originalViews, revisedViews); if (!viewsPatch.getDeltas().isEmpty()) { log("--- %s views\n", originalFile); log("+++ %s views\n", revisedFile); log(viewsPatch); } else { for (String viewName : originalViews) { JCas originalView = originalJCas.getView(viewName); JCas revisedView = revisedJCas.getView(viewName); List<FeatureStructure> originalFSes = toFeatureStructures(originalView, annotationClasses); List<FeatureStructure> revisedFSes = toFeatureStructures(revisedView, annotationClasses); Patch<FeatureStructure> fsPatch = fsDiff.diff(originalFSes, revisedFSes); if (!fsPatch.getDeltas().isEmpty()) { log("--- %s view %s\n", originalFile, viewName); log("+++ %s view %s\n", revisedFile, viewName); for (Delta<FeatureStructure> fsDelta : fsPatch.getDeltas()) { logHeader(fsDelta); switch (fsDelta.getType()) { case DELETE: case INSERT: log(fsDelta); break; case CHANGE: List<String> originalLines = toLines(fsDelta.getOriginal().getLines()); List<String> revisedLines = toLines(fsDelta.getRevised().getLines()); Patch<String> linesPatch = stringDiff.diff(originalLines, revisedLines); ListMultimap<Integer, String> deletes = ArrayListMultimap.create(); ListMultimap<Integer, String> inserts = ArrayListMultimap.create(); Set<Integer> skips = Sets.newHashSet(); for (Delta<String> linesDelta : linesPatch.getDeltas()) { Chunk<String> originalChunk = linesDelta.getOriginal(); Chunk<String> revisedChunk = linesDelta.getRevised(); int start = originalChunk.getPosition(); deletes.putAll(start, originalChunk.getLines()); inserts.putAll(start, revisedChunk.getLines()); for (int i = start; i < start + originalChunk.size(); ++i) { skips.add(i); } } for (int i = 0; i < originalLines.size(); ++i) { if (!skips.contains(i)) { log(" %s\n", originalLines.get(i)); } for (String line : deletes.get(i)) { log("-%s\n", line); } for (String line : inserts.get(i)) { log("+%s\n", line); } } break; } } } } } } } }
From source file:com.sina.dip.twill.HelloWorldResourcesSpecification.java
public static void main(String[] args) { String zkStr = "localhost:2181"; YarnConfiguration yarnConfiguration = new YarnConfiguration(); final TwillRunnerService twillRunner = new YarnTwillRunnerService(yarnConfiguration, zkStr); twillRunner.start();/*from w w w . j a va 2 s. c o m*/ String yarnClasspath = yarnConfiguration.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH, "/usr/lib/hadoop/*,/usr/lib/hadoop-0.20-mapreduce/*,/usr/lib/hadoop-hdfs/*,/usr/lib/hadoop-mapreduce/*,/usr/lib/hadoop-yarn/*"); List<String> applicationClassPaths = Lists.newArrayList(); Iterables.addAll(applicationClassPaths, Splitter.on(",").split(yarnClasspath)); final TwillController controller = twillRunner .prepare(new HelloWorldRunnable(), ResourceSpecification.Builder.with().setVirtualCores(1).setMemory(2, SizeUnit.GIGA) .setInstances(3).build()) .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true))) .withApplicationClassPaths(applicationClassPaths) .withBundlerClassAcceptor(new HadoopClassExcluder()).start(); Runtime.getRuntime().addShutdownHook(new Thread() { @Override public void run() { try { Futures.getUnchecked(controller.terminate()); } finally { twillRunner.stop(); } } }); try { controller.awaitTerminated(); } catch (ExecutionException e) { e.printStackTrace(); } }