List of usage examples for java.util.concurrent ArrayBlockingQueue add
public boolean add(E e)
From source file:efen.parsewiki.WikipediaDocumentSequence.java
@Override public DocumentIterator iterator() throws IOException { final SAXParserFactory saxParserFactory = SAXParserFactory.newInstance(); saxParserFactory.setNamespaceAware(true); final MutableString nameSpaceAccumulator = new MutableString(); final ObjectOpenHashSet<MutableString> nameSpacesAccumulator = new ObjectOpenHashSet<MutableString>(); final ArrayBlockingQueue<DocumentFactory> freeFactories = new ArrayBlockingQueue<DocumentFactory>(16); for (int i = freeFactories.remainingCapacity(); i-- != 0;) freeFactories.add(this.factory.copy()); final ArrayBlockingQueue<DocumentAndFactory> readyDocumentsAndFactories = new ArrayBlockingQueue<DocumentAndFactory>( freeFactories.size());/*from ww w.j a va2 s . co m*/ final SAXParser parser; try { parser = saxParserFactory.newSAXParser(); } catch (Exception e) { throw new RuntimeException(e.getMessage(), e); } final DefaultHandler handler = new DefaultHandler() { private final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); private boolean inText; private boolean inTitle; private boolean inId; private boolean inTimestamp; private boolean inNamespaceDef; private boolean redirect; private MutableString text = new MutableString(); private MutableString title = new MutableString(); private MutableString id = new MutableString(); private MutableString timestamp = new MutableString(); private final Reference2ObjectMap<Enum<?>, Object> metadata = new Reference2ObjectOpenHashMap<Enum<?>, Object>(); { metadata.put(PropertyBasedDocumentFactory.MetadataKeys.ENCODING, "UTF-8"); metadata.put(MetadataKeys.REDIRECT, redirectAnchors); } @Override public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException { if ("page".equals(localName)) { redirect = inText = inTitle = inId = inTimestamp = false; text.length(0); title.length(0); id.length(0); timestamp.length(0); } else if ("text".equals(localName)) inText = true; else if ("title".equals(localName) && title.length() == 0) inTitle = true; // We catch only the first id/title elements. else if ("id".equals(localName) && id.length() == 0) inId = true; else if ("timestamp".equals(localName) && timestamp.length() == 0) inTimestamp = true; else if ("redirect".equals(localName)) { redirect = true; if (attributes.getValue("title") != null) // Accumulate the title of the page as virtual text of the redirect page. synchronized (redirectAnchors) { final String link = Encoder.encodeTitleToUrl(attributes.getValue("title"), true); redirectAnchors.add( new AnchorExtractor.Anchor(new MutableString(baseURL.length() + link.length()) .append(baseURL).append(link), title.copy())); } } else if ("namespace".equals(localName)) { // Found a new namespace inNamespaceDef = true; nameSpaceAccumulator.length(0); } } @Override public void endElement(String uri, String localName, String qName) throws SAXException { if ("namespace".equals(localName)) { // Collecting a namespace if (nameSpaceAccumulator.length() != 0) nameSpacesAccumulator.add(nameSpaceAccumulator.copy().toLowerCase()); return; } if ("namespaces".equals(localName)) { // All namespaces collected nameSpaces = ImmutableSet.copyOf(nameSpacesAccumulator); return; } if (!redirect) { if ("title".equals(localName)) { // Set basic metadata for the page metadata.put(PropertyBasedDocumentFactory.MetadataKeys.TITLE, title.copy()); String link = Encoder.encodeTitleToUrl(title.toString(), true); metadata.put(PropertyBasedDocumentFactory.MetadataKeys.URI, new MutableString(baseURL.length() + link.length()).append(baseURL).append(link)); inTitle = false; } else if ("id".equals(localName)) { metadata.put(MetadataKeys.ID, Long.valueOf(id.toString())); inId = false; } else if ("timestamp".equals(localName)) { try { metadata.put(MetadataKeys.LASTEDIT, dateFormat.parse(timestamp.toString())); } catch (ParseException e) { throw new RuntimeException(e.getMessage(), e); } inTimestamp = false; } else if ("text".equals(localName)) { inText = false; if (!keepNamespaced) { // Namespaces are case-insensitive and language-dependent final int pos = title.indexOf(':'); if (pos != -1 && isATrueNamespace(title.substring(0, pos))) return; } try { final MutableString html = new MutableString(); DocumentFactory freeFactory; try { freeFactory = freeFactories.take(); } catch (InterruptedException e) { throw new RuntimeException(e.getMessage(), e); } if (parseText) { if (DISAMBIGUATION.search(text) != -1) { // It's a disambiguation page. /* Roi's hack: duplicate links using the page title, so the generic name will end up as anchor text. */ final MutableString newLinks = new MutableString(); for (int start = 0, end; (start = BRACKETS_OPEN.search(text, start)) != -1; start = end) { end = start; final int endOfLink = text.indexOfAnyOf(END_OF_DISAMBIGUATION_LINK, start); // Note that we don't escape title because we are working at the Wikipedia raw text level. if (endOfLink != -1) { newLinks.append(text.array(), start, endOfLink - start).append('|') .append(title).append("]]\n"); end = endOfLink; } end++; } text.append(newLinks); } // We separate categories by OXOXO, so we don't get overflowing phrases. final MutableString category = new MutableString(); for (int start = 0, end; (start = CATEGORY_START.search(text, start)) != -1; start = end) { end = BRACKETS_CLOSED.search(text, start += CATEGORY_START.length()); if (end != -1) category.append(text.subSequence(start, end)).append(" OXOXO "); else break; } metadata.put(MetadataKeys.CATEGORY, category); // Heuristics to get the first paragraph metadata.put(MetadataKeys.FIRSTPAR, new MutableString()); String plainText = new WikiModel(imageBaseURL, linkBaseURL) .render(new PlainTextConverter(true), text.toString()); for (int start = 0; start < plainText.length(); start++) { //System.err.println("Examining " + plainText.charAt( start ) ); if (Character.isWhitespace(plainText.charAt(start))) continue; if (plainText.charAt(start) == '{') { //System.err.print( "Braces " + start + " text: \"" + plainText.subSequence( start, start + 10 ) + "\" -> " ); start = BRACES_CLOSED.search(plainText, start); //System.err.println( start + " text: \"" + plainText.subSequence( start, start + 10 ) + "\"" ); if (start == -1) break; start++; } else if (plainText.charAt(start) == '[') { start = BRACKETS_CLOSED.search(plainText, start); if (start == -1) break; start++; } else { final int end = plainText.indexOf('\n', start); if (end != -1) metadata.put(MetadataKeys.FIRSTPAR, new MutableString(plainText.substring(start, end)));//new MutableString( new WikiModel( imageBaseURL, linkBaseURL ).render( new PlainTextConverter( true ), text.substring( start, end ).toString() ) ) ); break; } } try { WikiModel wikiModel = new WikiModel(imageBaseURL, linkBaseURL); wikiModel.render(new HTMLConverter(), text.toString(), html, false, false); final Map<String, String> categories = wikiModel.getCategories(); // Put back category links in the page (they have been parsed by bliki and to not appear anymore in the HTML rendering) for (Entry<String, String> entry : categories.entrySet()) { final String key = entry.getKey(); final String value = entry.getValue().trim(); if (value.length() != 0) // There are empty such things html.append("\n<a href=\"").append(baseURL).append("Category:") .append(Encoder.encodeTitleToUrl(key, true)).append("\">") .append(HtmlEscapers.htmlEscaper().escape(key)) .append("</a>\n"); } } catch (Exception e) { LOGGER.error("Unexpected exception while parsing " + title, e); } } readyDocumentsAndFactories.put(new DocumentAndFactory( freeFactory.getDocument(IOUtils.toInputStream(html, Charsets.UTF_8), new Reference2ObjectOpenHashMap<Enum<?>, Object>(metadata)), freeFactory)); } catch (InterruptedException e) { throw new RuntimeException(e.getMessage(), e); } catch (IOException e) { throw new RuntimeException(e.getMessage(), e); } } } } @Override public void characters(char[] ch, int start, int length) throws SAXException { if (inText && parseText) text.append(ch, start, length); if (inTitle) title.append(ch, start, length); if (inId) id.append(ch, start, length); if (inTimestamp) timestamp.append(ch, start, length); if (inNamespaceDef) { nameSpaceAccumulator.append(ch, start, length); inNamespaceDef = false; // Dirty, but it works } } @Override public void ignorableWhitespace(char[] ch, int start, int length) throws SAXException { if (inText && parseText) text.append(ch, start, length); if (inTitle) title.append(ch, start, length); } }; final Thread parsingThread = new Thread() { public void run() { try { InputStream in = new FileInputStream(wikipediaXmlDump); if (bzipped) in = new BZip2CompressorInputStream(in); parser.parse( new InputSource(new InputStreamReader(new FastBufferedInputStream(in), Charsets.UTF_8)), handler); readyDocumentsAndFactories.put(END); } catch (Exception e) { throw new RuntimeException(e.getMessage(), e); } } }; parsingThread.start(); return new AbstractDocumentIterator() { private DocumentFactory lastFactory; @Override public Document nextDocument() throws IOException { try { final DocumentAndFactory documentAndFactory = readyDocumentsAndFactories.take(); if (lastFactory != null) freeFactories.put(lastFactory); if (documentAndFactory == END) return null; lastFactory = documentAndFactory.factory; return documentAndFactory.document; } catch (InterruptedException e) { throw new RuntimeException(e.getMessage(), e); } } }; }
From source file:com.ibm.crail.tools.CrailBenchmark.java
void collectionTest(int size, int loop) throws Exception { System.out.println("collectionTest, size " + size + ", loop " + loop); RingBuffer<Object> ringBuffer = new RingBuffer<Object>(10); ArrayBlockingQueue<Object> arrayQueue = new ArrayBlockingQueue<Object>(10); LinkedBlockingQueue<Object> listQueue = new LinkedBlockingQueue<Object>(); Object obj = new Object(); long start = System.currentTimeMillis(); for (int i = 0; i < loop; i++) { for (int j = 0; j < size; j++) { ringBuffer.add(obj);/*from w w w . ja v a2s . c o m*/ Object tmp = ringBuffer.peek(); tmp = ringBuffer.poll(); } } long end = System.currentTimeMillis(); double executionTime = ((double) (end - start)); System.out.println("ringbuffer, execution time [ms] " + executionTime); start = System.currentTimeMillis(); for (int i = 0; i < loop; i++) { for (int j = 0; j < size; j++) { arrayQueue.add(obj); Object tmp = arrayQueue.peek(); tmp = arrayQueue.poll(); } } end = System.currentTimeMillis(); executionTime = ((double) (end - start)); System.out.println("arrayQueue, execution time [ms] " + executionTime); start = System.currentTimeMillis(); for (int i = 0; i < loop; i++) { for (int j = 0; j < size; j++) { listQueue.add(obj); Object tmp = listQueue.peek(); tmp = listQueue.poll(); } } end = System.currentTimeMillis(); executionTime = ((double) (end - start)); System.out.println("arrayQueue, execution time [ms] " + executionTime); }
From source file:org.apache.hadoop.net.unix.TestDomainSocket.java
/** * Test a simple client/server interaction. * * @throws IOException// w ww .j a v a2 s. c om */ void testClientServer1(final Class<? extends WriteStrategy> writeStrategyClass, final Class<? extends ReadStrategy> readStrategyClass, final DomainSocket preConnectedSockets[]) throws Exception { final String TEST_PATH = new File(sockDir.getDir(), "test_sock_client_server1").getAbsolutePath(); final byte clientMsg1[] = new byte[] { 0x1, 0x2, 0x3, 0x4, 0x5, 0x6 }; final byte serverMsg1[] = new byte[] { 0x9, 0x8, 0x7, 0x6, 0x5 }; final byte clientMsg2 = 0x45; final ArrayBlockingQueue<Throwable> threadResults = new ArrayBlockingQueue<Throwable>(2); final DomainSocket serv = (preConnectedSockets != null) ? null : DomainSocket.bindAndListen(TEST_PATH); Thread serverThread = new Thread() { public void run() { // Run server DomainSocket conn = null; try { conn = preConnectedSockets != null ? preConnectedSockets[0] : serv.accept(); byte in1[] = new byte[clientMsg1.length]; ReadStrategy reader = readStrategyClass.newInstance(); reader.init(conn); reader.readFully(in1, 0, in1.length); Assert.assertTrue(Arrays.equals(clientMsg1, in1)); WriteStrategy writer = writeStrategyClass.newInstance(); writer.init(conn); writer.write(serverMsg1); InputStream connInputStream = conn.getInputStream(); int in2 = connInputStream.read(); Assert.assertEquals((int) clientMsg2, in2); conn.close(); } catch (Throwable e) { threadResults.add(e); Assert.fail(e.getMessage()); } threadResults.add(new Success()); } }; serverThread.start(); Thread clientThread = new Thread() { public void run() { try { DomainSocket client = preConnectedSockets != null ? preConnectedSockets[1] : DomainSocket.connect(TEST_PATH); WriteStrategy writer = writeStrategyClass.newInstance(); writer.init(client); writer.write(clientMsg1); ReadStrategy reader = readStrategyClass.newInstance(); reader.init(client); byte in1[] = new byte[serverMsg1.length]; reader.readFully(in1, 0, in1.length); Assert.assertTrue(Arrays.equals(serverMsg1, in1)); OutputStream clientOutputStream = client.getOutputStream(); clientOutputStream.write(clientMsg2); client.close(); } catch (Throwable e) { threadResults.add(e); } threadResults.add(new Success()); } }; clientThread.start(); for (int i = 0; i < 2; i++) { Throwable t = threadResults.take(); if (!(t instanceof Success)) { Assert.fail(t.getMessage() + ExceptionUtils.getStackTrace(t)); } } serverThread.join(120000); clientThread.join(120000); if (serv != null) { serv.close(); } }
From source file:org.apache.hadoop.net.unix.TestDomainSocket.java
/** * Test file descriptor passing./*from w ww . ja v a2 s . c o m*/ * * @throws IOException */ @Test(timeout = 180000) public void testFdPassing() throws Exception { final String TEST_PATH = new File(sockDir.getDir(), "test_sock").getAbsolutePath(); final byte clientMsg1[] = new byte[] { 0x11, 0x22, 0x33, 0x44, 0x55, 0x66 }; final byte serverMsg1[] = new byte[] { 0x31, 0x30, 0x32, 0x34, 0x31, 0x33, 0x44, 0x1, 0x1, 0x1, 0x1, 0x1 }; final ArrayBlockingQueue<Throwable> threadResults = new ArrayBlockingQueue<Throwable>(2); final DomainSocket serv = DomainSocket.bindAndListen(TEST_PATH); final PassedFile passedFiles[] = new PassedFile[] { new PassedFile(1), new PassedFile(2) }; final FileDescriptor passedFds[] = new FileDescriptor[passedFiles.length]; for (int i = 0; i < passedFiles.length; i++) { passedFds[i] = passedFiles[i].getInputStream().getFD(); } Thread serverThread = new Thread() { public void run() { // Run server DomainSocket conn = null; try { conn = serv.accept(); byte in1[] = new byte[clientMsg1.length]; InputStream connInputStream = conn.getInputStream(); IOUtils.readFully(connInputStream, in1, 0, in1.length); Assert.assertTrue(Arrays.equals(clientMsg1, in1)); DomainSocket domainConn = (DomainSocket) conn; domainConn.sendFileDescriptors(passedFds, serverMsg1, 0, serverMsg1.length); conn.close(); } catch (Throwable e) { threadResults.add(e); Assert.fail(e.getMessage()); } threadResults.add(new Success()); } }; serverThread.start(); Thread clientThread = new Thread() { public void run() { try { DomainSocket client = DomainSocket.connect(TEST_PATH); OutputStream clientOutputStream = client.getOutputStream(); InputStream clientInputStream = client.getInputStream(); clientOutputStream.write(clientMsg1); DomainSocket domainConn = (DomainSocket) client; byte in1[] = new byte[serverMsg1.length]; FileInputStream recvFis[] = new FileInputStream[passedFds.length]; int r = domainConn.recvFileInputStreams(recvFis, in1, 0, in1.length - 1); Assert.assertTrue(r > 0); IOUtils.readFully(clientInputStream, in1, r, in1.length - r); Assert.assertTrue(Arrays.equals(serverMsg1, in1)); for (int i = 0; i < passedFds.length; i++) { Assert.assertNotNull(recvFis[i]); passedFiles[i].checkInputStream(recvFis[i]); } for (FileInputStream fis : recvFis) { fis.close(); } client.close(); } catch (Throwable e) { threadResults.add(e); } threadResults.add(new Success()); } }; clientThread.start(); for (int i = 0; i < 2; i++) { Throwable t = threadResults.take(); if (!(t instanceof Success)) { Assert.fail(t.getMessage() + ExceptionUtils.getStackTrace(t)); } } serverThread.join(120000); clientThread.join(120000); serv.close(); for (PassedFile pf : passedFiles) { pf.cleanup(); } }