List of usage examples for org.apache.lucene.util BytesRef BytesRef
public BytesRef(CharSequence text)
From source file:com.sindicetech.siren.index.codecs.siren10.TestSiren10PostingsFormat.java
License:Open Source License
@Test public void testSkipDataCheckIndex() throws IOException { // The Lucene CheckIndex was catching a problem with how skip data level // were computed on this configuration. this.setPostingsFormat(new Siren10VIntPostingsFormat(256)); final MockSirenDocument[] docs = new MockSirenDocument[1000]; for (int i = 0; i < 1000; i++) { docs[i] = doc(token("aaa", node(1)), token("bbb", node(1, 0)), token("aaa", node(2))); }//from w ww . java 2 s. com this.addDocuments(docs); final AtomicReader aReader = SlowCompositeReaderWrapper.wrap(reader); final DocsEnum docsEnum = aReader.termDocsEnum(new Term(DEFAULT_TEST_FIELD, new BytesRef("aaa"))); assertTrue(docsEnum instanceof Siren10DocsEnum); }
From source file:com.sindicetech.siren.index.codecs.siren10.TestSiren10PostingsFormat.java
License:Open Source License
@Test public void testDeltaNode() throws IOException { final MockSirenDocument[] docs = new MockSirenDocument[2048]; for (int i = 0; i < 2048; i += 2) { docs[i] = doc(token("aaa", node(1, 1)), token("aaa", node(2, 1)), token("aaa", node(2, 5))); docs[i + 1] = doc(token("aaa", node(5, 3, 1)), token("aaa", node(5, 3, 6, 3)), token("aaa", node(5, 3, 6, 5)), token("aaa", node(6))); }/*from w w w.j a v a2 s .co m*/ this.addDocuments(docs); final AtomicReader aReader = SlowCompositeReaderWrapper.wrap(reader); final DocsEnum docsEnum = aReader.termDocsEnum(new Term(DEFAULT_TEST_FIELD, new BytesRef("aaa"))); assertTrue(docsEnum instanceof Siren10DocsEnum); final Siren10DocsNodesAndPositionsEnum e = ((Siren10DocsEnum) docsEnum).getDocsNodesAndPositionsEnum(); for (int i = 0; i < 2048; i += 2) { assertTrue(e.nextDocument()); assertTrue(e.nextNode()); assertEquals(node(1, 1), e.node()); assertTrue(e.nextNode()); assertEquals(node(2, 1), e.node()); assertTrue(e.nextNode()); assertEquals(node(2, 5), e.node()); assertTrue(e.nextDocument()); assertTrue(e.nextNode()); assertEquals(node(5, 3, 1), e.node()); assertTrue(e.nextNode()); assertEquals(node(5, 3, 6, 3), e.node()); assertTrue(e.nextNode()); assertEquals(node(5, 3, 6, 5), e.node()); assertTrue(e.nextNode()); assertEquals(node(6), e.node()); } }
From source file:com.sindicetech.siren.qparser.keyword.BasicSyntaxTest.java
License:Open Source License
@Test public void testRangeQueries() throws Exception { NodeQuery q = new NodeTermRangeQuery(SirenTestCase.DEFAULT_TEST_FIELD, new BytesRef("a"), new BytesRef("b"), true, true);/*w ww .ja va 2 s .c o m*/ this._assertSirenQuery(new LuceneProxyNodeQuery(q), "[ a TO b ]"); q = new NodeTermRangeQuery(SirenTestCase.DEFAULT_TEST_FIELD, new BytesRef("a"), new BytesRef("b"), false, true); this._assertSirenQuery(new LuceneProxyNodeQuery(q), "{ a TO b ]"); q = new NodeTermRangeQuery(SirenTestCase.DEFAULT_TEST_FIELD, new BytesRef("a"), new BytesRef("b"), true, false); this._assertSirenQuery(new LuceneProxyNodeQuery(q), "[ a TO b }"); q = new NodeTermRangeQuery(SirenTestCase.DEFAULT_TEST_FIELD, new BytesRef("a"), new BytesRef("b"), false, false); this._assertSirenQuery(new LuceneProxyNodeQuery(q), "{ a TO b }"); final TwigQuery twq1 = new TwigQuery(1); twq1.addChild(q, NodeBooleanClause.Occur.MUST); // TODO parsing the output of #toString of twq1 is not possible because of GH-52 assertEquals(new LuceneProxyNodeQuery(twq1), this.parse(null, "* : { a TO b }")); final TwigQuery twq2 = new TwigQuery(1); twq2.addChild(new NodeTermRangeQuery(SirenTestCase.DEFAULT_TEST_FIELD, new BytesRef("a"), new BytesRef("b"), true, true), NodeBooleanClause.Occur.MUST); twq2.addChild(q, NodeBooleanClause.Occur.MUST); assertEquals(new LuceneProxyNodeQuery(twq2), this.parse(null, "* : [ [ a TO b ], { a TO b } ]")); }
From source file:com.sindicetech.siren.qparser.keyword.ConciseKeywordQueryParserTest.java
License:Open Source License
@Test public void testTermRangeQuery() throws Exception { ConciseKeywordQueryParser parser = new ConciseKeywordQueryParser(); parser.setAttribute("aaa"); NodeQuery q = new NodeTermRangeQuery(SirenTestCase.DEFAULT_TEST_FIELD, new BytesRef("aaa:a"), new BytesRef("aaa:b"), true, true); assertEquals(q, parser.parse("[ a TO b ]", SirenTestCase.DEFAULT_TEST_FIELD)); // with no attribute defined parser = new ConciseKeywordQueryParser(); q = new NodeTermRangeQuery(SirenTestCase.DEFAULT_TEST_FIELD, new BytesRef("a"), new BytesRef("b"), true, true);//w w w . j av a 2 s. com assertEquals(q, parser.parse("[ a TO b ]", SirenTestCase.DEFAULT_TEST_FIELD)); }
From source file:com.sindicetech.siren.search.node.NodeTermRangeQuery.java
License:Open Source License
/** * Factory that creates a new TermRangeQuery using Strings for term text. *///from ww w.j a v a2 s .c o m public static NodeTermRangeQuery newStringRange(final String field, final String lowerTerm, final String upperTerm, final boolean includeLower, final boolean includeUpper) { final BytesRef lower = lowerTerm == null ? null : new BytesRef(lowerTerm); final BytesRef upper = upperTerm == null ? null : new BytesRef(upperTerm); return new NodeTermRangeQuery(field, lower, upper, includeLower, includeUpper); }
From source file:com.sindicetech.siren.search.node.TestNodeNumericRangeQuery32.java
License:Open Source License
private void testRandomTrieAndClassicRangeQuery(final int precisionStep) throws Exception { final String field = "field" + precisionStep; int totalTermCountT = 0, totalTermCountC = 0, termCountT, termCountC; final int num = TestUtil.nextInt(random(), 10, 20); BooleanQuery.setMaxClauseCount(Integer.MAX_VALUE); for (int i = 0; i < num; i++) { int lower = (int) (random().nextDouble() * noDocs * distance) + startOffset; int upper = (int) (random().nextDouble() * noDocs * distance) + startOffset; if (lower > upper) { final int a = lower; lower = upper;// w w w. j a v a 2 s . c o m upper = a; } /* * In SIREn, the numeric type and the precision step are prepended to the * indexed numeric terms. */ final BytesRef lowerBytes = new BytesRef(NumericType.INT.toString() + precisionStep); final BytesRef upperBytes = new BytesRef(NumericType.INT.toString() + precisionStep); final BytesRef lBytes = new BytesRef(NumericUtils.BUF_SIZE_INT); final BytesRef uBytes = new BytesRef(NumericUtils.BUF_SIZE_INT); NumericUtils.intToPrefixCoded(lower, 0, lBytes); NumericUtils.intToPrefixCoded(upper, 0, uBytes); lowerBytes.append(lBytes); upperBytes.append(uBytes); // test inclusive range MultiNodeTermQuery tq = (MultiNodeTermQuery) nmqInt(field, precisionStep, lower, upper, true, true) .getQuery(); MultiNodeTermQuery cq = new NodeTermRangeQuery(field, lowerBytes, upperBytes, true, true); TopDocs tTopDocs = index.searcher.search(dq(tq), 1); TopDocs cTopDocs = index.searcher.search(dq(cq), 1); assertEquals("Returned count for NumericRangeQuery and TermRangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits); totalTermCountT += termCountT = this.countTerms(tq); totalTermCountC += termCountC = this.countTerms(cq); this.checkTermCounts(precisionStep, termCountT, termCountC); // test exclusive range tq = (MultiNodeTermQuery) nmqInt(field, precisionStep, lower, upper, false, false).getQuery(); cq = new NodeTermRangeQuery(field, lowerBytes, upperBytes, false, false); tTopDocs = index.searcher.search(dq(tq), 1); cTopDocs = index.searcher.search(dq(cq), 1); assertEquals("Returned count for NumericRangeQuery and TermRangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits); totalTermCountT += termCountT = this.countTerms(tq); totalTermCountC += termCountC = this.countTerms(cq); this.checkTermCounts(precisionStep, termCountT, termCountC); // test left exclusive range tq = (MultiNodeTermQuery) nmqInt(field, precisionStep, lower, upper, false, true).getQuery(); cq = new NodeTermRangeQuery(field, lowerBytes, upperBytes, false, true); tTopDocs = index.searcher.search(dq(tq), 1); cTopDocs = index.searcher.search(dq(cq), 1); assertEquals("Returned count for NumericRangeQuery and TermRangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits); totalTermCountT += termCountT = this.countTerms(tq); totalTermCountC += termCountC = this.countTerms(cq); this.checkTermCounts(precisionStep, termCountT, termCountC); // test right exclusive range tq = (MultiNodeTermQuery) nmqInt(field, precisionStep, lower, upper, true, false).getQuery(); cq = new NodeTermRangeQuery(field, lowerBytes, upperBytes, true, false); tTopDocs = index.searcher.search(dq(tq), 1); cTopDocs = index.searcher.search(dq(cq), 1); assertEquals("Returned count for NumericRangeQuery and TermRangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits); totalTermCountT += termCountT = this.countTerms(tq); totalTermCountC += termCountC = this.countTerms(cq); this.checkTermCounts(precisionStep, termCountT, termCountC); } this.checkTermCounts(precisionStep, totalTermCountT, totalTermCountC); if (VERBOSE && precisionStep != Integer.MAX_VALUE) { System.out.println("Average number of terms during random search on '" + field + "':"); System.out.println(" Numeric query: " + (((double) totalTermCountT) / (num * 4))); System.out.println(" Classical query: " + (((double) totalTermCountC) / (num * 4))); } }
From source file:com.sindicetech.siren.search.node.TestNodeNumericRangeQuery64.java
License:Open Source License
private void testRandomTrieAndClassicRangeQuery(final int precisionStep) throws Exception { final String field = "field" + precisionStep; int totalTermCountT = 0, totalTermCountC = 0, termCountT, termCountC; final int num = TestUtil.nextInt(random(), 10, 20); BooleanQuery.setMaxClauseCount(Integer.MAX_VALUE); for (int i = 0; i < num; i++) { long lower = (long) (random().nextDouble() * noDocs * distance) + startOffset; long upper = (long) (random().nextDouble() * noDocs * distance) + startOffset; if (lower > upper) { final long a = lower; lower = upper;//from w w w.j av a 2 s. c o m upper = a; } /* * In SIREn, the numeric type and the precision step are prepended to the * indexed numeric terms. */ final BytesRef lowerBytes = new BytesRef(NumericType.LONG.toString() + precisionStep); final BytesRef upperBytes = new BytesRef(NumericType.LONG.toString() + precisionStep); final BytesRef lBytes = new BytesRef(NumericUtils.BUF_SIZE_LONG); final BytesRef uBytes = new BytesRef(NumericUtils.BUF_SIZE_LONG); NumericUtils.longToPrefixCoded(lower, 0, lBytes); NumericUtils.longToPrefixCoded(upper, 0, uBytes); lowerBytes.append(lBytes); upperBytes.append(uBytes); // test inclusive range MultiNodeTermQuery tq = (MultiNodeTermQuery) nmqLong(field, precisionStep, lower, upper, true, true) .getQuery(); MultiNodeTermQuery cq = new NodeTermRangeQuery(field, lowerBytes, upperBytes, true, true); TopDocs tTopDocs = index.searcher.search(dq(tq), 1); TopDocs cTopDocs = index.searcher.search(dq(cq), 1); assertEquals("Returned count for NumericRangeQuery and TermRangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits); totalTermCountT += termCountT = this.countTerms(tq); totalTermCountC += termCountC = this.countTerms(cq); this.checkTermCounts(precisionStep, termCountT, termCountC); // test exclusive range tq = (MultiNodeTermQuery) nmqLong(field, precisionStep, lower, upper, false, false).getQuery(); cq = new NodeTermRangeQuery(field, lowerBytes, upperBytes, false, false); tTopDocs = index.searcher.search(dq(tq), 1); cTopDocs = index.searcher.search(dq(cq), 1); assertEquals("Returned count for NumericRangeQuery and TermRangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits); totalTermCountT += termCountT = this.countTerms(tq); totalTermCountC += termCountC = this.countTerms(cq); this.checkTermCounts(precisionStep, termCountT, termCountC); // test left exclusive range tq = (MultiNodeTermQuery) nmqLong(field, precisionStep, lower, upper, false, true).getQuery(); cq = new NodeTermRangeQuery(field, lowerBytes, upperBytes, false, true); tTopDocs = index.searcher.search(dq(tq), 1); cTopDocs = index.searcher.search(dq(cq), 1); assertEquals("Returned count for NumericRangeQuery and TermRangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits); totalTermCountT += termCountT = this.countTerms(tq); totalTermCountC += termCountC = this.countTerms(cq); this.checkTermCounts(precisionStep, termCountT, termCountC); // test right exclusive range tq = (MultiNodeTermQuery) nmqLong(field, precisionStep, lower, upper, true, false).getQuery(); cq = new NodeTermRangeQuery(field, lowerBytes, upperBytes, true, false); tTopDocs = index.searcher.search(dq(tq), 1); cTopDocs = index.searcher.search(dq(cq), 1); assertEquals("Returned count for NumericRangeQuery and TermRangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits); totalTermCountT += termCountT = this.countTerms(tq); totalTermCountC += termCountC = this.countTerms(cq); this.checkTermCounts(precisionStep, termCountT, termCountC); } this.checkTermCounts(precisionStep, totalTermCountT, totalTermCountC); if (VERBOSE && precisionStep != Integer.MAX_VALUE) { System.out.println("Average number of terms during random search on '" + field + "':"); System.out.println(" Numeric query: " + (((double) totalTermCountT) / (num * 4))); System.out.println(" Classical query: " + (((double) totalTermCountC) / (num * 4))); } }
From source file:com.stratio.cassandra.index.TokenMapperGeneric.java
License:Apache License
/** * Returns the Lucene {@link BytesRef} represented by the specified Cassandra {@link Token}. * * @param token A Cassandra {@link Token}. * @return The Lucene {@link BytesRef} represented by the specified Cassandra {@link Token}. *//* w w w . j ava 2 s. c om*/ @SuppressWarnings("unchecked") public BytesRef bytesRef(Token token) { ByteBuffer bb = factory.toByteArray(token); byte[] bytes = ByteBufferUtils.asArray(bb); return new BytesRef(bytes); }
From source file:com.stratio.cassandra.lucene.index.FSIndexTest.java
License:Apache License
@Test public void testCRUD() throws IOException, InterruptedException { FSIndex index = new FSIndex("test_index", "com.stratio.cassandra.lucene:type=LuceneIndexes", Paths.get(folder.newFolder("directory" + UUID.randomUUID()).getPath()), new StandardAnalyzer(), REFRESH_SECONDS, IndexOptions.DEFAULT_RAM_BUFFER_MB, IndexOptions.DEFAULT_MAX_MERGE_MB, IndexOptions.DEFAULT_MAX_CACHED_MB, null); Sort sort = new Sort(new SortField("field", SortField.Type.STRING)); assertEquals("Index must be empty", 0, index.getNumDocs()); Term term1 = new Term("field", "value1"); Document document1 = new Document(); document1.add(new StringField("field", "value1", Field.Store.NO)); document1.add(new SortedDocValuesField("field", new BytesRef("value1"))); index.upsert(term1, document1);// w ww . j av a2 s . c o m Term term2 = new Term("field", "value2"); Document document2 = new Document(); document2.add(new StringField("field", "value2", Field.Store.NO)); document2.add(new SortedDocValuesField("field", new BytesRef("value2"))); index.upsert(term2, document2); index.commit(); Thread.sleep(REFRESH_MILLISECONDS); assertEquals("Expected 2 documents", 2, index.getNumDocs()); Query query = new WildcardQuery(new Term("field", "value*")); Set<String> fields = Sets.newHashSet("field"); // Search // Iterator<Document> iterator; // iterator = index.search( query, sort, null, 1, fields); // assertEquals("Expected 1 document", 1, results.size()); // ScoreDoc last3 = results.values().iterator().next(); // results = index.search(searcher, query, sort, last3, 1, fields); // assertEquals("Expected 1 document", 1, results.size()); // Delete by term index.delete(term1); index.commit(); Thread.sleep(WAIT_MILLISECONDS); assertEquals("Expected 1 document", 1, index.getNumDocs()); // Delete by query index.upsert(term1, document1); index.commit(); Thread.sleep(WAIT_MILLISECONDS); assertEquals("Expected 2 documents", 2, index.getNumDocs()); index.delete(new TermQuery(term1)); Thread.sleep(WAIT_MILLISECONDS); assertEquals("Expected 1 document", 1, index.getNumDocs()); // Upsert index.upsert(term1, document1); index.upsert(term2, document2); index.upsert(term2, document2); index.commit(); Thread.sleep(WAIT_MILLISECONDS); assertEquals("Expected 2 documents", 2, index.getNumDocs()); // Truncate index.truncate(); index.commit(); Thread.sleep(WAIT_MILLISECONDS); assertEquals("Expected 0 documents", 0, index.getNumDocs()); // Delete index.delete(); // Cleanup folder.delete(); }
From source file:com.stratio.cassandra.lucene.schema.mapping.KeywordMapper.java
License:Apache License
/** {@inheritDoc} */ @Override//from w w w . j av a2s . c o m public Field sortedField(String name, String value, boolean isCollection) { BytesRef bytes = new BytesRef(value); if (isCollection) { return new SortedSetDocValuesField(name, bytes); } else { return new SortedDocValuesField(name, bytes); } }