Example usage for org.apache.lucene.util BytesRef BytesRef

List of usage examples for org.apache.lucene.util BytesRef BytesRef

  1. HOME
  2. Java
  3. org.apache.lucene
  4. org.apache.lucene.util.*
  5. BytesRef
  6. BytesRef

Introduction

In this page you can find the example usage for org.apache.lucene.util BytesRef BytesRef.

Prototype

public BytesRef(CharSequence text)

Source Link

Document

Initialize the byte[] from the UTF8 bytes for the provided String.

Usage

From source file:io.crate.Id.java

License:Apache License

@Override
public void writeTo(StreamOutput out) throws IOException {
    out.writeVInt(values.size());//  ww  w  .  ja v a2s.co  m
    for (String value : values) {
        out.writeBytesRef(new BytesRef(value));
    }
}

From source file:io.crate.integrationtests.ColumnPolicyIntegrationTest.java

License:Apache License

@Test
public void testStrictPartitionedTableInsert() throws Exception {
    execute("create table numbers (" + "  num int, " + "  odd boolean," + "  prime boolean"
            + ") partitioned by (odd) with (column_policy='strict', number_of_replicas=0)");
    ensureYellow();// w w w  .j a  v  a2  s  . c o  m

    GetIndexTemplatesResponse response = client().admin().indices()
            .prepareGetTemplates(PartitionName.templateName(null, "numbers")).execute().actionGet();
    assertThat(response.getIndexTemplates().size(), is(1));
    IndexTemplateMetaData template = response.getIndexTemplates().get(0);
    CompressedString mappingStr = template.mappings().get(Constants.DEFAULT_MAPPING_TYPE);
    assertThat(mappingStr, is(notNullValue()));
    Tuple<XContentType, Map<String, Object>> typeAndMap = XContentHelper.convertToMap(mappingStr.uncompressed(),
            false);
    @SuppressWarnings("unchecked")
    Map<String, Object> mapping = (Map<String, Object>) typeAndMap.v2().get(Constants.DEFAULT_MAPPING_TYPE);
    assertThat(String.valueOf(mapping.get("dynamic")), is(ColumnPolicy.STRICT.value()));

    execute("insert into numbers (num, odd, prime) values (?, ?, ?)", new Object[] { 6, true, false });
    execute("refresh table numbers");

    MappingMetaData partitionMetaData = clusterService().state().metaData().indices()
            .get(new PartitionName("numbers", Arrays.asList(new BytesRef("true"))).asIndexName()).getMappings()
            .get(Constants.DEFAULT_MAPPING_TYPE);
    assertThat(String.valueOf(partitionMetaData.getSourceAsMap().get("dynamic")),
            is(ColumnPolicy.STRICT.value()));

    expectedException.expect(SQLActionException.class);
    expectedException.expectMessage("Column perfect unknown");
    execute("insert into numbers (num, odd, prime, perfect) values (?, ?, ?, ?)",
            new Object[] { 28, true, false, true });
}

From source file:io.crate.integrationtests.ColumnPolicyIntegrationTest.java

License:Apache License

@Test
public void testStrictPartitionedTableUpdate() throws Exception {
    execute("create table numbers (" + "  num int, " + "  odd boolean," + "  prime boolean"
            + ") partitioned by (odd) with (column_policy='strict', number_of_replicas=0)");
    ensureYellow();//from   w  ww. j a  v  a2 s  .c  o m

    GetIndexTemplatesResponse response = client().admin().indices()
            .prepareGetTemplates(PartitionName.templateName(null, "numbers")).execute().actionGet();
    assertThat(response.getIndexTemplates().size(), is(1));
    IndexTemplateMetaData template = response.getIndexTemplates().get(0);
    CompressedString mappingStr = template.mappings().get(Constants.DEFAULT_MAPPING_TYPE);
    assertThat(mappingStr, is(notNullValue()));
    Tuple<XContentType, Map<String, Object>> typeAndMap = XContentHelper.convertToMap(mappingStr.uncompressed(),
            false);
    @SuppressWarnings("unchecked")
    Map<String, Object> mapping = (Map<String, Object>) typeAndMap.v2().get(Constants.DEFAULT_MAPPING_TYPE);
    assertThat(String.valueOf(mapping.get("dynamic")), is(ColumnPolicy.STRICT.value()));

    execute("insert into numbers (num, odd, prime) values (?, ?, ?)", new Object[] { 6, true, false });
    execute("refresh table numbers");

    MappingMetaData partitionMetaData = clusterService().state().metaData().indices()
            .get(new PartitionName("numbers", Arrays.asList(new BytesRef("true"))).asIndexName()).getMappings()
            .get(Constants.DEFAULT_MAPPING_TYPE);
    assertThat(String.valueOf(partitionMetaData.getSourceAsMap().get("dynamic")),
            is(ColumnPolicy.STRICT.value()));

    expectedException.expect(SQLActionException.class);
    expectedException.expectMessage("Column perfect unknown");
    execute("update numbers set num=?, perfect=? where num=6", new Object[] { 28, true });
}

From source file:io.crate.integrationtests.ColumnPolicyIntegrationTest.java

License:Apache License

@Test
public void testDynamicPartitionedTable() throws Exception {
    execute("create table numbers (" + "  num int, " + "  odd boolean," + "  prime boolean"
            + ") partitioned by (odd) with (column_policy='dynamic', number_of_replicas=0)");
    ensureYellow();/*from   w  ww.ja v  a 2 s  .c o m*/

    GetIndexTemplatesResponse templateResponse = client().admin().indices()
            .prepareGetTemplates(PartitionName.templateName(null, "numbers")).execute().actionGet();
    assertThat(templateResponse.getIndexTemplates().size(), is(1));
    IndexTemplateMetaData template = templateResponse.getIndexTemplates().get(0);
    CompressedString mappingStr = template.mappings().get(Constants.DEFAULT_MAPPING_TYPE);
    assertThat(mappingStr, is(notNullValue()));
    Tuple<XContentType, Map<String, Object>> typeAndMap = XContentHelper.convertToMap(mappingStr.uncompressed(),
            false);
    @SuppressWarnings("unchecked")
    Map<String, Object> mapping = (Map<String, Object>) typeAndMap.v2().get(Constants.DEFAULT_MAPPING_TYPE);
    assertThat(String.valueOf(mapping.get("dynamic")), is("true"));

    execute("insert into numbers (num, odd, prime) values (?, ?, ?)", new Object[] { 6, true, false });
    execute("refresh table numbers");

    MappingMetaData partitionMetaData = clusterService().state().metaData().indices()
            .get(new PartitionName("numbers", Collections.singletonList(new BytesRef("true"))).asIndexName())
            .getMappings().get(Constants.DEFAULT_MAPPING_TYPE);
    assertThat(String.valueOf(partitionMetaData.getSourceAsMap().get("dynamic")), is("true"));

    execute("insert into numbers (num, odd, prime, perfect) values (?, ?, ?, ?)",
            new Object[] { 28, true, false, true });
    ensureYellow();
    execute("refresh table numbers");

    waitForMappingUpdateOnAll("numbers", "perfect");
    execute("select * from numbers order by num");
    assertThat(response.rowCount(), is(2L));
    assertThat(response.cols(), arrayContaining("num", "odd", "perfect", "prime"));
    assertThat(TestingHelpers.printedTable(response.rows()),
            is("6| true| NULL| false\n" + "28| true| true| false\n"));

    execute("update numbers set prime=true, changed='2014-10-23T10:20', author='troll' where num=28");
    assertThat(response.rowCount(), is(1L));

    waitForMappingUpdateOnAll("numbers", "changed");
}

From source file:io.crate.integrationtests.ColumnPolicyIntegrationTest.java

License:Apache License

@Test
public void testResetColumnPolicyPartitioned() throws Exception {
    execute("create table dynamic_table (" + "  id integer, " + "  score double"
            + ") partitioned by (score) with (number_of_replicas=0)");
    ensureYellow();/* w w w .j a  v  a 2 s . c  o  m*/
    execute("insert into dynamic_table (id, score) values (1, 10)");
    execute("refresh table dynamic_table");
    execute("alter table dynamic_table set (column_policy = 'strict')");
    waitNoPendingTasksOnAll();
    MappingMetaData partitionMetaData = clusterService().state().metaData().indices()
            .get(new PartitionName("dynamic_table", Arrays.asList(new BytesRef("10.0"))).asIndexName())
            .getMappings().get(Constants.DEFAULT_MAPPING_TYPE);
    assertThat(String.valueOf(partitionMetaData.getSourceAsMap().get("dynamic")),
            is(ColumnPolicy.STRICT.value()));
    execute("alter table dynamic_table reset (column_policy)");
    waitNoPendingTasksOnAll();
    partitionMetaData = clusterService().state().metaData().indices()
            .get(new PartitionName("dynamic_table", Arrays.asList(new BytesRef("10.0"))).asIndexName())
            .getMappings().get(Constants.DEFAULT_MAPPING_TYPE);
    assertThat(String.valueOf(partitionMetaData.getSourceAsMap().get("dynamic")),
            is(String.valueOf(ColumnPolicy.DYNAMIC.mappingValue())));
}

From source file:io.crate.integrationtests.ColumnPolicyIntegrationTest.java

License:Apache License

@Test
public void testAlterPartitionedTable() throws Exception {
    execute("create table dynamic_table (" + "  id integer, " + "  score double"
            + ") partitioned by (score) with (number_of_replicas=0, column_policy='strict')");
    ensureYellow();//  w w w . j  a v  a2s.  co  m
    execute("insert into dynamic_table (id, score) values (1, 10)");
    execute("refresh table dynamic_table");
    ensureYellow();
    execute("alter table dynamic_table set (column_policy= 'dynamic')");
    waitNoPendingTasksOnAll();
    // After changing the column_policy it's possible to add new columns to existing and new
    // partitions
    execute("insert into dynamic_table (id, score, comment) values (2,10,'this is a new column')");
    execute("insert into dynamic_table (id, score, new_comment) values (2,5,'this is a new column on a new partition')");
    execute("refresh table dynamic_table");
    ensureYellow();
    GetIndexTemplatesResponse response = client().admin().indices()
            .prepareGetTemplates(PartitionName.templateName(null, "dynamic_table")).execute().actionGet();
    assertThat(response.getIndexTemplates().size(), is(1));
    IndexTemplateMetaData template = response.getIndexTemplates().get(0);
    CompressedString mappingStr = template.mappings().get(Constants.DEFAULT_MAPPING_TYPE);
    assertThat(mappingStr, is(notNullValue()));
    Tuple<XContentType, Map<String, Object>> typeAndMap = XContentHelper.convertToMap(mappingStr.uncompressed(),
            false);
    @SuppressWarnings("unchecked")
    Map<String, Object> mapping = (Map<String, Object>) typeAndMap.v2().get(Constants.DEFAULT_MAPPING_TYPE);
    assertThat(String.valueOf(mapping.get("dynamic")), is(String.valueOf(ColumnPolicy.DYNAMIC.mappingValue())));

    execute("insert into dynamic_table (id, score, new_col) values (?, ?, ?)", new Object[] { 6, 3, "hello" });
    execute("refresh table dynamic_table");
    ensureYellow();

    MappingMetaData partitionMetaData = clusterService().state().metaData().indices()
            .get(new PartitionName("dynamic_table", Arrays.asList(new BytesRef("10.0"))).asIndexName())
            .getMappings().get(Constants.DEFAULT_MAPPING_TYPE);
    assertThat(String.valueOf(partitionMetaData.getSourceAsMap().get("dynamic")),
            is(String.valueOf(ColumnPolicy.DYNAMIC.mappingValue())));

    partitionMetaData = clusterService().state().metaData().indices()
            .get(new PartitionName("dynamic_table", Arrays.asList(new BytesRef("5.0"))).asIndexName())
            .getMappings().get(Constants.DEFAULT_MAPPING_TYPE);
    assertThat(String.valueOf(partitionMetaData.getSourceAsMap().get("dynamic")),
            is(String.valueOf(ColumnPolicy.DYNAMIC.mappingValue())));

    partitionMetaData = clusterService().state().metaData().indices()
            .get(new PartitionName("dynamic_table", Arrays.asList(new BytesRef("3.0"))).asIndexName())
            .getMappings().get(Constants.DEFAULT_MAPPING_TYPE);
    assertThat(String.valueOf(partitionMetaData.getSourceAsMap().get("dynamic")),
            is(String.valueOf(ColumnPolicy.DYNAMIC.mappingValue())));
}

From source file:io.crate.integrationtests.DDLIntegrationTest.java

License:Apache License

@Test
public void testAlterShardsOfPartitionedTable() throws Exception {
    execute("create table quotes (id integer, quote string, date timestamp) "
            + "partitioned by(date) clustered into 3 shards with (number_of_replicas='0-all')");
    ensureYellow();// ww w  .  java2  s.  co m

    execute("insert into quotes (id, quote, date) values (?, ?, ?), (?, ?, ?)",
            new Object[] { 1, "Don't panic", 1395874800000L, 2, "Now panic", 1395961200000L });
    execute("alter table quotes set (number_of_shards=5)");

    String templateName = PartitionName.templateName(null, "quotes");
    GetIndexTemplatesResponse templatesResponse = client().admin().indices().prepareGetTemplates(templateName)
            .execute().actionGet();
    Settings templateSettings = templatesResponse.getIndexTemplates().get(0).getSettings();
    assertThat(templateSettings.getAsInt(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 0), is(5));

    execute("insert into quotes (id, quote, date) values (?, ?, ?)",
            new Object[] { 3, "Time is a illusion. Lunchtime doubles so", 1495961200000L });
    String partition = new PartitionName("quotes", Arrays.asList(new BytesRef("1495961200000"))).asIndexName();
    GetSettingsResponse settingsResponse = client().admin().indices().prepareGetSettings(partition).execute()
            .get();
    assertThat(settingsResponse.getSetting(partition, IndexMetaData.SETTING_NUMBER_OF_SHARDS), is("5"));
}

From source file:io.crate.integrationtests.FetchOperationIntegrationTest.java

License:Apache License

@SuppressWarnings("ConstantConditions")
@Test/* w  w w  .  jav  a  2  s .  c  om*/
public void testFetchProjection() throws Exception {
    setUpCharacters();

    Plan plan = analyzeAndPlan("select id, name, substr(name, 2) from characters order by id");
    assertThat(plan, instanceOf(CollectAndMerge.class));
    CollectAndMerge qtf = (CollectAndMerge) plan;

    assertThat(((FetchProjection) qtf.localMerge().projections().get(1)).nodeReaders(), notNullValue());
    assertThat(((FetchProjection) qtf.localMerge().projections().get(1)).readerIndices(), notNullValue());

    Job job = executor.newJob(plan);
    ListenableFuture<List<TaskResult>> results = Futures.allAsList(executor.execute(job));

    final List<Object[]> resultingRows = new ArrayList<>();
    final CountDownLatch latch = new CountDownLatch(1);
    Futures.addCallback(results, new FutureCallback<List<TaskResult>>() {
        @Override
        public void onSuccess(List<TaskResult> resultList) {
            for (Row row : resultList.get(0).rows()) {
                resultingRows.add(row.materialize());
            }
            latch.countDown();
        }

        @Override
        public void onFailure(Throwable t) {
            latch.countDown();
            fail(t.getMessage());
        }
    });

    latch.await();
    assertThat(resultingRows.size(), is(2));
    assertThat(resultingRows.get(0).length, is(3));
    assertThat((Integer) resultingRows.get(0)[0], is(1));
    assertThat((BytesRef) resultingRows.get(0)[1], is(new BytesRef("Arthur")));
    assertThat((BytesRef) resultingRows.get(0)[2], is(new BytesRef("rthur")));
    assertThat((Integer) resultingRows.get(1)[0], is(2));
    assertThat((BytesRef) resultingRows.get(1)[1], is(new BytesRef("Ford")));
    assertThat((BytesRef) resultingRows.get(1)[2], is(new BytesRef("ord")));
}

From source file:io.crate.integrationtests.PartitionedTableConcurrentIntegrationTest.java

License:Apache License

/**
 * Test depends on 2 data nodes//from   ww w .ja va 2s.c o m
 */
@Test
public void testSelectWhileShardsAreRelocating() throws Throwable {
    execute("create table t (name string, p string) " + "clustered into 2 shards "
            + "partitioned by (p) with (number_of_replicas = 0)");
    ensureYellow();

    execute("insert into t (name, p) values (?, ?)",
            new Object[][] { new Object[] { "Marvin", "a" }, new Object[] { "Trillian", "a" }, });
    execute("refresh table t");
    execute("set global stats.enabled=true");

    final AtomicReference<Throwable> lastThrowable = new AtomicReference<>();
    final CountDownLatch selects = new CountDownLatch(100);

    Thread t = new Thread(new Runnable() {
        @Override
        public void run() {
            while (selects.getCount() > 0) {
                try {
                    execute("select * from t");
                } catch (Throwable t) {
                    // The failed job should have three started operations
                    SQLResponse res = execute(
                            "select id from sys.jobs_log where error is not null order by started desc limit 1");
                    if (res.rowCount() > 0) {
                        String id = (String) res.rows()[0][0];
                        res = execute(
                                "select count(*) from sys.operations_log where name=? or name = ?and job_id = ?",
                                new Object[] { "collect", "fetchContext", id });
                        if ((long) res.rows()[0][0] < 3) {
                            // set the error if there where less than three attempts
                            lastThrowable.set(t);
                        }
                    }
                } finally {
                    selects.countDown();
                }
            }
        }
    });
    t.start();

    PartitionName partitionName = new PartitionName("t", Collections.singletonList(new BytesRef("a")));
    final String indexName = partitionName.asIndexName();

    ClusterService clusterService = internalCluster().getInstance(ClusterService.class);
    DiscoveryNodes nodes = clusterService.state().nodes();
    List<String> nodeIds = new ArrayList<>(2);
    for (DiscoveryNode node : nodes) {
        if (node.isDataNode()) {
            nodeIds.add(node.getId());
        }
    }
    final Map<String, String> nodeSwap = new HashMap<>(2);
    nodeSwap.put(nodeIds.get(0), nodeIds.get(1));
    nodeSwap.put(nodeIds.get(1), nodeIds.get(0));

    final CountDownLatch relocations = new CountDownLatch(20);
    Thread relocatingThread = new Thread(new Runnable() {
        @Override
        public void run() {
            while (relocations.getCount() > 0) {
                ClusterStateResponse clusterStateResponse = admin().cluster().prepareState()
                        .setIndices(indexName).execute().actionGet();
                List<ShardRouting> shardRoutings = clusterStateResponse.getState().routingTable()
                        .allShards(indexName);

                ClusterRerouteRequestBuilder clusterRerouteRequestBuilder = admin().cluster().prepareReroute();
                int numMoves = 0;
                for (ShardRouting shardRouting : shardRoutings) {
                    if (shardRouting.currentNodeId() == null) {
                        continue;
                    }
                    if (shardRouting.state() != ShardRoutingState.STARTED) {
                        continue;
                    }
                    String toNode = nodeSwap.get(shardRouting.currentNodeId());
                    clusterRerouteRequestBuilder.add(new MoveAllocationCommand(shardRouting.getIndexName(),
                            shardRouting.shardId().id(), shardRouting.currentNodeId(), toNode));
                    numMoves++;
                }

                if (numMoves > 0) {
                    clusterRerouteRequestBuilder.execute().actionGet();
                    client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID)
                            .setWaitForNoRelocatingShards(false).setTimeout(ACCEPTABLE_RELOCATION_TIME)
                            .execute().actionGet();
                    relocations.countDown();
                }
            }
        }
    });
    relocatingThread.start();
    relocations.await(SQLTransportExecutor.REQUEST_TIMEOUT.getSeconds() + 1, TimeUnit.SECONDS);
    selects.await(SQLTransportExecutor.REQUEST_TIMEOUT.getSeconds() + 1, TimeUnit.SECONDS);

    Throwable throwable = lastThrowable.get();
    if (throwable != null) {
        throw throwable;
    }

    t.join();
    relocatingThread.join();
}

From source file:io.crate.integrationtests.PartitionedTableConcurrentIntegrationTest.java

License:Apache License

private void deletePartitionWhileInsertingData(final boolean useBulk) throws Exception {
    execute("create table parted (id int, name string) " + "partitioned by (id) "
            + "with (number_of_replicas = 0)");
    ensureYellow();//from ww  w .  j a  va2  s  . c om

    int numberOfDocs = 1000;
    final Object[][] bulkArgs = new Object[numberOfDocs][];
    for (int i = 0; i < numberOfDocs; i++) {
        bulkArgs[i] = new Object[] { i % 2, randomAsciiOfLength(10) };
    }

    // partition to delete
    final int idToDelete = 1;

    final AtomicReference<Exception> exceptionRef = new AtomicReference<>();
    final CountDownLatch insertLatch = new CountDownLatch(1);
    final String insertStmt = "insert into parted (id, name) values (?, ?)";
    Thread insertThread = new Thread(new Runnable() {
        @Override
        public void run() {
            try {
                if (useBulk) {
                    execute(insertStmt, bulkArgs);
                } else {
                    for (Object[] args : bulkArgs) {
                        execute(insertStmt, args);
                    }
                }
            } catch (Exception t) {
                exceptionRef.set(t);
            } finally {
                insertLatch.countDown();
            }
        }
    });

    final CountDownLatch deleteLatch = new CountDownLatch(1);
    final String partitionName = new PartitionName("parted",
            Collections.singletonList(new BytesRef(String.valueOf(idToDelete)))).asIndexName();
    final Object[] deleteArgs = new Object[] { idToDelete };
    Thread deleteThread = new Thread(new Runnable() {
        @Override
        public void run() {
            boolean deleted = false;
            while (!deleted) {
                try {
                    MetaData metaData = client().admin().cluster().prepareState().execute().actionGet()
                            .getState().metaData();
                    if (metaData.indices().get(partitionName) != null) {
                        execute("delete from parted where id = ?", deleteArgs);
                        deleted = true;
                    }
                } catch (Throwable t) {
                    // ignore (mostly partition index does not exists yet)
                }
            }
            deleteLatch.countDown();
        }
    });

    insertThread.start();
    deleteThread.start();
    deleteLatch.await(SQLTransportExecutor.REQUEST_TIMEOUT.getSeconds() + 1, TimeUnit.SECONDS);
    insertLatch.await(SQLTransportExecutor.REQUEST_TIMEOUT.getSeconds() + 1, TimeUnit.SECONDS);

    Exception exception = exceptionRef.get();
    if (exception != null) {
        throw exception;
    }

    insertThread.join();
    deleteThread.join();
}