Example usage for org.apache.commons.io IOUtils readLines

Introduction

In this page you can find the example usage for org.apache.commons.io IOUtils readLines.

Prototype

public static List readLines(InputStream input, String encoding) throws IOException 

Source Link

Document

Get the contents of an InputStream as a list of Strings, one entry per line, using the specified character encoding.

Usage

From source file:org.apache.streams.data.data.util.ActivitySerDeTest.java

/**
 * Tests that all example activities can be loaded into Activity beans
 * @throws Exception//w  w w  .  j av  a  2s.c om
 */
@Test
public void testActivitySerDe() throws Exception {

    InputStream testActivityFolderStream = ActivitySerDeTest.class.getClassLoader()
            .getResourceAsStream("activities");
    List<String> files = IOUtils.readLines(testActivityFolderStream, Charsets.UTF_8);

    for (String file : files) {
        LOGGER.info("File: " + file);
        LOGGER.info("Serializing: activities/" + file);
        InputStream testActivityFileStream = ActivitySerDeTest.class.getClassLoader()
                .getResourceAsStream("activities/" + file);
        Activity activity = MAPPER.readValue(testActivityFileStream, Activity.class);
        String activityString = MAPPER.writeValueAsString(activity);
        LOGGER.info("Deserialized: " + activityString);
    }
}

---

From source file:org.apache.streams.elasticsearch.test.ElasticsearchParentChildUpdaterIT.java

@BeforeClass
public void prepareTestParentChildPersistUpdater() throws Exception {

    Config reference = ConfigFactory.load();
    File conf_file = new File("target/test-classes/ElasticsearchParentChildUpdaterIT.conf");
    assertTrue(conf_file.exists());//from w w  w. j a va 2s  . com
    Config testResourceConfig = ConfigFactory.parseFileAnySyntax(conf_file,
            ConfigParseOptions.defaults().setAllowMissing(false));
    Config typesafe = testResourceConfig.withFallback(reference).resolve();
    testConfiguration = new ComponentConfigurator<>(ElasticsearchWriterConfiguration.class)
            .detectConfiguration(typesafe, "elasticsearch");
    testClient = ElasticsearchClientManager.getInstance(testConfiguration).client();

    ClusterHealthRequest clusterHealthRequest = Requests.clusterHealthRequest();
    ClusterHealthResponse clusterHealthResponse = testClient.admin().cluster().health(clusterHealthRequest)
            .actionGet();
    assertNotEquals(clusterHealthResponse.getStatus(), ClusterHealthStatus.RED);

    IndicesExistsRequest indicesExistsRequest = Requests.indicesExistsRequest(testConfiguration.getIndex());
    IndicesExistsResponse indicesExistsResponse = testClient.admin().indices().exists(indicesExistsRequest)
            .actionGet();
    assertTrue(indicesExistsResponse.isExists());

    Reflections reflections = new Reflections(
            new ConfigurationBuilder().setUrls(ClasspathHelper.forPackage("org.apache.streams.pojo.json"))
                    .setScanners(new SubTypesScanner()));
    objectTypes = reflections.getSubTypesOf(ActivityObject.class);

    InputStream testActivityFolderStream = ElasticsearchParentChildUpdaterIT.class.getClassLoader()
            .getResourceAsStream("activities");
    files = IOUtils.readLines(testActivityFolderStream, StandardCharsets.UTF_8);

}

---

From source file:org.apache.streams.elasticsearch.test.ElasticsearchParentChildWriterIT.java

@BeforeClass
public void prepareTestParentChildPersistWriter() throws Exception {

    Config reference = ConfigFactory.load();
    File conf_file = new File("target/test-classes/ElasticsearchParentChildWriterIT.conf");
    assertTrue(conf_file.exists());//from w w  w  . ja v  a  2 s .  c  o m
    Config testResourceConfig = ConfigFactory.parseFileAnySyntax(conf_file,
            ConfigParseOptions.defaults().setAllowMissing(false));
    Config typesafe = testResourceConfig.withFallback(reference).resolve();
    testConfiguration = new ComponentConfigurator<>(ElasticsearchWriterConfiguration.class)
            .detectConfiguration(typesafe, "elasticsearch");
    testClient = ElasticsearchClientManager.getInstance(testConfiguration).client();

    ClusterHealthRequest clusterHealthRequest = Requests.clusterHealthRequest();
    ClusterHealthResponse clusterHealthResponse = testClient.admin().cluster().health(clusterHealthRequest)
            .actionGet();
    assertNotEquals(clusterHealthResponse.getStatus(), ClusterHealthStatus.RED);

    IndicesExistsRequest indicesExistsRequest = Requests.indicesExistsRequest(testConfiguration.getIndex());
    IndicesExistsResponse indicesExistsResponse = testClient.admin().indices().exists(indicesExistsRequest)
            .actionGet();
    if (indicesExistsResponse.isExists()) {
        DeleteIndexRequest deleteIndexRequest = Requests.deleteIndexRequest(testConfiguration.getIndex());
        DeleteIndexResponse deleteIndexResponse = testClient.admin().indices().delete(deleteIndexRequest)
                .actionGet();
        assertTrue(deleteIndexResponse.isAcknowledged());
    }

    PutIndexTemplateRequestBuilder putTemplateRequestBuilder = testClient.admin().indices()
            .preparePutTemplate("mappings");
    URL templateURL = ElasticsearchParentChildWriterIT.class.getResource("/ActivityChildObjectParent.json");
    ObjectNode template = MAPPER.readValue(templateURL, ObjectNode.class);
    String templateSource = MAPPER.writeValueAsString(template);
    putTemplateRequestBuilder.setSource(templateSource);

    testClient.admin().indices().putTemplate(putTemplateRequestBuilder.request()).actionGet();

    Reflections reflections = new Reflections(
            new ConfigurationBuilder().setUrls(ClasspathHelper.forPackage("org.apache.streams.pojo.json"))
                    .setScanners(new SubTypesScanner()));
    objectTypes = reflections.getSubTypesOf(ActivityObject.class);

    InputStream testActivityFolderStream = ElasticsearchParentChildWriterIT.class.getClassLoader()
            .getResourceAsStream("activities");
    files = IOUtils.readLines(testActivityFolderStream, StandardCharsets.UTF_8);

}

---

From source file:org.apache.streams.elasticsearch.test.ElasticsearchPersistUpdaterIT.java

@Test
public void testPersistUpdater() throws Exception {

    IndicesExistsRequest indicesExistsRequest = Requests.indicesExistsRequest(testConfiguration.getIndex());
    IndicesExistsResponse indicesExistsResponse = testClient.admin().indices().exists(indicesExistsRequest)
            .actionGet();//ww  w . j  a  v  a2 s. c  o m
    assertTrue(indicesExistsResponse.isExists());

    SearchRequestBuilder countRequest = testClient.prepareSearch(testConfiguration.getIndex())
            .setTypes(testConfiguration.getType());
    SearchResponse countResponse = countRequest.execute().actionGet();

    long count = countResponse.getHits().getTotalHits();

    ElasticsearchPersistUpdater testPersistUpdater = new ElasticsearchPersistUpdater(testConfiguration);
    testPersistUpdater.prepare(null);

    InputStream testActivityFolderStream = ElasticsearchPersistUpdaterIT.class.getClassLoader()
            .getResourceAsStream("activities");
    List<String> files = IOUtils.readLines(testActivityFolderStream, StandardCharsets.UTF_8);

    for (String file : files) {
        LOGGER.info("File: " + file);
        InputStream testActivityFileStream = ElasticsearchPersistUpdaterIT.class.getClassLoader()
                .getResourceAsStream("activities/" + file);
        Activity activity = MAPPER.readValue(testActivityFileStream, Activity.class);
        Activity update = new Activity();
        update.setAdditionalProperty("updated", Boolean.TRUE);
        update.setAdditionalProperty("str", "str");
        update.setAdditionalProperty("long", 10L);
        update.setActor(new ActivityObject().withAdditionalProperty("updated", Boolean.TRUE)
                .withAdditionalProperty("double", 10d).withAdditionalProperty("map",
                        MAPPER.createObjectNode().set("field", MAPPER.createArrayNode().add("item"))));

        StreamsDatum datum = new StreamsDatum(update, activity.getVerb());
        testPersistUpdater.write(datum);
        LOGGER.info("Updated: " + activity.getVerb());
    }

    testPersistUpdater.cleanUp();

    SearchRequestBuilder updatedCountRequest = testClient.prepareSearch(testConfiguration.getIndex())
            .setTypes(testConfiguration.getType()).setQuery(QueryBuilders.existsQuery("updated"));
    SearchResponse updatedCount = updatedCountRequest.execute().actionGet();

    LOGGER.info("updated: {}", updatedCount.getHits().getTotalHits());

    assertEquals(count, updatedCount.getHits().getTotalHits());

    SearchRequestBuilder actorUpdatedCountRequest = testClient.prepareSearch(testConfiguration.getIndex())
            .setTypes(testConfiguration.getType()).setQuery(QueryBuilders.termQuery("actor.updated", true));
    SearchResponse actorUpdatedCount = actorUpdatedCountRequest.execute().actionGet();

    LOGGER.info("actor.updated: {}", actorUpdatedCount.getHits().getTotalHits());

    assertEquals(count, actorUpdatedCount.getHits().getTotalHits());

    SearchRequestBuilder strUpdatedCountRequest = testClient.prepareSearch(testConfiguration.getIndex())
            .setTypes(testConfiguration.getType()).setQuery(QueryBuilders.termQuery("str", "str"));
    SearchResponse strUpdatedCount = strUpdatedCountRequest.execute().actionGet();

    LOGGER.info("strupdated: {}", strUpdatedCount.getHits().getTotalHits());

    assertEquals(count, strUpdatedCount.getHits().getTotalHits());

    SearchRequestBuilder longUpdatedCountRequest = testClient.prepareSearch(testConfiguration.getIndex())
            .setTypes(testConfiguration.getType()).setQuery(QueryBuilders.rangeQuery("long").from(9).to(11));
    SearchResponse longUpdatedCount = longUpdatedCountRequest.execute().actionGet();

    LOGGER.info("longupdated: {}", longUpdatedCount.getHits().getTotalHits());

    assertEquals(count, longUpdatedCount.getHits().getTotalHits());

    SearchRequestBuilder doubleUpdatedCountRequest = testClient.prepareSearch(testConfiguration.getIndex())
            .setTypes(testConfiguration.getType()).setQuery(QueryBuilders.rangeQuery("long").from(9).to(11));
    SearchResponse doubleUpdatedCount = doubleUpdatedCountRequest.execute().actionGet();

    LOGGER.info("doubleupdated: {}", doubleUpdatedCount.getHits().getTotalHits());

    assertEquals(count, doubleUpdatedCount.getHits().getTotalHits());

    SearchRequestBuilder mapUpdatedCountRequest = testClient.prepareSearch(testConfiguration.getIndex())
            .setTypes(testConfiguration.getType()).setQuery(QueryBuilders.termQuery("actor.map.field", "item"));
    SearchResponse mapUpdatedCount = mapUpdatedCountRequest.execute().actionGet();

    LOGGER.info("mapfieldupdated: {}", mapUpdatedCount.getHits().getTotalHits());

    assertEquals(count, mapUpdatedCount.getHits().getTotalHits());

}

---

From source file:org.apache.streams.elasticsearch.test.ElasticsearchPersistWriterIT.java

@Test
public void testPersistWriter() throws Exception {

    ElasticsearchPersistWriter testPersistWriter = new ElasticsearchPersistWriter(testConfiguration);
    testPersistWriter.prepare(null);/*ww w . ja  va 2  s . co m*/

    InputStream testActivityFolderStream = ElasticsearchPersistWriterIT.class.getClassLoader()
            .getResourceAsStream("activities");
    List<String> files = IOUtils.readLines(testActivityFolderStream, StandardCharsets.UTF_8);

    for (String file : files) {
        LOGGER.info("File: " + file);
        InputStream testActivityFileStream = ElasticsearchPersistWriterIT.class.getClassLoader()
                .getResourceAsStream("activities/" + file);
        Activity activity = MAPPER.readValue(testActivityFileStream, Activity.class);
        StreamsDatum datum = new StreamsDatum(activity, activity.getVerb());
        testPersistWriter.write(datum);
        LOGGER.info("Wrote: " + activity.getVerb());
    }

    testPersistWriter.cleanUp();

    SearchRequestBuilder countRequest = testClient.prepareSearch(testConfiguration.getIndex())
            .setTypes(testConfiguration.getType());
    SearchResponse countResponse = countRequest.execute().actionGet();

    assertEquals(89, countResponse.getHits().getTotalHits());

}

---

From source file:org.apache.streams.elasticsearch.test.TestMetadataFromDocumentProcessor.java

@Test
public void testMetadataFromDocumentProcessor() throws Exception {

    MetadataFromDocumentProcessor processor = new MetadataFromDocumentProcessor();

    processor.prepare(null);//from   ww w.  j  a  v a2s  .  co m

    InputStream testActivityFolderStream = TestMetadataFromDocumentProcessor.class.getClassLoader()
            .getResourceAsStream("activities");
    List<String> files = IOUtils.readLines(testActivityFolderStream, StandardCharsets.UTF_8);

    Set<ActivityObject> objects = new HashSet<>();

    for (String file : files) {
        LOGGER.info("File: " + file);
        InputStream testActivityFileStream = TestMetadataFromDocumentProcessor.class.getClassLoader()
                .getResourceAsStream("activities/" + file);
        Activity activity = MAPPER.readValue(testActivityFileStream, Activity.class);
        activity.setId(activity.getVerb());
        activity.getAdditionalProperties().remove("$license");

        if (activity.getActor().getObjectType() != null)
            objects.add(activity.getActor());
        if (activity.getObject().getObjectType() != null)
            objects.add(activity.getObject());

        StreamsDatum datum = new StreamsDatum(activity);

        List<StreamsDatum> resultList = processor.process(datum);
        assertNotNull(resultList);
        assertEquals(1, resultList.size());

        StreamsDatum result = resultList.get(0);
        assertNotNull(result);
        assertNotNull(result.getDocument());
        assertNotNull(result.getId());
        assertNotNull(result.getMetadata());
        assertNotNull(result.getMetadata().get("id"));
        assertNotNull(result.getMetadata().get("type"));

        LOGGER.info("valid: " + activity.getVerb());
    }

    for (ActivityObject activityObject : objects) {
        LOGGER.info("Object: " + MAPPER.writeValueAsString(activityObject));

        activityObject.setId(activityObject.getObjectType());
        StreamsDatum datum = new StreamsDatum(activityObject);

        List<StreamsDatum> resultList = processor.process(datum);
        assertNotNull(resultList);
        assertEquals(1, resultList.size());

        StreamsDatum result = resultList.get(0);
        assertNotNull(result);
        assertNotNull(result.getDocument());
        assertNotNull(result.getId());
        assertNotNull(result.getMetadata());
        assertNotNull(result.getMetadata().get("id"));
        assertNotNull(result.getMetadata().get("type"));

        LOGGER.info("valid: " + activityObject.getObjectType());
    }
}

---

From source file:org.apache.streams.example.elasticsearch.test.ElasticsearchHdfsIT.java

@Before
public void prepareTest() throws Exception {

    testConfiguration = new ElasticsearchConfiguration();
    testConfiguration.setHosts(Lists.newArrayList("localhost"));
    testConfiguration.setClusterName(cluster().getClusterName());

    ElasticsearchWriterConfiguration setupWriterConfiguration = MAPPER.convertValue(testConfiguration,
            ElasticsearchWriterConfiguration.class);
    setupWriterConfiguration.setIndex("source");
    setupWriterConfiguration.setType("activity");
    setupWriterConfiguration.setBatchSize(5l);

    ElasticsearchPersistWriter setupWriter = new ElasticsearchPersistWriter(setupWriterConfiguration);
    setupWriter.prepare(null);//from w  w  w  .  j  a v a 2 s .c o m

    InputStream testActivityFolderStream = ElasticsearchHdfsIT.class.getClassLoader()
            .getResourceAsStream("activities");
    List<String> files = IOUtils.readLines(testActivityFolderStream, Charsets.UTF_8);

    for (String file : files) {
        LOGGER.info("File: " + file);
        InputStream testActivityFileStream = ElasticsearchHdfsIT.class.getClassLoader()
                .getResourceAsStream("activities/" + file);
        Activity activity = MAPPER.readValue(testActivityFileStream, Activity.class);
        StreamsDatum datum = new StreamsDatum(activity, activity.getVerb());
        setupWriter.write(datum);
        LOGGER.info("Wrote: " + activity.getVerb());
    }

    setupWriter.cleanUp();

    flushAndRefresh();

}

---

From source file:org.apache.streams.example.elasticsearch.test.ElasticsearchReindexIT.java

@Before
public void prepareTest() throws Exception {

    testConfiguration = new ElasticsearchConfiguration();
    testConfiguration.setHosts(Lists.newArrayList("localhost"));
    testConfiguration.setClusterName(cluster().getClusterName());

    ElasticsearchWriterConfiguration setupWriterConfiguration = MAPPER.convertValue(testConfiguration,
            ElasticsearchWriterConfiguration.class);
    setupWriterConfiguration.setIndex("source");
    setupWriterConfiguration.setType("activity");
    setupWriterConfiguration.setBatchSize(5l);

    ElasticsearchPersistWriter setupWriter = new ElasticsearchPersistWriter(setupWriterConfiguration);
    setupWriter.prepare(null);//from   w ww . j av  a 2s.co  m

    InputStream testActivityFolderStream = ElasticsearchReindexIT.class.getClassLoader()
            .getResourceAsStream("activities");
    List<String> files = IOUtils.readLines(testActivityFolderStream, Charsets.UTF_8);

    for (String file : files) {
        LOGGER.info("File: " + file);
        InputStream testActivityFileStream = ElasticsearchReindexIT.class.getClassLoader()
                .getResourceAsStream("activities/" + file);
        Activity activity = MAPPER.readValue(testActivityFileStream, Activity.class);
        StreamsDatum datum = new StreamsDatum(activity, activity.getVerb());
        setupWriter.write(datum);
        LOGGER.info("Wrote: " + activity.getVerb());
    }

    setupWriter.cleanUp();

    flushAndRefresh();

}

---

From source file:org.apache.streams.example.elasticsearch.test.ElasticsearchReindexParentChildIT.java

@Before
public void prepareTest() throws Exception {

    testConfiguration = new ElasticsearchConfiguration();
    testConfiguration.setHosts(Lists.newArrayList("localhost"));
    testConfiguration.setClusterName(cluster().getClusterName());

    PutIndexTemplateRequestBuilder putTemplateRequestBuilder = client().admin().indices()
            .preparePutTemplate("mappings");
    URL templateURL = ElasticsearchReindexParentChildIT.class.getResource("/ActivityChildObjectParent.json");
    ObjectNode template = MAPPER.readValue(templateURL, ObjectNode.class);
    String templateSource = MAPPER.writeValueAsString(template);
    putTemplateRequestBuilder.setSource(templateSource);

    client().admin().indices().putTemplate(putTemplateRequestBuilder.request()).actionGet();

    Reflections reflections = new Reflections(
            new ConfigurationBuilder().setUrls(ClasspathHelper.forPackage("org.apache.streams.pojo.json"))
                    .setScanners(new SubTypesScanner()));
    Set<Class<? extends ActivityObject>> objectTypes = reflections.getSubTypesOf(ActivityObject.class);

    ElasticsearchWriterConfiguration setupWriterConfiguration = MAPPER.convertValue(testConfiguration,
            ElasticsearchWriterConfiguration.class);
    setupWriterConfiguration.setIndex("source");
    setupWriterConfiguration.setBatchSize(5l);

    ElasticsearchPersistWriter setupWriter = new ElasticsearchPersistWriter(setupWriterConfiguration);
    setupWriter.prepare(null);//from w  w  w . j av a 2s.  co m

    for (Class objectType : objectTypes) {
        Object object = objectType.newInstance();
        ActivityObject activityObject = MAPPER.convertValue(object, ActivityObject.class);
        StreamsDatum datum = new StreamsDatum(activityObject, activityObject.getObjectType());
        datum.getMetadata().put("type", "object");
        setupWriter.write(datum);
    }

    InputStream testActivityFolderStream = ElasticsearchReindexIT.class.getClassLoader()
            .getResourceAsStream("activities");
    List<String> files = IOUtils.readLines(testActivityFolderStream, Charsets.UTF_8);

    for (String file : files) {
        LOGGER.info("File: " + file);
        InputStream testActivityFileStream = ElasticsearchReindexIT.class.getClassLoader()
                .getResourceAsStream("activities/" + file);
        Activity activity = MAPPER.readValue(testActivityFileStream, Activity.class);
        StreamsDatum datum = new StreamsDatum(activity, activity.getVerb());
        datum.getMetadata().put("parent", activity.getObject().getObjectType());
        datum.getMetadata().put("type", "activity");
        setupWriter.write(datum);
        LOGGER.info("Wrote: " + activity.getVerb());
    }

    setupWriter.cleanUp();

    flushAndRefresh();

}

---

From source file:org.apache.streams.example.elasticsearch.test.MongoElasticsearchSyncIT.java

@Before
public void prepareTest() throws Exception {

    syncConfiguration = MAPPER.readValue(MongoElasticsearchSyncIT.class.getResourceAsStream("/testSync.json"),
            MongoElasticsearchSyncConfiguration.class);

    syncConfiguration.getDestination().setClusterName(cluster().getClusterName());

    MongoPersistWriter setupWriter = new MongoPersistWriter(syncConfiguration.getSource());

    setupWriter.prepare(null);//from w ww  .  j av  a 2s .  c  o m

    InputStream testActivityFolderStream = MongoElasticsearchSyncIT.class.getClassLoader()
            .getResourceAsStream("activities");
    List<String> files = IOUtils.readLines(testActivityFolderStream, Charsets.UTF_8);

    for (String file : files) {
        LOGGER.info("File: " + file);
        InputStream testActivityFileStream = MongoElasticsearchSyncIT.class.getClassLoader()
                .getResourceAsStream("activities/" + file);
        Activity activity = MAPPER.readValue(testActivityFileStream, Activity.class);
        activity.getAdditionalProperties().remove("$license");
        StreamsDatum datum = new StreamsDatum(activity, activity.getVerb());
        setupWriter.write(datum);
        LOGGER.info("Wrote: " + activity.getVerb());
        srcCount++;
    }

    setupWriter.cleanUp();

}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• »