Example usage for org.apache.commons.configuration BaseConfiguration BaseConfiguration

Introduction

In this page you can find the example usage for org.apache.commons.configuration BaseConfiguration BaseConfiguration.

Prototype

BaseConfiguration

Source Link

Usage

From source file:org.apache.tinkerpop.gremlin.spark.structure.io.PersistedInputOutputRDDIntegrateTest.java

@Test
public void testBulkLoaderVertexProgramChainWithInputOutputHelperMapping() throws Exception {
    Spark.create("local[4]");

    final String rddName = TestHelper.makeTestDataDirectory(PersistedInputOutputRDDIntegrateTest.class,
            UUID.randomUUID().toString());
    final Configuration readConfiguration = super.getBaseConfiguration();
    readConfiguration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER,
            GryoInputFormat.class.getCanonicalName());
    readConfiguration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION,
            SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern.kryo"));
    readConfiguration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER,
            PersistedOutputRDD.class.getCanonicalName());
    readConfiguration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, rddName);
    readConfiguration.setProperty(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, true);
    Graph pageRankGraph = GraphFactory.open(readConfiguration);
    ///////////////
    final Configuration writeConfiguration = new BaseConfiguration();
    writeConfiguration.setProperty(Graph.GRAPH, TinkerGraph.class.getCanonicalName());
    writeConfiguration.setProperty(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_FORMAT, "gryo");
    writeConfiguration.setProperty(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_LOCATION,
            TestHelper.makeTestDataDirectory(PersistedInputOutputRDDIntegrateTest.class)
                    + "testBulkLoaderVertexProgramChainWithInputOutputHelperMapping.kryo");
    final Graph bulkLoaderGraph = pageRankGraph.compute(SparkGraphComputer.class)
            .persist(GraphComputer.Persist.EDGES).program(PageRankVertexProgram.build().create(pageRankGraph))
            .submit().get().graph();// w ww  .  j a va 2s  .  c o  m
    bulkLoaderGraph.compute(SparkGraphComputer.class).persist(GraphComputer.Persist.NOTHING).workers(1)
            .program(BulkLoaderVertexProgram.build().userSuppliedIds(true).writeGraph(writeConfiguration)
                    .create(bulkLoaderGraph))
            .submit().get();
    ////
    Spark.create(readConfiguration);
    assertTrue(Spark.hasRDD(Constants.getGraphLocation(rddName)));
    assertEquals(1, Spark.getContext().getPersistentRDDs().size());
    ////
    final Graph graph = TinkerGraph.open();
    final GraphTraversalSource g = graph.traversal();
    graph.io(IoCore.gryo())
            .readGraph(TestHelper.makeTestDataDirectory(PersistedInputOutputRDDIntegrateTest.class)
                    + "testBulkLoaderVertexProgramChainWithInputOutputHelperMapping.kryo");
    assertEquals(6l, g.V().count().next().longValue());
    assertEquals(6l, g.E().count().next().longValue());
    assertEquals("marko", g.V().has("name", "marko").values("name").next());
    assertEquals(6l, g.V().values(PageRankVertexProgram.PAGE_RANK).count().next().longValue());
    ////
    Spark.close();
}

---

From source file:org.apache.tinkerpop.gremlin.spark.structure.io.PersistedInputOutputRDDTest.java

@Test
public void testBulkLoaderVertexProgramChain() throws Exception {
    Spark.create("local[4]");
    final String rddName = TestHelper.makeTestDataDirectory(PersistedInputOutputRDDTest.class,
            UUID.randomUUID().toString());
    final Configuration readConfiguration = super.getBaseConfiguration();
    readConfiguration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT,
            GryoInputFormat.class.getCanonicalName());
    readConfiguration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION,
            SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern.kryo"));
    readConfiguration.setProperty(Constants.GREMLIN_SPARK_GRAPH_OUTPUT_RDD,
            PersistedOutputRDD.class.getCanonicalName());
    readConfiguration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, rddName);
    readConfiguration.setProperty(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, true);
    Graph pageRankGraph = GraphFactory.open(readConfiguration);
    ///////////////
    final Configuration writeConfiguration = new BaseConfiguration();
    writeConfiguration.setProperty(Graph.GRAPH, TinkerGraph.class.getCanonicalName());
    writeConfiguration.setProperty(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_FORMAT, "gryo");
    writeConfiguration.setProperty(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_LOCATION,
            TestHelper.makeTestDataDirectory(PersistedInputOutputRDDTest.class)
                    + "testBulkLoaderVertexProgramChain.kryo");
    final Graph bulkLoaderGraph = pageRankGraph.compute(SparkGraphComputer.class)
            .persist(GraphComputer.Persist.VERTEX_PROPERTIES)
            .program(PageRankVertexProgram.build().create(pageRankGraph)).submit().get().graph();
    bulkLoaderGraph.compute(SparkGraphComputer.class).persist(GraphComputer.Persist.NOTHING).workers(1)
            .configure(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD, PersistedInputRDD.class.getCanonicalName())
            .configure(Constants.GREMLIN_HADOOP_INPUT_LOCATION, rddName)
            .configure(Constants.GREMLIN_SPARK_GRAPH_OUTPUT_RDD, null)
            .configure(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, null).program(BulkLoaderVertexProgram.build()
                    .userSuppliedIds(true).writeGraph(writeConfiguration).create(bulkLoaderGraph))
            .submit().get();//from w w w . j a v  a  2  s  .c o m
    ////
    assertTrue(Spark.hasRDD(Constants.getGraphLocation(rddName)));
    assertEquals(1, Spark.getContext().getPersistentRDDs().size());
    ////
    final Graph graph = TinkerGraph.open();
    final GraphTraversalSource g = graph.traversal();
    graph.io(IoCore.gryo()).readGraph(TestHelper.makeTestDataDirectory(PersistedInputOutputRDDTest.class)
            + "testBulkLoaderVertexProgramChain.kryo");
    assertEquals(6l, g.V().count().next().longValue());
    assertEquals(0l, g.E().count().next().longValue());
    assertEquals("marko", g.V().has("name", "marko").values("name").next());
    assertEquals(6l, g.V().values(PageRankVertexProgram.PAGE_RANK).count().next().longValue());
    ////
    Spark.close();
}

---

From source file:org.apache.tinkerpop.gremlin.spark.structure.io.PersistedInputOutputRDDTest.java

@Test
public void testBulkLoaderVertexProgramChainWithInputOutputHelperMapping() throws Exception {
    Spark.create("local[4]");

    final String rddName = TestHelper.makeTestDataDirectory(PersistedInputOutputRDDTest.class,
            UUID.randomUUID().toString());
    final Configuration readConfiguration = super.getBaseConfiguration();
    readConfiguration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT,
            GryoInputFormat.class.getCanonicalName());
    readConfiguration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION,
            SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern.kryo"));
    readConfiguration.setProperty(Constants.GREMLIN_SPARK_GRAPH_OUTPUT_RDD,
            PersistedOutputRDD.class.getCanonicalName());
    readConfiguration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, rddName);
    readConfiguration.setProperty(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, true);
    Graph pageRankGraph = GraphFactory.open(readConfiguration);
    ///////////////
    final Configuration writeConfiguration = new BaseConfiguration();
    writeConfiguration.setProperty(Graph.GRAPH, TinkerGraph.class.getCanonicalName());
    writeConfiguration.setProperty(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_FORMAT, "gryo");
    writeConfiguration.setProperty(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_LOCATION,
            TestHelper.makeTestDataDirectory(PersistedInputOutputRDDTest.class)
                    + "testBulkLoaderVertexProgramChainWithInputOutputHelperMapping.kryo");
    final Graph bulkLoaderGraph = pageRankGraph.compute(SparkGraphComputer.class)
            .persist(GraphComputer.Persist.EDGES).program(PageRankVertexProgram.build().create(pageRankGraph))
            .submit().get().graph();/*  w  w  w  .  jav  a  2s  .c o m*/
    bulkLoaderGraph.compute(SparkGraphComputer.class).persist(GraphComputer.Persist.NOTHING).workers(1)
            .program(BulkLoaderVertexProgram.build().userSuppliedIds(true).writeGraph(writeConfiguration)
                    .create(bulkLoaderGraph))
            .submit().get();
    ////
    Spark.create(readConfiguration);
    assertTrue(Spark.hasRDD(Constants.getGraphLocation(rddName)));
    assertEquals(1, Spark.getContext().getPersistentRDDs().size());
    ////
    final Graph graph = TinkerGraph.open();
    final GraphTraversalSource g = graph.traversal();
    graph.io(IoCore.gryo()).readGraph(TestHelper.makeTestDataDirectory(PersistedInputOutputRDDTest.class)
            + "testBulkLoaderVertexProgramChainWithInputOutputHelperMapping.kryo");
    assertEquals(6l, g.V().count().next().longValue());
    assertEquals(6l, g.E().count().next().longValue());
    assertEquals("marko", g.V().has("name", "marko").values("name").next());
    assertEquals(6l, g.V().values(PageRankVertexProgram.PAGE_RANK).count().next().longValue());
    ////
    Spark.close();
}

---

From source file:org.apache.tinkerpop.gremlin.spark.structure.io.SparkContextStorage.java

@Override
public Iterator<Vertex> head(final String location, final Class parserClass, final int totalLines) {
    final Configuration configuration = new BaseConfiguration();
    configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, location);
    configuration.setProperty(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD, parserClass.getCanonicalName());
    configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, parserClass.getCanonicalName());
    try {/*  www .ja  v a 2  s  .c om*/
        if (InputRDD.class.isAssignableFrom(parserClass)) {
            return IteratorUtils.map(((InputRDD) parserClass.getConstructor().newInstance())
                    .readGraphRDD(configuration, new JavaSparkContext(Spark.getContext())).take(totalLines)
                    .iterator(), tuple -> tuple._2().get());
        } else if (InputFormat.class.isAssignableFrom(parserClass)) {
            return IteratorUtils.map(
                    new InputFormatRDD().readGraphRDD(configuration, new JavaSparkContext(Spark.getContext()))
                            .take(totalLines).iterator(),
                    tuple -> tuple._2().get());
        }
    } catch (final Exception e) {
        throw new IllegalArgumentException(e.getMessage(), e);
    }
    throw new IllegalArgumentException(
            "The provided parserClass must be an " + InputFormat.class.getCanonicalName() + " or an "
                    + InputRDD.class.getCanonicalName() + ": " + parserClass.getCanonicalName());
}

---

From source file:org.apache.tinkerpop.gremlin.spark.structure.io.SparkContextStorage.java

@Override
public <K, V> Iterator<KeyValue<K, V>> head(final String location, final String memoryKey,
        final Class parserClass, final int totalLines) {
    final Configuration configuration = new BaseConfiguration();
    configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, location);
    configuration.setProperty(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD, parserClass.getCanonicalName());
    configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, parserClass.getCanonicalName());
    try {/* w w  w  .  j  ava2s  .c  o  m*/
        if (InputRDD.class.isAssignableFrom(parserClass)) {
            return IteratorUtils.map(((InputRDD) parserClass.getConstructor().newInstance())
                    .readMemoryRDD(configuration, memoryKey, new JavaSparkContext(Spark.getContext()))
                    .take(totalLines).iterator(), tuple -> new KeyValue(tuple._1(), tuple._2()));
        } else if (InputFormat.class.isAssignableFrom(parserClass)) {
            return IteratorUtils.map(new InputFormatRDD()
                    .readMemoryRDD(configuration, memoryKey, new JavaSparkContext(Spark.getContext()))
                    .take(totalLines).iterator(), tuple -> new KeyValue(tuple._1(), tuple._2()));
        }
    } catch (final Exception e) {
        throw new IllegalArgumentException(e.getMessage(), e);
    }
    throw new IllegalArgumentException(
            "The provided parserClass must be an " + InputFormat.class.getCanonicalName() + " or an "
                    + InputRDD.class.getCanonicalName() + ": " + parserClass.getCanonicalName());
}

---

From source file:org.apache.tinkerpop.gremlin.spark.structure.SparkTest.java

@Test
public void testSparkRDDPersistence() throws Exception {
    final String root = TestHelper.makeTestDataDirectory(SparkTest.class, "testSparkRDDPersistence");
    final String prefix = root + File.separator + "graphRDD-";
    final Configuration configuration = new BaseConfiguration();
    configuration.setProperty("spark.master", "local[4]");
    Spark.create(configuration);//  www.java  2 s .c  o m

    configuration.setProperty("spark.serializer", GryoSerializer.class.getCanonicalName());
    configuration.setProperty(Graph.GRAPH, HadoopGraph.class.getName());
    configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION,
            SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern.kryo"));
    configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT,
            GryoInputFormat.class.getCanonicalName());
    configuration.setProperty(Constants.GREMLIN_SPARK_GRAPH_OUTPUT_RDD,
            PersistedOutputRDD.class.getCanonicalName());
    configuration.setProperty(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, false);
    configuration.setProperty(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, true);

    for (int i = 0; i < 10; i++) {
        final String graphRDDName = Constants.getGraphLocation(prefix + i);
        assertEquals(i, Spark.getRDDs().size());
        configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, prefix + i);
        Graph graph = GraphFactory.open(configuration);
        graph.compute(SparkGraphComputer.class).persist(GraphComputer.Persist.VERTEX_PROPERTIES)
                .program(PageRankVertexProgram.build().iterations(1).create(graph)).submit().get();
        assertNotNull(Spark.getRDD(graphRDDName));
        assertEquals(i + 1, Spark.getRDDs().size());
    }

    for (int i = 9; i >= 0; i--) {
        final String graphRDDName = Constants.getGraphLocation(prefix + i);
        assertEquals(i + 1, getPersistedRDDSize());
        assertEquals(i + 1, Spark.getRDDs().size());
        assertTrue(hasPersistedRDD(graphRDDName));
        Spark.removeRDD(graphRDDName);
        assertFalse(hasPersistedRDD(graphRDDName));
    }

    assertEquals(0, getPersistedRDDSize());
    assertEquals(0, Spark.getRDDs().size());
    Spark.close();
}

---

From source file:org.apache.tinkerpop.gremlin.structure.io.gryo.GryoPoolTest.java

@Test
public void shouldDoWithReaderWriterMethods() throws Exception {
    final Configuration conf = new BaseConfiguration();
    final GryoPool pool = GryoPool.build()
            .ioRegistries(conf.getList(GryoPool.CONFIG_IO_REGISTRY, Collections.emptyList())).create();
    try (final ByteArrayOutputStream os = new ByteArrayOutputStream()) {
        pool.doWithWriter(writer -> writer.writeObject(os, 1));
        os.flush();//from  ww  w .j a  v  a2 s.  co m
        try (final ByteArrayInputStream is = new ByteArrayInputStream(os.toByteArray())) {
            assertEquals(1,
                    pool.<Integer>doWithReader(
                            FunctionUtils.wrapFunction(reader -> reader.readObject(is, Integer.class)))
                            .intValue());
        }
    }
    assertReaderWriter(pool.takeWriter(), pool.takeReader(), 1, Integer.class);
}

---

From source file:org.apache.tinkerpop.gremlin.structure.io.gryo.GryoPoolTest.java

@Test
public void shouldConfigPoolOnConstructionWithDefaults() throws Exception {
    final Configuration conf = new BaseConfiguration();
    final GryoPool pool = GryoPool.build()
            .ioRegistries(conf.getList(GryoPool.CONFIG_IO_REGISTRY, Collections.emptyList())).create();
    assertReaderWriter(pool.takeWriter(), pool.takeReader(), 1, Integer.class);
}

---

From source file:org.apache.tinkerpop.gremlin.structure.io.gryo.GryoPoolTest.java

@Test
public void shouldConfigPoolOnConstructionWithPoolSizeOneAndNoIoRegistry() throws Exception {
    final Configuration conf = new BaseConfiguration();
    final GryoPool pool = GryoPool.build().poolSize(1)
            .ioRegistries(conf.getList(GryoPool.CONFIG_IO_REGISTRY, Collections.emptyList())).create();
    final GryoReader reader = pool.takeReader();
    final GryoWriter writer = pool.takeWriter();

    pool.offerReader(reader);/*from   w w w .j av a2  s  .c o m*/
    pool.offerWriter(writer);

    for (int ix = 0; ix < 100; ix++) {
        final GryoReader r = pool.takeReader();
        final GryoWriter w = pool.takeWriter();
        assertReaderWriter(w, r, 1, Integer.class);

        // should always return the same original instance
        assertEquals(reader, r);
        assertEquals(writer, w);

        pool.offerReader(r);
        pool.offerWriter(w);
    }
}

---

From source file:org.apache.tinkerpop.gremlin.structure.io.gryo.GryoPoolTest.java

@Test
public void shouldConfigPoolOnConstructionWithCustomIoRegistryConstructor() throws Exception {
    final Configuration conf = new BaseConfiguration();
    conf.setProperty(GryoPool.CONFIG_IO_REGISTRY, IoXIoRegistry.ConstructorBased.class.getName());
    final GryoPool pool = GryoPool.build()
            .ioRegistries(conf.getList(GryoPool.CONFIG_IO_REGISTRY, Collections.emptyList())).create();
    assertReaderWriter(pool.takeWriter(), pool.takeReader(), new IoX("test"), IoX.class);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• »