List of usage examples for org.apache.commons.collections IteratorUtils toList
public static List toList(Iterator iterator)
From source file:org.pentaho.metaverse.step.MdiValidationIT.java
@Test public void testMdiInjectorStreamReadingAndStreamingSameStep() throws Exception { final String transNodeName = "injector_stream_same_as_mapping"; final String subTransNodeName = "template_stream_same_as_mapping"; initTest(transNodeName);//from w w w . j a va 2 s . c o m final TransformationNode injectorTransNode = verifyTransformationNode(transNodeName, false); final TransformationNode subTransNode = verifyTransformationNode(subTransNodeName, true); // smoke test - verify that the right number of nodes and edges exist in the graph and that the expected top // level nodes of expected types exist assertEquals("Unexpected number of nodes", 35, getIterableSize(framedGraph.getVertices())); assertEquals("Unexpected number of edges", 96, getIterableSize(framedGraph.getEdges())); verifyNodesTypes(ImmutableMap.of(NODE_TYPE_TRANS, Arrays.asList(new String[] { transNodeName, subTransNodeName }), NODE_TYPE_TRANS_FIELD, Arrays.asList(new String[] { "File Name", "Separator", "Field Name", "Type", "Trim Type", "Dummy", "First Name", "First Name", "First Name", "First Name", "Last Name", "Last Name", "Last Name", "Last Name", "Age", "Age" }), NODE_TYPE_STEP_PROPERTY, Arrays.asList(new String[] { "FILENAME", "SEPARATOR" }))); // The injector should "contain" 4 step nodes and no virtual nodes final Map<String, FramedMetaverseNode> stepNodeMap = verifyTransformationSteps(injectorTransNode, new String[] { "Text Output - Fields", "Text Output", "ETL Metadata Injection", "Text file output" }, false); // the template subTransformation should "contain" no step nodes (non-virtual) and one virtual nodes final Map<String, FramedMetaverseNode> subTransStepNodeMap = verifyTransformationSteps(subTransNode, new String[] { "My Text file output", "My Generate Rows" }, false); final TransformationStepNode textOutputFieldsNode = (TransformationStepNode) stepNodeMap .get("Text Output - Fields"); final TransformationStepNode textOutputNode = (TransformationStepNode) stepNodeMap.get("Text Output"); final TransformationStepNode metaInject = (TransformationStepNode) stepNodeMap .get("ETL Metadata Injection"); final TransformationStepNode textFileOutput = (TransformationStepNode) stepNodeMap.get("Text file output"); final TransformationStepNode myTextFileOutputNode = (TransformationStepNode) subTransStepNodeMap .get("My Text file output"); // Text Output verifyNodes(IteratorUtils.toList(textOutputNode.getPreviousSteps().iterator())); verifyNodes(IteratorUtils.toList(textOutputNode.getNextSteps().iterator()), testStepNode(metaInject.getName())); verifyNodes(IteratorUtils.toList(textOutputNode.getInputStreamFields().iterator())); verifyNodes(IteratorUtils.toList(textOutputNode.getOutputStreamFields().iterator()), testFieldNode("File Name"), testFieldNode("Separator")); assertEquals(2, getIterableSize(textOutputNode.getAllInNodes())); assertEquals(1, getIterableSize(textOutputNode.getInNodes(LINK_CONTAINS))); assertEquals(1, getIterableSize(textOutputNode.getInNodes(LINK_TYPE_CONCEPT))); assertEquals(3, getIterableSize(textOutputNode.getAllOutNodes())); assertEquals(1, getIterableSize(textOutputNode.getOutNodes(LINK_HOPSTO))); // ETL Metadata injection // verify the mdi node and connections to the node assertEquals(metaInject, verifyMdiNode(stepNodeMap, subTransNode, new String[] { "First Name", "Last Name" })); verifyInjectorTextFileOutputNode(metaInject, "stream_same_as_mapping_injector.txt", new String[] { "First Name", "Last Name" }, myTextFileOutputNode); // verify all the expected mdi properties and links from fields to properties exist verifyMdiInputs(metaInject, textOutputNode, myTextFileOutputNode.getName(), ImmutableMap.of("File Name", "FILENAME", "Separator", "SEPARATOR")); verifyMdiInputs(metaInject, textOutputFieldsNode, myTextFileOutputNode.getName(), ImmutableMap.of("Field Name", "", "Trim Type", "", "Type", "", "Dummy", "")); verifyNodes(IteratorUtils.toList(metaInject.getPreviousSteps().iterator()), testStepNode(textOutputNode.getName()), testStepNode(textOutputFieldsNode.getName())); verifyNodes(IteratorUtils.toList(metaInject.getNextSteps().iterator()), testStepNode(textFileOutput.getName())); verifyNodes(IteratorUtils.toList(metaInject.getInputStreamFields().iterator()), testFieldNode("File Name"), testFieldNode("Separator"), testFieldNode("Field Name"), testFieldNode("Type"), testFieldNode("Trim Type"), testFieldNode("Dummy"), testFieldNode("First Name"), testFieldNode("Last Name"), testFieldNode("Age")); verifyNodes(IteratorUtils.toList(metaInject.getStreamFieldNodesUses().iterator()), testFieldNode("File Name"), testFieldNode("Separator"), testFieldNode("Field Name"), testFieldNode("Trim Type")); verifyNodes(IteratorUtils.toList(metaInject.getOutputStreamFields().iterator()), testFieldNode("First Name"), testFieldNode("Last Name")); assertEquals(13, getIterableSize(metaInject.getAllInNodes())); assertEquals(1, getIterableSize(metaInject.getInNodes(LINK_CONTAINS))); assertEquals(1, getIterableSize(metaInject.getInNodes(LINK_TYPE_CONCEPT))); assertEquals(8, getIterableSize(metaInject.getAllOutNodes())); assertEquals(1, getIterableSize(metaInject.getOutNodes(LINK_HOPSTO))); assertEquals(1, getIterableSize(metaInject.getOutNodes(LINK_EXECUTES))); // Verify the following chains: // Chain 1: Text Output > outputs First Name > populates > FILENAME // Chain 2: My Text file output > contains > FILENAME final FramedMetaverseNode textOutput_output_fileName = verifyLinkedNode(textOutputNode, LINK_OUTPUTS, "File Name"); final FramedMetaverseNode textOutput_property_fileName = verifyLinkedNode(textOutput_output_fileName, LINK_POPULATES, "FILENAME"); assertEquals(textOutput_property_fileName, verifyLinkedNode(myTextFileOutputNode, LINK_CONTAINS, "FILENAME")); // Verify the following chains: // - Text Output - Fields > outputs > Field Name > derives > My Text file output:First Name > inputs > ETL Metadata // injection // - Text Output - Fields > outputs > Type > derives > My Text file output:Last Name > inputs > ETL Metadata // injection // - Text Output - Fields > outputs > Trim Type > derives > My Text file output:Age > inputs > ETL Metadata // injection final FramedMetaverseNode textOutputFields_output_fieldName = verifyLinkedNode(textOutputFieldsNode, LINK_OUTPUTS, "Field Name"); final FramedMetaverseNode myTextFileOutput_output_firstName = verifyLinkedNode(myTextFileOutputNode, LINK_OUTPUTS, "First Name"); assertEquals(myTextFileOutput_output_firstName, verifyLinkedNode(textOutputFields_output_fieldName, LINK_DERIVES, "First Name")); assertEquals(metaInject, verifyLinkedNode(myTextFileOutput_output_firstName, LINK_INPUTS, metaInject.getName())); final FramedMetaverseNode textOutputFields_output_type = verifyLinkedNode(textOutputFieldsNode, LINK_OUTPUTS, "Type"); final FramedMetaverseNode myTextFileOutput_output_lastName = verifyLinkedNode(myTextFileOutputNode, LINK_OUTPUTS, "Last Name"); assertEquals(myTextFileOutput_output_lastName, verifyLinkedNode(textOutputFields_output_type, LINK_DERIVES, "Last Name")); assertEquals(metaInject, verifyLinkedNode(myTextFileOutput_output_lastName, LINK_INPUTS, metaInject.getName())); final FramedMetaverseNode textOutputFields_output_trimType = verifyLinkedNode(textOutputFieldsNode, LINK_OUTPUTS, "Trim Type"); final FramedMetaverseNode myTextFileOutput_output_age = verifyLinkedNode(myTextFileOutputNode, LINK_OUTPUTS, "Age"); assertEquals(myTextFileOutput_output_age, verifyLinkedNode(textOutputFields_output_trimType, LINK_DERIVES, "Age")); assertEquals(metaInject, verifyLinkedNode(myTextFileOutput_output_age, LINK_INPUTS, metaInject.getName())); // Verify the following chains: // - My Text file output:First Name > derives > ETL Metadata injection:First Name > inputs > Text file output > // outputs First Name final FramedMetaverseNode metaInject_output_firstName = verifyLinkedNode(metaInject, LINK_OUTPUTS, "First Name"); assertEquals(metaInject_output_firstName, verifyLinkedNode(myTextFileOutput_output_firstName, LINK_DERIVES, "First Name")); assertEquals(textFileOutput, verifyLinkedNode(metaInject_output_firstName, LINK_INPUTS, textFileOutput.getName())); // validate properties verifyNodeProperties(metaInject, new ImmutableMap.Builder<String, Object>().put(PROPERTY_STEP_TYPE, SKIP) .put("color", SKIP).put(PROPERTY_PLUGIN_ID, SKIP).put(PROPERTY_TYPE, SKIP) .put(PROPERTY_ANALYZER, SKIP).put(PROPERTY_CATEGORY, SKIP).put(PROPERTY_COPIES, SKIP) .put(PROPERTY_LOGICAL_ID, SKIP).put(PROPERTY_NAME, SKIP).put(PROPERTY_NAMESPACE, SKIP) .put(NODE_VIRTUAL, SKIP).put("subTransformation", SKIP).put("runResultingTransformation", "true") .put("streamTargetStepname", SKIP).put("streamSourceStepname", SKIP).put("targetFile", SKIP) .put("sourceStepName", SKIP) .put(PROPERTY_VERBOSE_DETAILS, "mapping [1],ignored mapping [1],mapping [2],ignored mapping [2]") .put("mapping [1]", "Text Output: Separator > [template_stream_same_as_mapping] My Text file output: SEPARATOR") .put("mapping [2]", "Text Output: File Name > [template_stream_same_as_mapping] My Text file output: FILENAME") .put("ignored mapping [1]", "Text Output - Fields: Trim Type > [template_stream_same_as_mapping] My Text file " + "output: OUTPUT_TRIM") .put("ignored mapping [2]", "Text Output - Fields: Field Name > [template_stream_same_as_mapping] My Text file" + " output: OUTPUT_FIELDNAME") .build()); }
From source file:org.pentaho.metaverse.step.MdiValidationIT.java
@Test public void testMdiInjectorStreamReadingAndStreamingDifferentSteps() throws Exception { final String transNodeName = "injector_stream_different_than_mapping"; final String subTransNodeName = "template_stream_different_than_mapping"; initTest(transNodeName);//from w ww .j av a 2 s. c o m final TransformationNode injectorTransNode = verifyTransformationNode(transNodeName, false); final TransformationNode subTransNode = verifyTransformationNode(subTransNodeName, true); // smoke test - verify that the right number of nodes and edges exist in the graph and that the expected top // level nodes of expected types exist assertEquals("Unexpected number of nodes", 40, getIterableSize(framedGraph.getVertices())); assertEquals("Unexpected number of edges", 113, getIterableSize(framedGraph.getEdges())); verifyNodesTypes(ImmutableMap.of(NODE_TYPE_TRANS, Arrays.asList(new String[] { transNodeName, subTransNodeName }), NODE_TYPE_TRANS_FIELD, Arrays.asList(new String[] { "File Name", "Separator", "Field Name", "Type", "Trim Type", "Dummy", "First Name", "First Name", "First Name", "First Name", "First Name", "Last Name", "Last Name", "Last Name", "Last Name", "Last Name", "Age", "Age", "Age" }), NODE_TYPE_STEP_PROPERTY, Arrays.asList(new String[] { "FILENAME", "SEPARATOR" }))); // The injector should "contain" 4 step nodes and no virtual nodes final Map<String, FramedMetaverseNode> stepNodeMap = verifyTransformationSteps(injectorTransNode, new String[] { "Text Output - Fields", "Text Output", "ETL Metadata Injection", "Text file output" }, false); // the template subTransformation should "contain" no step nodes (non-virtual) and one virtual nodes final Map<String, FramedMetaverseNode> subTransStepNodeMap = verifyTransformationSteps(subTransNode, new String[] { "My Text file output", "My Text file output [2]", "My Generate Rows" }, false); final TransformationStepNode textOutputFieldsNode = (TransformationStepNode) stepNodeMap .get("Text Output - Fields"); final TransformationStepNode textOutputNode = (TransformationStepNode) stepNodeMap.get("Text Output"); final TransformationStepNode metaInject = (TransformationStepNode) stepNodeMap .get("ETL Metadata Injection"); final TransformationStepNode textFileOutput = (TransformationStepNode) stepNodeMap.get("Text file output"); final TransformationStepNode myTextFileOutputNode = (TransformationStepNode) subTransStepNodeMap .get("My Text file output"); final TransformationStepNode myTextFileOutputNode2 = (TransformationStepNode) subTransStepNodeMap .get("My Text file output [2]"); // Text Output verifyNodes(IteratorUtils.toList(textOutputNode.getPreviousSteps().iterator())); verifyNodes(IteratorUtils.toList(textOutputNode.getNextSteps().iterator()), testStepNode(metaInject.getName())); verifyNodes(IteratorUtils.toList(textOutputNode.getInputStreamFields().iterator())); verifyNodes(IteratorUtils.toList(textOutputNode.getOutputStreamFields().iterator()), testFieldNode("File Name"), testFieldNode("Separator")); assertEquals(2, getIterableSize(textOutputNode.getAllInNodes())); assertEquals(1, getIterableSize(textOutputNode.getInNodes(LINK_CONTAINS))); assertEquals(1, getIterableSize(textOutputNode.getInNodes(LINK_TYPE_CONCEPT))); assertEquals(3, getIterableSize(textOutputNode.getAllOutNodes())); assertEquals(1, getIterableSize(textOutputNode.getOutNodes(LINK_HOPSTO))); // ETL Metadata injection // verify the mdi node and connections to the node assertEquals(metaInject, verifyMdiNode(stepNodeMap, subTransNode, new String[] { "First Name", "Last Name" })); verifyInjectorTextFileOutputNode(metaInject, "stream_different_than_mapping_injector.txt", new String[] { "First Name", "Last Name" }, myTextFileOutputNode); // verify all the expected mdi properties and links from fields to properties exist verifyMdiInputs(metaInject, textOutputNode, myTextFileOutputNode.getName(), ImmutableMap.of("File Name", "FILENAME", "Separator", "SEPARATOR")); verifyMdiInputs(metaInject, textOutputFieldsNode, myTextFileOutputNode.getName(), ImmutableMap.of("Field Name", "", "Trim Type", "", "Type", "", "Dummy", "")); verifyNodes(IteratorUtils.toList(metaInject.getPreviousSteps().iterator()), testStepNode(textOutputNode.getName()), testStepNode(textOutputFieldsNode.getName())); verifyNodes(IteratorUtils.toList(metaInject.getNextSteps().iterator()), testStepNode(textFileOutput.getName())); verifyNodes(IteratorUtils.toList(metaInject.getInputStreamFields().iterator()), testFieldNode("File Name"), testFieldNode("Separator"), testFieldNode("Field Name"), testFieldNode("Type"), testFieldNode("Trim Type"), testFieldNode("Dummy"), testFieldNode("First Name"), testFieldNode("Last Name"), testFieldNode("Age")); verifyNodes(IteratorUtils.toList(metaInject.getStreamFieldNodesUses().iterator()), testFieldNode("File Name"), testFieldNode("Separator"), testFieldNode("Field Name"), testFieldNode("Trim Type")); verifyNodes(IteratorUtils.toList(metaInject.getOutputStreamFields().iterator()), testFieldNode("First Name"), testFieldNode("Last Name")); assertEquals(13, getIterableSize(metaInject.getAllInNodes())); assertEquals(1, getIterableSize(metaInject.getInNodes(LINK_CONTAINS))); assertEquals(1, getIterableSize(metaInject.getInNodes(LINK_TYPE_CONCEPT))); assertEquals(8, getIterableSize(metaInject.getAllOutNodes())); assertEquals(1, getIterableSize(metaInject.getOutNodes(LINK_HOPSTO))); assertEquals(1, getIterableSize(metaInject.getOutNodes(LINK_EXECUTES))); // Verify the following chains: // Chain 1: Text Output > outputs First Name > populates > FILENAME // Chain 2: My Text file output > contains > FILENAME final FramedMetaverseNode textOutput_output_fileName = verifyLinkedNode(textOutputNode, LINK_OUTPUTS, "File Name"); final FramedMetaverseNode textOutput_property_fileName = verifyLinkedNode(textOutput_output_fileName, LINK_POPULATES, "FILENAME"); assertEquals(textOutput_property_fileName, verifyLinkedNode(myTextFileOutputNode, LINK_CONTAINS, "FILENAME")); // Verify the following chains: // - Text Output - Fields > outputs > Field Name > derives My Text file output [2]: First Name // - My Text file output: First Name > inputs > ETL Metadata injection // - My Text file output: First Name > derives > ETL Metadata injection:First Name final FramedMetaverseNode textOutputFields_output_fieldName = verifyLinkedNode(textOutputFieldsNode, LINK_OUTPUTS, "Field Name"); final FramedMetaverseNode myTextFileOutput_output_firstName = verifyLinkedNode(myTextFileOutputNode, LINK_OUTPUTS, "First Name"); final FramedMetaverseNode myTextFileOutput2_output_firstName = verifyLinkedNode(myTextFileOutputNode2, LINK_OUTPUTS, "First Name"); final FramedMetaverseNode metaInject_output_firstName = verifyLinkedNode(metaInject, LINK_OUTPUTS, "First Name"); assertEquals(myTextFileOutput2_output_firstName, verifyLinkedNode(textOutputFields_output_fieldName, LINK_DERIVES, "First Name")); assertEquals(metaInject, verifyLinkedNode(myTextFileOutput_output_firstName, LINK_INPUTS, metaInject.getName())); assertEquals(metaInject_output_firstName, verifyLinkedNode(myTextFileOutput_output_firstName, LINK_DERIVES, "First Name")); // validate properties verifyNodeProperties(metaInject, new ImmutableMap.Builder<String, Object>().put(PROPERTY_STEP_TYPE, SKIP) .put("color", SKIP).put(PROPERTY_PLUGIN_ID, SKIP).put(PROPERTY_TYPE, SKIP) .put(PROPERTY_ANALYZER, SKIP).put(PROPERTY_CATEGORY, SKIP).put(PROPERTY_COPIES, SKIP) .put(PROPERTY_LOGICAL_ID, SKIP).put(PROPERTY_NAME, SKIP).put(PROPERTY_NAMESPACE, SKIP) .put(NODE_VIRTUAL, SKIP).put("subTransformation", SKIP).put("runResultingTransformation", "true") .put("streamTargetStepname", SKIP).put("streamSourceStepname", SKIP).put("targetFile", SKIP) .put("sourceStepName", SKIP) .put(PROPERTY_VERBOSE_DETAILS, "mapping [1],ignored mapping [1],mapping [2],ignored mapping [2]") .put("mapping [1]", "Text Output: Separator > [template_stream_different_than_mapping] My Text file output: SEPARATOR") .put("mapping [2]", "Text Output: File Name > [template_stream_different_than_mapping] My Text file output: FILENAME") .put("ignored mapping [1]", "Text Output - Fields: Trim Type > [template_stream_different_than_mapping]" + " My Text file output: OUTPUT_TRIM") .put("ignored mapping [2]", "Text Output - Fields: Field Name > [template_stream_different_than_mapping]" + " My Text file output: OUTPUT_FIELDNAME") .build()); }
From source file:org.pentaho.metaverse.step.MdiValidationIT.java
@Test public void testMdiInjectorNoStreaMapTwoSteps() throws Exception { final String transNodeName = "injector_no_stream_map_2_steps"; final String subTransNodeName = "template_no_stream_map_2_steps"; initTest(transNodeName);// ww w . j av a 2 s . c o m final TransformationNode injectorTransNode = verifyTransformationNode(transNodeName, false); final TransformationNode subTransNode = verifyTransformationNode(subTransNodeName, true); // smoke test - verify that the right number of nodes and edges exist in the graph and that the expected top // level nodes of expected types exist assertEquals("Unexpected number of nodes", 38, getIterableSize(framedGraph.getVertices())); assertEquals("Unexpected number of edges", 99, getIterableSize(framedGraph.getEdges())); verifyNodesTypes(ImmutableMap.of(NODE_TYPE_TRANS, Arrays.asList(new String[] { transNodeName, subTransNodeName }), NODE_TYPE_TRANS_FIELD, Arrays.asList(new String[] { "File Name", "Separator", "Field Name", "Trim Type", "First Name", "First Name", "First Name", "Last Name", "Last Name", "Last Name", "Age", "Age", "Age" }), NODE_TYPE_STEP_PROPERTY, Arrays.asList(new String[] { "FILENAME", "SEPARATOR", "SEPARATOR", "OUTPUT_FIELDNAME", "OUTPUT_FIELDNAME", "OUTPUT_TRIM" }))); // The injector should "contain" 4 step nodes and no virtual nodes final Map<String, FramedMetaverseNode> stepNodeMap = verifyTransformationSteps(injectorTransNode, new String[] { "Text Output - Fields", "Text Output", "ETL Metadata Injection", "Text file output" }, false); // the template subTransformation should "contain" no step nodes (non-virtual) and one virtual nodes final Map<String, FramedMetaverseNode> subTransStepNodeMap = verifyTransformationSteps(subTransNode, new String[] { "My Text file output", "My Text file output [2]", "My Generate Rows" }, false); final TransformationStepNode textOutputFieldsNode = (TransformationStepNode) stepNodeMap .get("Text Output - Fields"); final TransformationStepNode textOutputNode = (TransformationStepNode) stepNodeMap.get("Text Output"); final TransformationStepNode metaInject = (TransformationStepNode) stepNodeMap .get("ETL Metadata Injection"); final TransformationStepNode textFileOutput = (TransformationStepNode) stepNodeMap.get("Text file output"); final TransformationStepNode myTextFileOutputNode = (TransformationStepNode) subTransStepNodeMap .get("My Text file output"); final TransformationStepNode myTextFileOutputNode2 = (TransformationStepNode) subTransStepNodeMap .get("My Text file output [2]"); // Text Output verifyNodes(IteratorUtils.toList(textOutputNode.getPreviousSteps().iterator())); verifyNodes(IteratorUtils.toList(textOutputNode.getNextSteps().iterator()), testStepNode(metaInject.getName())); verifyNodes(IteratorUtils.toList(textOutputNode.getInputStreamFields().iterator())); verifyNodes(IteratorUtils.toList(textOutputNode.getOutputStreamFields().iterator()), testFieldNode("File Name"), testFieldNode("Separator")); assertEquals(2, getIterableSize(textOutputNode.getAllInNodes())); assertEquals(1, getIterableSize(textOutputNode.getInNodes(LINK_CONTAINS))); assertEquals(1, getIterableSize(textOutputNode.getInNodes(LINK_TYPE_CONCEPT))); assertEquals(3, getIterableSize(textOutputNode.getAllOutNodes())); assertEquals(1, getIterableSize(textOutputNode.getOutNodes(LINK_HOPSTO))); // ETL Metadata injection // verify the mdi node and connections to the node - since we are not streaming, the mdi node should have no // output fields assertEquals(metaInject, verifyMdiNode(stepNodeMap, subTransNode, new String[] {})); verifyInjectorTextFileOutputNode(metaInject, "injector_no_stream_map_2_steps.txt", new String[] {}, myTextFileOutputNode); // verify all the expected mdi properties and links from fields to properties exist verifyMdiInputs(metaInject, textOutputNode, myTextFileOutputNode.getName(), ImmutableMap.of("File Name", "FILENAME", "Separator", "SEPARATOR")); verifyMdiInputs(metaInject, textOutputFieldsNode, myTextFileOutputNode.getName(), ImmutableMap.of("Field Name", "OUTPUT_FIELDNAME", "Trim Type", "OUTPUT_TRIM")); verifyNodes(IteratorUtils.toList(metaInject.getPreviousSteps().iterator()), testStepNode(textOutputNode.getName()), testStepNode(textOutputFieldsNode.getName())); verifyNodes(IteratorUtils.toList(metaInject.getNextSteps().iterator()), testStepNode(textFileOutput.getName())); verifyNodes(IteratorUtils.toList(metaInject.getInputStreamFields().iterator()), testFieldNode("File Name"), testFieldNode("Separator"), testFieldNode("Field Name"), testFieldNode("Trim Type")); verifyNodes(IteratorUtils.toList(metaInject.getStreamFieldNodesUses().iterator()), testFieldNode("File Name"), testFieldNode("Separator"), testFieldNode("Field Name"), testFieldNode("Trim Type")); verifyNodes(IteratorUtils.toList(metaInject.getOutputStreamFields().iterator())); assertEquals(8, getIterableSize(metaInject.getAllInNodes())); assertEquals(1, getIterableSize(metaInject.getInNodes(LINK_CONTAINS))); assertEquals(1, getIterableSize(metaInject.getInNodes(LINK_TYPE_CONCEPT))); assertEquals(6, getIterableSize(metaInject.getAllOutNodes())); assertEquals(1, getIterableSize(metaInject.getOutNodes(LINK_HOPSTO))); assertEquals(1, getIterableSize(metaInject.getOutNodes(LINK_EXECUTES))); // Verify the following chains: // Chain 1: Text Output > outputs > First Name > populates > My Text file output:FILENAME // Chain 2: My Text file output > contains > FILENAME // Chain 3: Text Output > outputs > Separator > populates > My Text file output:SEPARATOR (x2) final FramedMetaverseNode textOutput_output_fileName = verifyLinkedNode(textOutputNode, LINK_OUTPUTS, "File Name"); final FramedMetaverseNode myTextFileOutput_property_fileName = verifyLinkedNode(myTextFileOutputNode, LINK_CONTAINS, "FILENAME"); assertEquals(myTextFileOutput_property_fileName, verifyLinkedNode(textOutput_output_fileName, LINK_POPULATES, "FILENAME")); final FramedMetaverseNode textOutput_output_separator = verifyLinkedNode(textOutputNode, LINK_OUTPUTS, "Separator"); final FramedMetaverseNode myTextFileOutput_property_separator = verifyLinkedNode(myTextFileOutputNode, LINK_CONTAINS, "SEPARATOR"); final List<FramedMetaverseNode> textOutput_property_separators = verifyLinkedNodes( textOutput_output_separator, LINK_POPULATES, "SEPARATOR"); assertEquals(2, textOutput_property_separators.size()); assertTrue(textOutput_property_separators.contains(myTextFileOutput_property_separator)); // Verify the following chains: // Chain 1: Text Output - Fields > outputs > First Name > populates > My Text file output [2]:OUTPUT_FIELDNAME (x2) // Chain 2: My Text file output [2] > contains > OUTPUT_FIELDNAME final FramedMetaverseNode textOutputFields_output_fieldName = verifyLinkedNode(textOutputFieldsNode, LINK_OUTPUTS, "Field Name"); final FramedMetaverseNode myTextFileOutput_property_fieldName = verifyLinkedNode(myTextFileOutputNode, LINK_CONTAINS, "OUTPUT_FIELDNAME"); final List<FramedMetaverseNode> textOutputFields_property_fieldNames = verifyLinkedNodes( textOutputFields_output_fieldName, LINK_POPULATES, "OUTPUT_FIELDNAME"); assertEquals(2, textOutputFields_property_fieldNames.size()); assertTrue(textOutputFields_property_fieldNames.contains(myTextFileOutput_property_fieldName)); // validate properties verifyNodeProperties(metaInject, new ImmutableMap.Builder<String, Object>().put(PROPERTY_STEP_TYPE, SKIP) .put("color", SKIP).put(PROPERTY_PLUGIN_ID, SKIP).put(PROPERTY_TYPE, SKIP) .put(PROPERTY_ANALYZER, SKIP).put(PROPERTY_CATEGORY, SKIP).put(PROPERTY_COPIES, SKIP) .put(PROPERTY_LOGICAL_ID, SKIP).put(PROPERTY_NAME, SKIP).put(PROPERTY_NAMESPACE, SKIP) .put(NODE_VIRTUAL, SKIP).put("subTransformation", SKIP).put("runResultingTransformation", "true") .put("targetFile", SKIP) .put(PROPERTY_VERBOSE_DETAILS, "mapping [1],mapping [2],mapping [3],mapping [4],mapping [5],mapping [6]") .put("mapping [1]", "Text Output: Separator > [template_no_stream_map_2_steps] My Text file output: SEPARATOR") .put("mapping [2]", "Text Output: Separator > [template_no_stream_map_2_steps] My Text file output [2]: " + "SEPARATOR") .put("mapping [3]", "Text Output - Fields: Trim Type > [template_no_stream_map_2_steps] My Text file output: " + "OUTPUT_TRIM") .put("mapping [4]", "Text Output - Fields: Field Name > [template_no_stream_map_2_steps] My Text file output: " + "OUTPUT_FIELDNAME") .put("mapping [5]", "Text Output: File Name > [template_no_stream_map_2_steps] My Text file output: FILENAME") .put("mapping [6]", "Text Output - Fields: Field Name > [template_no_stream_map_2_steps] My Text file output " + "[2]: OUTPUT_FIELDNAME") .build()); }
From source file:org.pentaho.metaverse.step.SimpleMappingAnalyzerValidationIT.java
@Test public void testMoreThanOneIOstepsMappingOut() throws Exception { final String transNodeName = "simple_moreThanOneIOstepsMappingOut"; initTest(transNodeName);//from ww w.j a v a2 s . com final TransformationNode transformationNode = verifyTransformationNode(transNodeName, false); final TransformationNode subTransNode = verifyTransformationNode("simple_sub", true); // smoke test - verify that the right number of nodes and edges exist in the graph and that the expected top // level nodes of expected types exist assertEquals("Unexpected number of nodes", 21, getIterableSize(framedGraph.getVertices())); assertEquals("Unexpected number of edges", 53, getIterableSize(framedGraph.getEdges())); verifyNodesTypes( ImmutableMap.of(NODE_TYPE_TRANS, Arrays.asList(new String[] { transNodeName, "simple_sub" }), NODE_TYPE_TRANS_FIELD, Arrays.asList(new String[] { RANDOM_VALUE, RANDOM_VALUE, VALUE, VALUE, VALUE, PARITY, PARITY, NEW_PARITY }))); // verify individual step nodes final Map<String, FramedMetaverseNode> parentStepNodeMap = verifyTransformationSteps(transformationNode, new String[] { "Generate random integer", "Simple mapping (sub-transformation)", "Write to log Parity" }, false); final Map<String, FramedMetaverseNode> subTransStepNodeMap = verifyTransformationSteps(subTransNode, new String[] { "Input parity", "output parity", "calc parity" }, false); final TransformationStepNode generateRandomInt = (TransformationStepNode) parentStepNodeMap .get("Generate random integer"); final TransformationStepNode simpleMapping = (TransformationStepNode) parentStepNodeMap .get("Simple mapping (sub-transformation)"); final TransformationStepNode writeToLogParity = (TransformationStepNode) parentStepNodeMap .get("Write to log Parity"); // virtual sub-trans nodes within the parent graph final TransformationStepNode inputParity = (TransformationStepNode) subTransStepNodeMap.get("Input parity"); final TransformationStepNode calcParity = (TransformationStepNode) subTransStepNodeMap.get("calc parity"); final TransformationStepNode outputParity = (TransformationStepNode) subTransStepNodeMap .get("output parity"); // ---------- Generate Random Int verifyNodes(IteratorUtils.toList(generateRandomInt.getPreviousSteps().iterator())); verifyNodes(IteratorUtils.toList(generateRandomInt.getNextSteps().iterator()), testLineageNode(simpleMapping)); verifyNodes(IteratorUtils.toList(generateRandomInt.getOutputStreamFields().iterator()), testFieldNode(RANDOM_VALUE, false)); assertEquals(2, getIterableSize(generateRandomInt.getAllInNodes())); assertEquals(1, getIterableSize(generateRandomInt.getInNodes(LINK_CONTAINS))); assertEquals(1, getIterableSize(generateRandomInt.getInNodes(LINK_TYPE_CONCEPT))); assertEquals(2, getIterableSize(generateRandomInt.getAllOutNodes())); // ---------- "Simple mapping (sub-transformation) verifyNodes(IteratorUtils.toList(simpleMapping.getPreviousSteps().iterator()), testLineageNode(generateRandomInt)); verifyNodes(IteratorUtils.toList(simpleMapping.getInputStreamFields().iterator()), testFieldNode(RANDOM_VALUE, false)); verifyNodes(IteratorUtils.toList(simpleMapping.getOutputStreamFields().iterator())); assertEquals(4, getIterableSize(simpleMapping.getAllInNodes())); assertEquals(1, getIterableSize(simpleMapping.getInNodes(LINK_CONTAINS))); assertEquals(1, getIterableSize(simpleMapping.getInNodes(LINK_TYPE_CONCEPT))); assertEquals(2, getIterableSize(simpleMapping.getAllOutNodes())); assertEquals(1, getIterableSize(simpleMapping.getOutNodes(LINK_EXECUTES))); // verify properties verifyNodeProperties(simpleMapping, new ImmutableMap.Builder<String, Object>().put(PROPERTY_STEP_TYPE, SKIP) .put("color", SKIP).put(PROPERTY_PLUGIN_ID, SKIP).put(PROPERTY_TYPE, SKIP) .put(PROPERTY_ANALYZER, SKIP).put(PROPERTY_CATEGORY, SKIP).put(PROPERTY_COPIES, SKIP) .put(PROPERTY_LOGICAL_ID, SKIP).put(PROPERTY_NAME, SKIP).put(PROPERTY_NAMESPACE, SKIP) .put(PROPERTY_PATH, SKIP).put(NODE_VIRTUAL, SKIP).put("subTransformation", SKIP) .put(PROPERTY_VERBOSE_DETAILS, "input [1],input [1] update field names,input [1] rename [1],output [1],output [1] " + "update field names,output [1] rename [1]") .put("input [1]", "Generate random integer > [simple_sub] Input parity") .put("input [1] update field names", "true").put("input [1] rename [1]", "randomValue > value") .put("output [1]", "[simple_sub] output parity > Write to log Parity") .put("output [1] rename [1]", "parity > newParity").put("output [1] update field names", "false") .build()); // ---------- output parity verifyNodes(IteratorUtils.toList(outputParity.getPreviousSteps().iterator()), testStepNode(calcParity.getName(), false)); verifyNodes(IteratorUtils.toList(outputParity.getNextSteps().iterator())); verifyNodes(IteratorUtils.toList(outputParity.getInputStreamFields().iterator()), testFieldNode(PARITY, false), testFieldNode(VALUE, false)); verifyNodes(IteratorUtils.toList(outputParity.getOutputStreamFields().iterator()), testFieldNode(PARITY, false), testFieldNode(VALUE, false)); assertEquals(5, getIterableSize(outputParity.getAllInNodes())); assertEquals(1, getIterableSize(outputParity.getInNodes(LINK_CONTAINS))); assertEquals(1, getIterableSize(outputParity.getInNodes(LINK_TYPE_CONCEPT))); assertEquals(2, getIterableSize(outputParity.getAllOutNodes())); // Verify the following link chains // - chain 1: Generate Random Int > outputs > randomValue > inputs > Input parity > outputs > value > inputs > // calc parity > outputs > value > inputs > output parity > outputs > value > inputs // > Write to log Parity > outputs > randomValue // - chain 2: Generate Random Int > outputs > randomValue > derives > Input Parity:value > derives // > calc parity:value > derives > output parity:value > derives > Write to log Parity:randomValue final FramedMetaverseNode generateRandomInt_output_randomValue = verifyLinkedNode(generateRandomInt, LINK_OUTPUTS, RANDOM_VALUE); final FramedMetaverseNode inputParity_output_value = verifyLinkedNode(inputParity, LINK_OUTPUTS, VALUE); final FramedMetaverseNode calcParity_output_value = verifyLinkedNode(calcParity, LINK_OUTPUTS, VALUE); final FramedMetaverseNode outputParity_output_value = verifyLinkedNode(outputParity, LINK_OUTPUTS, VALUE); final FramedMetaverseNode writeToLogParity_output_randomValue = verifyLinkedNode(writeToLogParity, LINK_OUTPUTS, RANDOM_VALUE); assertEquals(inputParity, verifyLinkedNode(generateRandomInt_output_randomValue, LINK_INPUTS, inputParity.getName())); assertEquals(calcParity, verifyLinkedNode(inputParity_output_value, LINK_INPUTS, calcParity.getName())); assertEquals(outputParity, verifyLinkedNode(calcParity_output_value, LINK_INPUTS, outputParity.getName())); assertEquals(writeToLogParity, verifyLinkedNode(outputParity_output_value, LINK_INPUTS, writeToLogParity.getName())); assertEquals(inputParity_output_value, verifyLinkedNode(generateRandomInt_output_randomValue, LINK_DERIVES, VALUE)); assertEquals(calcParity_output_value, verifyLinkedNode(inputParity_output_value, LINK_DERIVES, VALUE)); assertEquals(outputParity_output_value, verifyLinkedNode(calcParity_output_value, LINK_DERIVES, VALUE)); assertEquals(writeToLogParity_output_randomValue, verifyLinkedNode(outputParity_output_value, LINK_DERIVES, RANDOM_VALUE)); // Verify the following link chains // - chain 1: calc parity > outputs > parity > inputs > output parity > outputs > parity > inputs // > Write to log Parity > outputs > newParity // - chain 2: calc parity:parity > derives > output parity:parity > derives > Write to log // Parity:newParity final FramedMetaverseNode calcParity_output_parity = verifyLinkedNode(calcParity, LINK_OUTPUTS, PARITY); final FramedMetaverseNode outputParity_output_parity = verifyLinkedNode(outputParity, LINK_OUTPUTS, PARITY); final FramedMetaverseNode writeToLogParity_output_newParity = verifyLinkedNode(writeToLogParity, LINK_OUTPUTS, NEW_PARITY); assertEquals(outputParity, verifyLinkedNode(calcParity_output_parity, LINK_INPUTS, outputParity.getName())); assertEquals(writeToLogParity, verifyLinkedNode(outputParity_output_parity, LINK_INPUTS, writeToLogParity.getName())); assertEquals(outputParity_output_parity, verifyLinkedNode(calcParity_output_parity, LINK_DERIVES, PARITY)); assertEquals(writeToLogParity_output_newParity, verifyLinkedNode(outputParity_output_parity, LINK_DERIVES, NEW_PARITY)); }
From source file:org.pentaho.metaverse.step.SingleThreaderStepAnalyzerValidationIT.java
@Test public void testOneOIWithMappings() throws Exception { final String transNodeName = "parent-single-threader"; final String subTransNodeName = "sub-trans"; initTest(transNodeName);//from w ww . ja va 2 s .c o m final TransformationNode transformationNode = verifyTransformationNode(transNodeName, false); final TransformationNode subTransNode = verifyTransformationNode(subTransNodeName, true); // smoke test - verify that the right number of nodes and edges exist in the graph and that the expected top // level nodes of expected types exist assertEquals("Unexpected number of nodes", 30, getIterableSize(framedGraph.getVertices())); assertEquals("Unexpected number of edges", 87, getIterableSize(framedGraph.getEdges())); verifyNodesTypes(ImmutableMap.of(NODE_TYPE_TRANS, Arrays.asList(new String[] { transNodeName, subTransNodeName }), NODE_TYPE_TRANS_FIELD, Arrays.asList(new String[] { "strTest", "strTest", "strTest", "strTest", "strTest", "strTest", "strTest", "strTest", "counter", "counter", "counter", "counter", "counter", "counter", "counter" }))); // verify individual step nodes final Map<String, FramedMetaverseNode> parentStepNodeMap = verifyTransformationSteps(transformationNode, new String[] { "Data grid", "Add sequence", "Single threader", "Write to log", "Write to log 2" }, false); final Map<String, FramedMetaverseNode> subTransStepNodeMap = verifyTransformationSteps(subTransNode, new String[] { "Mapping input specification", "String operations", "Mapping output specification", }, false); final TransformationStepNode singleThreader = (TransformationStepNode) parentStepNodeMap .get("Single threader"); final TransformationStepNode writeToLog = (TransformationStepNode) parentStepNodeMap.get("Write to log"); final TransformationStepNode writeToLog2 = (TransformationStepNode) parentStepNodeMap.get("Write to log 2"); final TransformationStepNode inputSpec = (TransformationStepNode) subTransStepNodeMap .get("Mapping input specification"); final TransformationStepNode stringOperations = (TransformationStepNode) subTransStepNodeMap .get("String operations"); final TransformationStepNode outputSpec = (TransformationStepNode) subTransStepNodeMap .get("Mapping output specification"); // ---------- Generate Random Int verifyNodes(IteratorUtils.toList(singleThreader.getPreviousSteps().iterator()), testStepNode("Add sequence")); verifyNodes(IteratorUtils.toList(singleThreader.getNextSteps().iterator()), testStepNode(writeToLog.getName()), testStepNode(writeToLog2.getName())); verifyNodes(IteratorUtils.toList(singleThreader.getInputStreamFields().iterator()), testFieldNode("counter", false), testFieldNode("strTest", false)); verifyNodes(IteratorUtils.toList(singleThreader.getOutputStreamFields().iterator()), testFieldNode("counter", false), testFieldNode("strTest", false)); assertEquals(5, getIterableSize(singleThreader.getAllInNodes())); assertEquals(1, getIterableSize(singleThreader.getInNodes(LINK_CONTAINS))); assertEquals(1, getIterableSize(singleThreader.getInNodes(LINK_TYPE_CONCEPT))); assertEquals(5, getIterableSize(singleThreader.getAllOutNodes())); assertEquals(1, getIterableSize(singleThreader.getOutNodes(LINK_EXECUTES))); // verify the following link chain: // Single threader > outputs > counter > derives > Mapping input specification:counter > derives > // String operations:counter > Mapping output specification:counter > derives > Single threader:counter final FramedMetaverseNode singleThreader_output_counter = verifyLinkedNode(singleThreader, LINK_OUTPUTS, "counter"); final FramedMetaverseNode inputSpec_output_counter = verifyLinkedNode(inputSpec, LINK_OUTPUTS, "counter"); final FramedMetaverseNode stringOperations_output_counter = verifyLinkedNode(stringOperations, LINK_OUTPUTS, "counter"); final FramedMetaverseNode outputSpec_output_counter = verifyLinkedNode(outputSpec, LINK_OUTPUTS, "counter"); assertEquals(inputSpec_output_counter, verifyLinkedNode(singleThreader_output_counter, LINK_DERIVES, "counter")); assertEquals(stringOperations_output_counter, verifyLinkedNode(inputSpec_output_counter, LINK_DERIVES, "counter")); assertEquals(outputSpec_output_counter, verifyLinkedNode(stringOperations_output_counter, LINK_DERIVES, "counter")); assertEquals(singleThreader_output_counter, verifyLinkedNode(outputSpec_output_counter, LINK_DERIVES, "counter")); }
From source file:org.pentaho.metaverse.step.TableOutputValidationIT.java
@Test public void testTableIoBogusDbFieldUnchecked() throws Exception { final String transNodeName = "table_io_bogus_db_fields_unchecked"; initTest(transNodeName);//from w ww . j a va2s . c o m final TransformationNode transformationNode = verifyTransformationNode(transNodeName, false); // smoke test - verify that the right number of nodes and edges exist in the graph and that the expected top // level nodes of expected types exist assertEquals(26, getIterableSize(framedGraph.getVertices())); assertEquals(62, getIterableSize(framedGraph.getEdges())); verifyNodesTypes(ImmutableMap.of(NODE_TYPE_TRANS, Arrays.asList(new String[] { transNodeName }), NODE_TYPE_DATASOURCE, Arrays.asList(new String[] { "Postgres - localhost" }), NODE_TYPE_DATA_TABLE, Arrays.asList(new String[] { "people" }), NODE_TYPE_DATA_COLUMN, Arrays.asList(new String[] { "first_name", "first_name", "last_name", "last_name" }), NODE_TYPE_TRANS_FIELD, Arrays.asList(new String[] { "first_name", "first_name", "first_name", "last_name", "last_name", "last_name" }))); // verify the connection node final List<DatasourceNode> dbConnections = IteratorUtils.toList(root.getDatasourceNodes().iterator()); assertEquals(1, dbConnections.size()); final DatasourceNode dbConnection = dbConnections.get(0); assertEquals("localhost", dbConnection.getHostName()); assertEquals("postgres", dbConnection.getDatabaseName()); assertEquals("postgres", dbConnection.getUserName()); assertEquals("5432", dbConnection.getPort()); assertNull(dbConnection.getPassword()); // password should be null, do not include in graph // verify individual step nodes final Map<String, FramedMetaverseNode> stepNodeMap = verifyTransformationSteps(transformationNode, new String[] { "Table input", "Select values", "Table output" }, false); // ---------- Table input final TransformationStepNode tableInput = (TransformationStepNode) stepNodeMap.get("Table input"); verifyNodes(IteratorUtils.toList(tableInput.getReadByNodes().iterator()), testSqlQueryNode("SQL", false)); verifyNodes(IteratorUtils.toList(tableInput.getNextSteps().iterator()), testStepNode("Select values", false)); verifyNodes(IteratorUtils.toList(tableInput.getInputFields().iterator()), testColumnNode("first_name", false), testColumnNode("last_name", false)); verifyNodes(IteratorUtils.toList(tableInput.getOutputStreamFields().iterator()), testFieldNode("first_name", false), testFieldNode("last_name", false)); verifyStepIOLinks(tableInput, testLineageLink(testColumnNode("first_name", false), LINK_POPULATES, testFieldNode("first_name", false)), testLineageLink(testColumnNode("last_name", false), LINK_POPULATES, testFieldNode("last_name", false))); final FramedMetaverseNode sqlNode = (FramedMetaverseNode) IteratorUtils .toList(tableInput.getReadByNodes().iterator()).get(0); verifyNodes(IteratorUtils.toList(sqlNode.getContainedNodes().iterator()), testColumnNode("first_name", false), testColumnNode("last_name", false)); // ---------- Select values final TransformationStepNode selectValues = (TransformationStepNode) stepNodeMap.get("Select values"); verifyNodes(IteratorUtils.toList(selectValues.getPreviousSteps().iterator()), testStepNode("Table input", false)); verifyNodes(IteratorUtils.toList(selectValues.getNextSteps().iterator()), testStepNode("Table output", false)); verifyNodes(IteratorUtils.toList(selectValues.getInputStreamFields().iterator()), testFieldNode("first_name", false), testFieldNode("last_name", false)); verifyNodes(IteratorUtils.toList(selectValues.getOutputStreamFields().iterator()), testFieldNode("first_name", false), testFieldNode("last_name", false)); verifyNodes(IteratorUtils.toList(selectValues.getStreamFieldNodesUses().iterator()), testFieldNode("first_name", false), testFieldNode("last_name", false)); verifyStepIOLinks(selectValues, testLineageLink(testFieldNode("first_name", false), LINK_DERIVES, testFieldNode("first_name", false)), testLineageLink(testFieldNode("last_name", false), LINK_DERIVES, testFieldNode("last_name", false))); // ---------- Table output final TransformationStepNode tableOutput = (TransformationStepNode) stepNodeMap.get("Table output"); verifyNodes(IteratorUtils.toList(tableOutput.getPreviousSteps().iterator()), testStepNode("Select values", false)); verifyNodes(IteratorUtils.toList(tableOutput.getWritesToNodes().iterator()), testTableNode("people", false)); verifyNodes(IteratorUtils.toList(tableOutput.getInputStreamFields().iterator()), testFieldNode("first_name", false), testFieldNode("last_name", false)); verifyNodes(IteratorUtils.toList(tableOutput.getOutputStreamFields().iterator()), testFieldNode("first_name", false), testColumnNode("first_name", false), testFieldNode("last_name", false), testColumnNode("last_name", false)); verifyStepIOLinks(tableOutput, testLineageLink(testFieldNode("first_name", false), LINK_DERIVES, testFieldNode("first_name", false)), testLineageLink(testFieldNode("first_name", false), LINK_POPULATES, testColumnNode("first_name", false)), testLineageLink(testFieldNode("last_name", false), LINK_DERIVES, testFieldNode("last_name", false)), testLineageLink(testFieldNode("last_name", false), LINK_POPULATES, testColumnNode("last_name", false))); final FramedMetaverseNode people = (FramedMetaverseNode) IteratorUtils .toList(tableOutput.getWritesToNodes().iterator()).get(0); verifyNodes(IteratorUtils.toList(people.getContainedNodes().iterator()), testColumnNode("first_name", false), testColumnNode("last_name", false)); }
From source file:org.pentaho.metaverse.step.TableOutputValidationIT.java
@Test public void testTableIoDbFieldRenamedAndListTrimmed() throws Exception { final String transNodeName = "table_io_db_field_renamed_and_list_trimmed"; initTest(transNodeName);//from ww w. ja v a 2 s. c om final TransformationNode transformationNode = verifyTransformationNode(transNodeName, false); // smoke test - verify that the right number of nodes and edges exist in the graph and that the expected top // level nodes of expected types exist assertEquals(28, getIterableSize(framedGraph.getVertices())); assertEquals(69, getIterableSize(framedGraph.getEdges())); verifyNodesTypes(ImmutableMap.of(NODE_TYPE_TRANS, Arrays.asList(new String[] { transNodeName }), NODE_TYPE_DATASOURCE, Arrays.asList(new String[] { "Postgres - localhost" }), NODE_TYPE_DATA_TABLE, Arrays.asList(new String[] { "people" }), NODE_TYPE_DATA_COLUMN, Arrays.asList(new String[] { "first_name", "first_name", "last_name" }), NODE_TYPE_TRANS_FIELD, Arrays.asList(new String[] { "first_name", "first_name_renamed", "first_name_renamed", "first_name_renamed", "last_name", "last_name", "last_name", "last_name" }))); // verify individual step nodes final Map<String, FramedMetaverseNode> stepNodeMap = verifyTransformationSteps(transformationNode, new String[] { "Table input", "Select values", "Table output", "Dummy (do nothing)" }, false); // ---------- Table input final TransformationStepNode tableInput = (TransformationStepNode) stepNodeMap.get("Table input"); verifyNodes(IteratorUtils.toList(tableInput.getReadByNodes().iterator()), testSqlQueryNode("SQL", false)); verifyNodes(IteratorUtils.toList(tableInput.getNextSteps().iterator()), testStepNode("Select values", false)); verifyNodes(IteratorUtils.toList(tableInput.getInputFields().iterator()), testColumnNode("first_name", false), testColumnNode("last_name", false)); verifyNodes(IteratorUtils.toList(tableInput.getOutputStreamFields().iterator()), testFieldNode("first_name", false), testFieldNode("last_name", false)); verifyStepIOLinks(tableInput, testLineageLink(testColumnNode("first_name", false), LINK_POPULATES, testFieldNode("first_name", false)), testLineageLink(testColumnNode("last_name", false), LINK_POPULATES, testFieldNode("last_name", false))); final FramedMetaverseNode sqlNode = (FramedMetaverseNode) IteratorUtils .toList(tableInput.getReadByNodes().iterator()).get(0); verifyNodes(IteratorUtils.toList(sqlNode.getContainedNodes().iterator()), testColumnNode("first_name", false), testColumnNode("last_name", false)); // ---------- Select values final TransformationStepNode selectValues = (TransformationStepNode) stepNodeMap.get("Select values"); verifyNodes(IteratorUtils.toList(selectValues.getPreviousSteps().iterator()), testStepNode("Table input", false)); verifyNodes(IteratorUtils.toList(selectValues.getNextSteps().iterator()), testStepNode("Table output", false)); verifyNodes(IteratorUtils.toList(selectValues.getInputStreamFields().iterator()), testFieldNode("first_name", false), testFieldNode("last_name", false)); verifyNodes(IteratorUtils.toList(selectValues.getOutputStreamFields().iterator()), testFieldNode("first_name_renamed", false), testFieldNode("last_name", false)); verifyNodes(IteratorUtils.toList(selectValues.getStreamFieldNodesUses().iterator()), testFieldNode("first_name", false), testFieldNode("last_name", false)); verifyStepIOLinks(selectValues, Arrays.asList(new TestLineageLink[] { testLineageLink(testFieldNode("first_name", false), LINK_DERIVES, testFieldNode("first_name_renamed", false)), testLineageLink(testFieldNode("last_name", false), LINK_DERIVES, testFieldNode("last_name", false)) })); // ---------- Table output final TransformationStepNode tableOutput = (TransformationStepNode) stepNodeMap.get("Table output"); verifyNodes(IteratorUtils.toList(tableOutput.getPreviousSteps().iterator()), testStepNode("Select values", false)); verifyNodes(IteratorUtils.toList(tableOutput.getNextSteps().iterator()), testStepNode("Dummy (do nothing)", false)); verifyNodes(IteratorUtils.toList(tableOutput.getWritesToNodes().iterator()), testTableNode("people", false)); verifyNodes(IteratorUtils.toList(tableOutput.getInputStreamFields().iterator()), testFieldNode("first_name_renamed", false), testFieldNode("last_name", false)); verifyNodes(IteratorUtils.toList(tableOutput.getOutputStreamFields().iterator()), testFieldNode("first_name_renamed", false), testColumnNode("first_name", false), testFieldNode("last_name", false)); verifyStepIOLinks(tableOutput, testLineageLink(testFieldNode("first_name_renamed", false), LINK_DERIVES, testFieldNode("first_name_renamed", false)), testLineageLink(testFieldNode("first_name_renamed", false), LINK_POPULATES, testColumnNode("first_name", false)), testLineageLink(testFieldNode("last_name", false), LINK_DERIVES, testFieldNode("last_name", false))); final FramedMetaverseNode people = (FramedMetaverseNode) IteratorUtils .toList(tableOutput.getWritesToNodes().iterator()).get(0); verifyNodes(IteratorUtils.toList(people.getContainedNodes().iterator()), testColumnNode("first_name", false)); // ---------- Table output final TransformationStepNode dummy = (TransformationStepNode) stepNodeMap.get("Dummy (do nothing)"); verifyNodes(IteratorUtils.toList(dummy.getPreviousSteps().iterator()), testStepNode("Table output", false)); verifyNodes(IteratorUtils.toList(tableOutput.getInputStreamFields().iterator()), testFieldNode("first_name_renamed", false), testFieldNode("last_name", false)); verifyNodes(IteratorUtils.toList(tableOutput.getOutputStreamFields().iterator()), testFieldNode("first_name_renamed", false), testColumnNode("first_name", false), testFieldNode("last_name", false)); verifyStepIOLinks(tableOutput, testLineageLink(testFieldNode("first_name_renamed", false), LINK_DERIVES, testFieldNode("first_name_renamed", false)), testLineageLink(testFieldNode("last_name", false), LINK_DERIVES, testFieldNode("last_name", false))); }
From source file:org.springframework.data.elasticsearch.repository.support.ParentChildElasticsearchRepositoryTests.java
@Test public void shouldSearchDocumentsGivenParentSearchQuery() { // given//from w ww .java 2s. c o m String parentId1 = randomNumeric(5); ParentEntity parentEntity1 = new ParentEntity(); parentEntity1.setId(parentId1); parentEntity1.setText("parent_text_1"); String parentId2 = randomNumeric(5); ParentEntity parentEntity2 = new ParentEntity(); parentEntity2.setId(parentId2); parentEntity2.setText("parent_text_2"); String childId1 = randomNumeric(5); ChildEntity childEntity1 = new ChildEntity(); childEntity1.setId(childId1); childEntity1.setParentId(parentId1); childEntity1.setText("child_text_1"); String childId2 = randomNumeric(5); ChildEntity childEntity2 = new ChildEntity(); childEntity2.setId(childId2); childEntity2.setParentId(parentId1); childEntity2.setText("child_text_2"); String childId3 = randomNumeric(5); ChildEntity childEntity3 = new ChildEntity(); childEntity3.setId(childId3); childEntity3.setParentId(parentId2); childEntity3.setText("child_text_2"); // when parentRepository.save(Arrays.asList(parentEntity1, parentEntity2)); childRepository.save(Arrays.asList(childEntity1, childEntity2, childEntity3)); // then // simple query TermQueryBuilder simpleBuilder = QueryBuilders.termQuery("text", "child_text_2"); List<ChildEntity> children = IteratorUtils.toList(childRepository.search(simpleBuilder).iterator()); assertThat(children, hasSize(2)); assertThat(children.get(0).getId(), equalTo(childId2)); assertThat(children.get(1).getId(), equalTo(childId3)); // parent query BoolQueryBuilder parentBuilder = QueryBuilders.boolQuery(); parentBuilder.must(QueryBuilders.termQuery("text", "child_text_2")); parentBuilder.must( QueryBuilders.hasParentQuery("parententity", QueryBuilders.termQuery("text", "parent_text_1"))); // We should use PageRequest search, since single search calls _count request and it doesn't work with hasParent // filter in ES 0.90.2, detailed: https://github.com/elasticsearch/elasticsearch/issues/3190 children = IteratorUtils.toList(childRepository.search(parentBuilder, new PageRequest(0, 10)).iterator()); assertThat(children, hasSize(1)); assertThat(children.get(0).getId(), equalTo(childId2)); }
From source file:org.wso2.carbon.analytics.api.AnalyticsDataAPIUtil.java
public static List<Record> listRecords(AnalyticsDataAPI ads, AnalyticsDataResponse response) throws AnalyticsException { List<Record> result = new ArrayList<Record>(); for (AnalyticsDataResponse.Entry entry : response.getEntries()) { result.addAll(/*from w w w . ja va2 s.c o m*/ IteratorUtils.toList(ads.readRecords(entry.getRecordStoreName(), entry.getRecordGroup()))); } return result; }
From source file:org.wso2.carbon.analytics.data.commons.test.AnalyticsRecordStoreTest.java
@Test @SuppressWarnings("unchecked") public void testDataRecordAddRetrieve() throws AnalyticsException { this.cleanupT1(); String serverName = "ESB1"; String ip = "10.0.0.1"; int tenant = 44; String log = "Boom!"; this.analyticsRS.createTable("T1"); Record record = this.createRecord("T1", serverName, ip, tenant, log); List<Record> records = new ArrayList<>(); records.add(record);/* ww w . j a v a2 s . c o m*/ this.analyticsRS.put(records); String id = record.getId(); List<String> ids = new ArrayList<>(); ids.add(id); RecordGroup[] rgs = this.analyticsRS.get("T1", 3, null, ids); Assert.assertEquals(rgs.length, 1); List<Record> recordsIn = IteratorUtils.toList(this.analyticsRS.readRecords(rgs[0])); Assert.assertEquals(recordsIn.size(), 1); Record recordIn = recordsIn.get(0); Assert.assertEquals(record.getId(), recordIn.getId()); Assert.assertEquals(record.getTableName(), recordIn.getTableName()); Assert.assertEquals(record.getTimestamp(), recordIn.getTimestamp()); Assert.assertEquals(record.getNotNullValues(), recordIn.getNotNullValues()); Assert.assertEquals(record, recordIn); this.cleanupT1(); }