Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package com.datatorrent.lib.projection; import java.lang.reflect.Field; import java.util.ArrayList; import java.util.List; import javax.validation.constraints.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.commons.lang3.ClassUtils; import org.apache.hadoop.classification.InterfaceStability; import com.google.common.annotations.VisibleForTesting; import com.datatorrent.api.AutoMetric; import com.datatorrent.api.Context; import com.datatorrent.api.Context.PortContext; import com.datatorrent.api.DefaultInputPort; import com.datatorrent.api.DefaultOutputPort; import com.datatorrent.api.Operator; import com.datatorrent.api.annotation.InputPortFieldAnnotation; import com.datatorrent.api.annotation.OutputPortFieldAnnotation; import com.datatorrent.common.util.BaseOperator; import com.datatorrent.lib.util.PojoUtils; /** * <b>ProjectionOperator</b> * Projection Operator projects defined set of fields from given selectFields/dropFields * * <b>Parameters</b> * - selectFields: comma separated list of fields to be selected from input tuples * - dropFields: comma separated list of fields to be dropped from input tuples * selectFields and dropFields are optional and either of them shall be specified * When both are not specified, all fields shall be projected to downstream operator * When both are specified, selectFields shall be given the preference * * <b>Input Port</b> takes POJOs as an input * * <b>Output Ports</b> * - projected port emits POJOs with projected fields from input POJOs * - remainder port, if connected, emits POJOs with remainder fields from input POJOs * - error port emits input POJOs as is upon error situations * * <b>Examples</b> * For {a, b, c} type of input tuples * - when selectFields = "" and dropFields = "", projected port shall emit {a, b, c} * - when selectFields = "a" and dropFields = "b", projected port shall emit {a}, remainder {b, c} * - when selectFields = "b", projected port shall emit {b} and remainder port shall emit {a, c} * - when dropFields = "b", projected port shall emit {a, c} and remainder port shall emit {b} * * * @since 3.4.0 */ @InterfaceStability.Evolving public class ProjectionOperator extends BaseOperator implements Operator.ActivationListener<Context> { @NotNull private List<String> selectFields; @NotNull private List<String> dropFields; static class TypeInfo { String name; Class type; PojoUtils.Setter setter; PojoUtils.Getter getter; public TypeInfo(String name, Class<?> type) { this.name = name; this.type = type; } public String toString() { String s = new String("'name': " + name + " 'type': " + type); return s; } } private transient List<TypeInfo> projectedFields = new ArrayList<>(); private transient List<TypeInfo> remainderFields = new ArrayList<>(); @VisibleForTesting List<TypeInfo> getProjectedFields() { return projectedFields; } @VisibleForTesting List<TypeInfo> getRemainderFields() { return remainderFields; } @AutoMetric protected long projectedTuples; @AutoMetric protected long remainderTuples; @AutoMetric protected long errorTuples; protected Class<?> inClazz = null; protected Class<?> projectedClazz = null; protected Class<?> remainderClazz = null; @InputPortFieldAnnotation(schemaRequired = true) public transient DefaultInputPort<Object> input = new DefaultInputPort<Object>() { public void setup(PortContext context) { inClazz = context.getValue(Context.PortContext.TUPLE_CLASS); } @Override public void process(Object t) { handleProjection(t); } }; @OutputPortFieldAnnotation(schemaRequired = true) public final transient DefaultOutputPort<Object> projected = new DefaultOutputPort<Object>() { public void setup(PortContext context) { projectedClazz = context.getValue(Context.PortContext.TUPLE_CLASS); } }; @OutputPortFieldAnnotation(schemaRequired = true) public final transient DefaultOutputPort<Object> remainder = new DefaultOutputPort<Object>() { public void setup(PortContext context) { remainderClazz = context.getValue(Context.PortContext.TUPLE_CLASS); } }; public final transient DefaultOutputPort<Object> error = new DefaultOutputPort<>(); /** * addProjectedField: Add field details (name, type, getter and setter) for field with given name * in projectedFields list */ protected void addProjectedField(String s) { try { Field f = inClazz.getDeclaredField(s); TypeInfo t = new TypeInfo(f.getName(), ClassUtils.primitiveToWrapper(f.getType())); t.getter = PojoUtils.createGetter(inClazz, t.name, t.type); t.setter = PojoUtils.createSetter(projectedClazz, t.name, t.type); projectedFields.add(t); } catch (NoSuchFieldException e) { throw new RuntimeException("Field " + s + " not found in class " + inClazz, e); } } /** * addRemainderField: Add field details (name, type, getter and setter) for field with given name * in remainderFields list */ protected void addRemainderField(String s) { try { Field f = inClazz.getDeclaredField(s); TypeInfo t = new TypeInfo(f.getName(), ClassUtils.primitiveToWrapper(f.getType())); t.getter = PojoUtils.createGetter(inClazz, t.name, t.type); t.setter = PojoUtils.createSetter(remainderClazz, t.name, t.type); remainderFields.add(t); } catch (NoSuchFieldException e) { throw new RuntimeException("Field " + s + " not found in class " + inClazz, e); } } @Override public void activate(Context context) { final Field[] allFields = inClazz.getDeclaredFields(); if ((selectFields != null) && !selectFields.isEmpty()) { for (String s : selectFields) { addProjectedField(s); } if (remainderClazz != null) { for (Field f : allFields) { if (!selectFields.contains(f.getName())) { addRemainderField(f.getName()); } } } else { logger.info("Remainder Port does not have Schema class defined"); } } else { if ((dropFields != null) && !dropFields.isEmpty()) { if (remainderClazz != null) { for (String s : dropFields) { addRemainderField(s); } } else { logger.info("Remainder Port does not have Schema class defined"); } } for (Field f : allFields) { if ((dropFields == null) || !dropFields.contains(f.getName())) { addProjectedField(f.getName()); } } } logger.debug("projected fields: {}", projectedFields); logger.debug("remainder fields: {}", remainderFields); } @Override public void deactivate() { projectedFields.clear(); remainderFields.clear(); } @Override public void beginWindow(long windowId) { errorTuples = projectedTuples = remainderTuples = 0; } protected Object getProjectedObject(Object t) throws IllegalAccessException { try { Object p = projectedClazz.newInstance(); for (TypeInfo ti : projectedFields) { ti.setter.set(p, ti.getter.get(t)); } return p; } catch (InstantiationException e) { throw new RuntimeException(e); } catch (IllegalAccessException e) { throw e; } } protected Object getRemainderObject(Object t) throws IllegalAccessException { try { Object r = remainderClazz.newInstance(); for (TypeInfo ti : remainderFields) { ti.setter.set(r, ti.getter.get(t)); } return r; } catch (InstantiationException e) { throw new RuntimeException(e); } catch (IllegalAccessException e) { throw e; } } /** * handleProjection: emit projected object on projected port * and remainder object on remainder port if that is connected. */ private void handleProjection(Object t) { try { Object p = getProjectedObject(t); if (remainder.isConnected()) { Object r = getRemainderObject(t); remainder.emit(r); remainderTuples++; } projected.emit(p); projectedTuples++; } catch (IllegalAccessException e) { error.emit(t); errorTuples++; } } /** * set selectFields, a list of fields to be selected from incoming POJO * * @param selectFields List of fields from POJO to be selected * @description $[] Field which become part of selected fields * @useSchema $[] input.fields[].name */ public void setSelectFields(List<String> selectFields) { this.selectFields = selectFields; } /** * get selectFields, a list of fields to be selected from incoming POJO * * @return selectFields list of fields from POJO to be selected */ public List<String> getSelectFields() { return selectFields; } /** * set dropFields, a list of fields to be dropped from incoming POJO * * @param dropFields List of fields from POJO to be selected * @description $[] Field which become part of dropped fields * @useSchema $[] input.fields[].name */ public void setDropFields(List<String> dropFields) { this.dropFields = dropFields; } /** * get dropFields, a list of fields to be dropped from incoming POJO * * @return dropFields list of fields from POJO to be selected */ public List<String> getDropFields() { return dropFields; } private static final Logger logger = LoggerFactory.getLogger(ProjectionOperator.class); }