Java tutorial
/** * Licensed to JumpMind Inc under one or more contributor * license agreements. See the NOTICE file distributed * with this work for additional information regarding * copyright ownership. JumpMind Inc licenses this file * to you under the GNU General Public License, version 3.0 (GPLv3) * (the "License"); you may not use this file except in compliance * with the License. * * You should have received a copy of the GNU General Public License, * version 3.0 (GPLv3) along with this library; if not, see * <http://www.gnu.org/licenses/>. * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.jumpmind.symmetric.io.data.writer; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.StringUtils; import org.jumpmind.db.model.Table; import org.jumpmind.db.platform.IDatabasePlatform; import org.jumpmind.symmetric.io.data.Batch; import org.jumpmind.symmetric.io.data.CsvData; import org.jumpmind.symmetric.io.data.DataContext; import org.jumpmind.symmetric.io.data.DataEventType; import org.jumpmind.symmetric.io.data.IDataWriter; import org.jumpmind.symmetric.io.data.transform.DeleteAction; import org.jumpmind.symmetric.io.data.transform.IColumnTransform; import org.jumpmind.symmetric.io.data.transform.IgnoreColumnException; import org.jumpmind.symmetric.io.data.transform.IgnoreRowException; import org.jumpmind.symmetric.io.data.transform.NewAndOldValue; import org.jumpmind.symmetric.io.data.transform.TransformColumn; import org.jumpmind.symmetric.io.data.transform.TransformColumn.IncludeOnType; import org.jumpmind.symmetric.io.data.transform.TransformColumnException; import org.jumpmind.symmetric.io.data.transform.TransformPoint; import org.jumpmind.symmetric.io.data.transform.TransformTable; import org.jumpmind.symmetric.io.data.transform.TransformedData; import org.jumpmind.util.Statistics; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class TransformWriter extends NestedDataWriter { protected static final Logger log = LoggerFactory.getLogger(TransformWriter.class); protected TransformPoint transformPoint; protected IDatabasePlatform platform; protected Map<String, List<TransformTable>> transformsBySourceTable; protected Table sourceTable; protected List<TransformTable> activeTransforms; protected Batch batch; protected Map<String, IColumnTransform<?>> columnTransforms; protected Table lastTransformedTable; public TransformWriter(IDatabasePlatform platform, TransformPoint transformPoint, IDataWriter targetWriter, Map<String, IColumnTransform<?>> columnTransforms, TransformTable... transforms) { super(targetWriter); this.columnTransforms = columnTransforms; this.platform = platform; this.transformPoint = transformPoint == null ? TransformPoint.LOAD : transformPoint; this.transformsBySourceTable = toMap(transforms); } protected Map<String, List<TransformTable>> toMap(TransformTable[] transforms) { Map<String, List<TransformTable>> transformsByTable = new HashMap<String, List<TransformTable>>(); if (transforms != null) { for (TransformTable transformTable : transforms) { if (transformPoint == transformTable.getTransformPoint()) { String sourceTableName = transformTable.getFullyQualifiedSourceTableName().toLowerCase(); List<TransformTable> tables = transformsByTable.get(sourceTableName); if (tables == null) { tables = new ArrayList<TransformTable>(); transformsByTable.put(sourceTableName, tables); } tables.add(transformTable); } } } return transformsByTable; } @Override public void start(Batch batch) { this.batch = batch; super.start(batch); } @Override public boolean start(Table table) { activeTransforms = transformsBySourceTable.get(table.getFullyQualifiedTableName().toLowerCase()); if (activeTransforms != null && activeTransforms.size() > 0) { this.sourceTable = table; return true; } else { this.sourceTable = null; return super.start(table); } } protected boolean isTransformable(DataEventType eventType) { return eventType != null && (eventType == DataEventType.INSERT || eventType == DataEventType.UPDATE || eventType == DataEventType.DELETE); } public void write(CsvData data) { DataEventType eventType = data.getDataEventType(); if (activeTransforms != null && activeTransforms.size() > 0 && isTransformable(eventType)) { if (data.requiresTable() && sourceTable == null && context.getLastParsedTable() != null) { // if we cross batches and the table isn't specified, then // use the last table we used start(context.getLastParsedTable()); } long ts = System.currentTimeMillis(); Map<String, String> sourceValues = data.toColumnNameValuePairs(this.sourceTable.getColumnNames(), CsvData.ROW_DATA); Map<String, String> oldSourceValues = null; if (data.contains(CsvData.OLD_DATA)) { oldSourceValues = data.toColumnNameValuePairs(this.sourceTable.getColumnNames(), CsvData.OLD_DATA); } Map<String, String> sourceKeyValues = null; if (data.contains(CsvData.PK_DATA)) { sourceKeyValues = data.toKeyColumnValuePairs(this.sourceTable); } if (eventType == DataEventType.DELETE) { sourceValues = oldSourceValues; if (sourceValues == null || sourceValues.size() == 0) { sourceValues = sourceKeyValues; } } if (log.isDebugEnabled()) { log.debug("{} transformation(s) started because of {} on {}. The original row data was: {}", new Object[] { activeTransforms.size(), eventType.toString(), this.sourceTable.getFullyQualifiedTableName(), sourceValues }); } List<TransformedData> dataThatHasBeenTransformed = new ArrayList<TransformedData>(); TransformTable[] transformTables = activeTransforms .toArray(new TransformTable[activeTransforms.size()]); if (eventType == DataEventType.DELETE) { CollectionUtils.reverseArray(transformTables); } for (TransformTable transformation : transformTables) { transformation = transformation.enhanceWithImpliedColumns( this.sourceTable.getPrimaryKeyColumnNames(), this.sourceTable.getColumnNames()); if (eventType == DataEventType.INSERT && transformation.isUpdateFirst()) { eventType = DataEventType.UPDATE; } dataThatHasBeenTransformed.addAll(transform(eventType, context, transformation, sourceKeyValues, oldSourceValues, sourceValues)); } for (TransformedData transformedData : dataThatHasBeenTransformed) { Table transformedTable = transformedData.buildTargetTable(); CsvData csvData = transformedData.buildTargetCsvData(); long transformTimeInMs = System.currentTimeMillis() - ts; boolean processData = true; if (lastTransformedTable == null || !lastTransformedTable.equals(transformedTable)) { if (lastTransformedTable != null) { this.nestedWriter.end(lastTransformedTable); } processData = this.nestedWriter.start(transformedTable); if (!processData) { lastTransformedTable = null; } else { lastTransformedTable = transformedTable; } } if (processData || !csvData.requiresTable()) { this.nestedWriter.write(csvData); } Statistics stats = this.nestedWriter.getStatistics().get(batch); if (stats != null) { stats.increment(DataWriterStatisticConstants.TRANSFORMMILLIS, transformTimeInMs); } ts = System.currentTimeMillis(); } } else { if (sourceTable != null) { super.start(sourceTable); } super.write(data); if (sourceTable != null) { super.end(sourceTable); } } } protected List<TransformedData> transform(DataEventType eventType, DataContext context, TransformTable transformation, Map<String, String> sourceKeyValues, Map<String, String> oldSourceValues, Map<String, String> sourceValues) { try { List<TransformedData> dataToTransform = create(context, eventType, transformation, sourceKeyValues, oldSourceValues, sourceValues); List<TransformedData> dataThatHasBeenTransformed = new ArrayList<TransformedData>( dataToTransform.size()); if (log.isDebugEnabled()) { log.debug("{} target data was created for the {} transformation. The target table is {}", new Object[] { dataToTransform.size(), transformation.getTransformId(), transformation.getFullyQualifiedTargetTableName() }); } int transformNumber = 0; for (TransformedData targetData : dataToTransform) { transformNumber++; if (perform(context, targetData, transformation, sourceValues, oldSourceValues)) { if (log.isDebugEnabled()) { log.debug( "Data has been transformed to a {} for the #{} transform. The mapped target columns are: {}. The mapped target values are: {}", new Object[] { targetData.getTargetDmlType().toString(), transformNumber, ArrayUtils.toString(targetData.getColumnNames()), ArrayUtils.toString(targetData.getColumnValues()) }); } dataThatHasBeenTransformed.add(targetData); } else { log.debug("Data has not been transformed for the #{} transform", transformNumber); } } return dataThatHasBeenTransformed; } catch (IgnoreRowException ex) { // ignore this row if (log.isDebugEnabled()) { log.debug("Transform indicated that the target row should be ignored with a target key of: {}", "unknown. Transformation aborted during tranformation of key"); } return new ArrayList<TransformedData>(0); } } protected boolean perform(DataContext context, TransformedData data, TransformTable transformation, Map<String, String> sourceValues, Map<String, String> oldSourceValues) throws IgnoreRowException { boolean persistData = false; try { DataEventType eventType = data.getSourceDmlType(); for (TransformColumn transformColumn : transformation.getTransformColumns()) { if (!transformColumn.isPk()) { IncludeOnType includeOn = transformColumn.getIncludeOn(); if (includeOn == IncludeOnType.ALL || (includeOn == IncludeOnType.INSERT && eventType == DataEventType.INSERT) || (includeOn == IncludeOnType.UPDATE && eventType == DataEventType.UPDATE) || (includeOn == IncludeOnType.DELETE && eventType == DataEventType.DELETE)) { if (StringUtils.isBlank(transformColumn.getSourceColumnName()) || sourceValues.containsKey(transformColumn.getSourceColumnName())) { try { Object value = transformColumn(context, data, transformColumn, sourceValues, oldSourceValues); if (value instanceof NewAndOldValue) { data.put(transformColumn, ((NewAndOldValue) value).getNewValue(), oldSourceValues != null ? ((NewAndOldValue) value).getOldValue() : null, false); } else if (value == null || value instanceof String) { data.put(transformColumn, (String) value, null, false); } else if (value instanceof List) { throw new IllegalStateException(String.format( "Column transform failed %s.%s. Transforms that multiply rows must be marked as part of the primary key", transformColumn.getTransformId(), transformColumn.getTargetColumnName())); } else { throw new IllegalStateException(String.format( "Column transform failed %s.%s. It returned an unexpected type of %s", transformColumn.getTransformId(), transformColumn.getTargetColumnName(), value.getClass().getSimpleName())); } } catch (IgnoreColumnException e) { // Do nothing. We are ignoring the column if (log.isDebugEnabled()) { log.debug("A transform indicated we should ignore the target column {}", transformColumn.getTargetColumnName()); } } } else { if (eventType != DataEventType.DELETE) { log.warn("Could not find a source column of {} for the transformation: {}", transformColumn.getSourceColumnName(), transformation.getTransformId()); } else { log.debug( "Could not find a source column of {} for the transformation: {}. This is probably because this was a DELETE event and no old data was captured.", transformColumn.getSourceColumnName(), transformation.getTransformId()); } } } } } // perform a transformation if there are columns defined for // transformation if (data.getColumnNames().length > 0) { if (data.getTargetDmlType() != DataEventType.DELETE) { persistData = true; } else { // handle the delete action DeleteAction deleteAction = transformation.getDeleteAction(); switch (deleteAction) { case DEL_ROW: data.setTargetDmlType(DataEventType.DELETE); persistData = true; break; case UPDATE_COL: data.setTargetDmlType(DataEventType.UPDATE); persistData = true; break; case NONE: default: if (log.isDebugEnabled()) { log.debug( "The {} transformation is not configured to delete row. Not sending the delete through.", transformation.getTransformId()); } } } } } catch (IgnoreRowException ex) { // ignore this row if (log.isDebugEnabled()) { log.debug("Transform indicated that the target row should be ignored with a target key of: {}", ArrayUtils.toString(data.getKeyValues())); } } return persistData; } protected List<TransformedData> create(DataContext context, DataEventType dataEventType, TransformTable transformation, Map<String, String> sourceKeyValues, Map<String, String> oldSourceValues, Map<String, String> sourceValues) throws IgnoreRowException { List<TransformColumn> columns = transformation.getPrimaryKeyColumns(); if (columns == null || columns.size() == 0) { log.error("No primary key defined for the transformation: {}", transformation.getTransformId()); return new ArrayList<TransformedData>(0); } else { List<TransformedData> datas = new ArrayList<TransformedData>(); TransformedData data = new TransformedData(transformation, dataEventType, sourceKeyValues, oldSourceValues, sourceValues); datas.add(data); DataEventType eventType = data.getSourceDmlType(); for (TransformColumn transformColumn : columns) { IncludeOnType includeOn = transformColumn.getIncludeOn(); if (includeOn == IncludeOnType.ALL || (includeOn == IncludeOnType.INSERT && eventType == DataEventType.INSERT) || (includeOn == IncludeOnType.UPDATE && eventType == DataEventType.UPDATE) || (includeOn == IncludeOnType.DELETE && eventType == DataEventType.DELETE)) { List<TransformedData> newDatas = null; try { Object columnValue = transformColumn(context, data, transformColumn, sourceValues, oldSourceValues); if (columnValue instanceof List) { @SuppressWarnings("unchecked") List<String> values = (List<String>) columnValue; if (values.size() > 0) { data.put(transformColumn, values.get(0), oldSourceValues != null ? values.get(0) : null, true); if (values.size() > 1) { if (newDatas == null) { newDatas = new ArrayList<TransformedData>(values.size() - 1); } for (int i = 1; i < values.size(); i++) { TransformedData newData = data.copy(); newData.put(transformColumn, values.get(i), oldSourceValues != null ? values.get(i) : null, true); newDatas.add(newData); } } } else { throw new IgnoreRowException(); } } else if (columnValue instanceof NewAndOldValue) { data.put(transformColumn, ((NewAndOldValue) columnValue).getNewValue(), oldSourceValues != null ? ((NewAndOldValue) columnValue).getOldValue() : null, true); } else { data.put(transformColumn, (String) columnValue, oldSourceValues != null ? (String) columnValue : null, true); } } catch (IgnoreColumnException e) { // Do nothing. We are suppose to ignore the column. } if (newDatas != null) { datas.addAll(newDatas); newDatas = null; } } } return datas; } } protected Object transformColumn(DataContext context, TransformedData data, TransformColumn transformColumn, Map<String, String> sourceValues, Map<String, String> oldSourceValues) throws IgnoreRowException, IgnoreColumnException { Object returnValue = null; String value = transformColumn.getSourceColumnName() != null ? sourceValues.get(transformColumn.getSourceColumnName()) : null; returnValue = value; IColumnTransform<?> transform = columnTransforms != null ? columnTransforms.get(transformColumn.getTransformType()) : null; if (transform != null) { try { String oldValue = null; if (oldSourceValues != null) { oldValue = oldSourceValues.get(transformColumn.getSourceColumnName()); } returnValue = transform.transform(platform, context, transformColumn, data, sourceValues, value, oldValue); } catch (RuntimeException ex) { log.warn("Column transform failed {}.{} ({}) for source values of {}", new Object[] { transformColumn.getTransformId(), transformColumn.getTargetColumnName(), transformColumn.getIncludeOn().name(), sourceValues.toString() }); throw ex; } } else { throw new TransformColumnException(String.format("Could not locate a column transform of type '%s'", transformColumn.getTransformType())); } return returnValue; } public void end(Table table) { if (this.lastTransformedTable != null) { this.nestedWriter.end(lastTransformedTable); this.lastTransformedTable = null; } if (activeTransforms != null && activeTransforms.size() > 0) { activeTransforms = null; } else { super.end(table); } } }