Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.flume.channel.jdbc.impl; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; import java.util.Enumeration; import java.util.HashMap; import java.util.Map; import java.util.Properties; import java.util.concurrent.atomic.AtomicLong; import javax.sql.DataSource; import org.apache.commons.dbcp.ConnectionFactory; import org.apache.commons.dbcp.DriverManagerConnectionFactory; import org.apache.commons.dbcp.PoolableConnectionFactory; import org.apache.commons.dbcp.PoolingDataSource; import org.apache.commons.pool.KeyedObjectPoolFactory; import org.apache.commons.pool.impl.GenericKeyedObjectPoolFactory; import org.apache.commons.pool.impl.GenericObjectPool; import org.apache.flume.Context; import org.apache.flume.Event; import org.apache.flume.channel.jdbc.ConfigurationConstants; import org.apache.flume.channel.jdbc.DatabaseType; import org.apache.flume.channel.jdbc.JdbcChannelException; import org.apache.flume.channel.jdbc.JdbcChannelProvider; import org.apache.flume.channel.jdbc.TransactionIsolation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class JdbcChannelProviderImpl implements JdbcChannelProvider { private static final Logger LOGGER = LoggerFactory.getLogger(JdbcChannelProviderImpl.class); private static final String EMBEDDED_DERBY_DRIVER_CLASSNAME = "org.apache.derby.jdbc.EmbeddedDriver"; private static final String DEFAULT_DRIVER_CLASSNAME = EMBEDDED_DERBY_DRIVER_CLASSNAME; private static final String DEFAULT_USERNAME = "sa"; private static final String DEFAULT_PASSWORD = ""; private static final String DEFAULT_DBTYPE = "DERBY"; /** The connection pool. */ private GenericObjectPool connectionPool; /** The statement cache pool */ private KeyedObjectPoolFactory statementPool; /** The data source. */ private DataSource dataSource; /** The database type. */ private DatabaseType databaseType; /** The schema handler. */ private SchemaHandler schemaHandler; /** Transaction factory */ private JdbcTransactionFactory txFactory; /** Connection URL */ private String connectUrl; /** Driver Class Name */ private String driverClassName; /** Capacity Counter if one is needed */ private long maxCapacity = 0L; /** The current size of the channel. */ private AtomicLong currentSize = new AtomicLong(0L); @Override public void initialize(Context context) { if (LOGGER.isDebugEnabled()) { LOGGER.debug("Initializing JDBC Channel provider with props: " + context); } initializeSystemProperties(context); initializeDataSource(context); initializeSchema(context); initializeChannelState(context); } private void initializeSystemProperties(Context context) { Map<String, String> sysProps = new HashMap<String, String>(); Map<String, String> sysPropsOld = context .getSubProperties(ConfigurationConstants.OLD_CONFIG_JDBC_SYSPROP_PREFIX); if (sysPropsOld.size() > 0) { LOGGER.warn("Long form configuration prefix \"" + ConfigurationConstants.OLD_CONFIG_JDBC_SYSPROP_PREFIX + "\" is deprecated. Please use the short form prefix \"" + ConfigurationConstants.CONFIG_JDBC_SYSPROP_PREFIX + "\" instead."); sysProps.putAll(sysPropsOld); } Map<String, String> sysPropsNew = context .getSubProperties(ConfigurationConstants.CONFIG_JDBC_SYSPROP_PREFIX); // Override the deprecated values with the non-deprecated if (sysPropsNew.size() > 0) { sysProps.putAll(sysPropsNew); } for (String key : sysProps.keySet()) { String value = sysProps.get(key); if (key != null && value != null) { System.setProperty(key, value); } } } private void initializeChannelState(Context context) { String maxCapacityStr = getConfigurationString(context, ConfigurationConstants.CONFIG_MAX_CAPACITY, ConfigurationConstants.OLD_CONFIG_MAX_CAPACITY, "0"); long maxCapacitySpecified = 0; try { maxCapacitySpecified = Long.parseLong(maxCapacityStr); } catch (NumberFormatException nfe) { LOGGER.warn("Invalid value specified for maximum channel capacity: " + maxCapacityStr, nfe); } if (maxCapacitySpecified > 0) { this.maxCapacity = maxCapacitySpecified; LOGGER.info("Maximum channel capacity: {}", maxCapacity); } else { LOGGER.warn("JDBC channel will operate without a capacity limit."); } if (maxCapacity > 0) { // Initialize current size JdbcTransactionImpl tx = null; try { tx = getTransaction(); tx.begin(); Connection conn = tx.getConnection(); currentSize.set(schemaHandler.getChannelSize(conn)); tx.commit(); } catch (Exception ex) { tx.rollback(); throw new JdbcChannelException("Failed to initialize current size", ex); } finally { if (tx != null) { tx.close(); } } long currentSizeLong = currentSize.get(); if (currentSizeLong > maxCapacity) { LOGGER.warn("The current size of channel (" + currentSizeLong + ") is more than the specified maximum capacity (" + maxCapacity + "). If this situation persists, it may require resizing and " + "replanning of your deployment."); } LOGGER.info("Current channel size: {}", currentSizeLong); } } private void initializeSchema(Context context) { String createSchemaFlag = getConfigurationString(context, ConfigurationConstants.CONFIG_CREATE_SCHEMA, ConfigurationConstants.OLD_CONFIG_CREATE_SCHEMA, "true"); boolean createSchema = Boolean.valueOf(createSchemaFlag); LOGGER.debug("Create schema flag set to: " + createSchema); // First check if the schema exists schemaHandler = SchemaHandlerFactory.getHandler(databaseType, dataSource); if (!schemaHandler.schemaExists()) { if (!createSchema) { throw new JdbcChannelException("Schema does not exist and " + "auto-generation is disabled. Please enable auto-generation of " + "schema and try again."); } String createIndexFlag = getConfigurationString(context, ConfigurationConstants.CONFIG_CREATE_INDEX, ConfigurationConstants.OLD_CONFIG_CREATE_INDEX, "true"); String createForeignKeysFlag = getConfigurationString(context, ConfigurationConstants.CONFIG_CREATE_FK, ConfigurationConstants.OLD_CONFIG_CREATE_FK, "true"); boolean createIndex = Boolean.valueOf(createIndexFlag); if (!createIndex) { LOGGER.warn("Index creation is disabled, indexes will not be created."); } boolean createForeignKeys = Boolean.valueOf(createForeignKeysFlag); if (createForeignKeys) { LOGGER.info("Foreign Key Constraints will be enabled."); } else { LOGGER.info("Foreign Key Constratins will be disabled."); } // Now create schema schemaHandler.createSchemaObjects(createForeignKeys, createIndex); } // Validate all schema objects are as expected schemaHandler.validateSchema(); } @Override public void close() { try { connectionPool.close(); } catch (Exception ex) { throw new JdbcChannelException("Unable to close connection pool", ex); } if (databaseType.equals(DatabaseType.DERBY) && driverClassName.equals(EMBEDDED_DERBY_DRIVER_CLASSNAME)) { // Need to shut down the embedded Derby instance if (connectUrl.startsWith("jdbc:derby:")) { int index = connectUrl.indexOf(";"); String baseUrl = null; if (index != -1) { baseUrl = connectUrl.substring(0, index + 1); } else { baseUrl = connectUrl + ";"; } String shutDownUrl = baseUrl + "shutdown=true"; LOGGER.debug("Attempting to shutdown embedded Derby using URL: " + shutDownUrl); try { DriverManager.getConnection(shutDownUrl); } catch (SQLException ex) { // Shutdown for one db instance is expected to raise SQL STATE 45000 if (ex.getErrorCode() != 45000) { throw new JdbcChannelException("Unable to shutdown embedded Derby: " + shutDownUrl + " Error Code: " + ex.getErrorCode(), ex); } LOGGER.info("Embedded Derby shutdown raised SQL STATE " + "45000 as expected."); } } else { LOGGER.warn("Even though embedded Derby drvier was loaded, the connect " + "URL is of an unexpected form: " + connectUrl + ". Therfore no " + "attempt will be made to shutdown embedded Derby instance."); } } dataSource = null; txFactory = null; schemaHandler = null; } @Override public void persistEvent(String channel, Event event) { PersistableEvent persistableEvent = new PersistableEvent(channel, event); JdbcTransactionImpl tx = null; try { tx = getTransaction(); tx.begin(); if (maxCapacity > 0) { long currentSizeLong = currentSize.get(); if (currentSizeLong >= maxCapacity) { throw new JdbcChannelException("Channel capacity reached: " + "maxCapacity: " + maxCapacity + ", currentSize: " + currentSizeLong); } } // Persist the persistableEvent schemaHandler.storeEvent(persistableEvent, tx.getConnection()); tx.incrementPersistedEventCount(); tx.commit(); } catch (Exception ex) { tx.rollback(); throw new JdbcChannelException("Failed to persist event", ex); } finally { if (tx != null) { tx.close(); } } LOGGER.debug("Persisted event: {}", persistableEvent.getEventId()); } @Override public Event removeEvent(String channelName) { PersistableEvent result = null; JdbcTransactionImpl tx = null; try { tx = getTransaction(); tx.begin(); // Retrieve the persistableEvent result = schemaHandler.fetchAndDeleteEvent(channelName, tx.getConnection()); if (result != null) { tx.incrementRemovedEventCount(); } tx.commit(); } catch (Exception ex) { tx.rollback(); throw new JdbcChannelException("Failed to persist event", ex); } finally { if (tx != null) { tx.close(); } } if (result != null) { LOGGER.debug("Removed event: {}", ((PersistableEvent) result).getEventId()); } else { LOGGER.debug("No event found for removal"); } return result; } @Override public JdbcTransactionImpl getTransaction() { return txFactory.get(); } /** * Initializes the datasource and the underlying connection pool. * @param properties */ private void initializeDataSource(Context context) { driverClassName = getConfigurationString(context, ConfigurationConstants.CONFIG_JDBC_DRIVER_CLASS, ConfigurationConstants.OLD_CONFIG_JDBC_DRIVER_CLASS, null); connectUrl = getConfigurationString(context, ConfigurationConstants.CONFIG_URL, ConfigurationConstants.OLD_CONFIG_URL, null); String userName = getConfigurationString(context, ConfigurationConstants.CONFIG_USERNAME, ConfigurationConstants.OLD_CONFIG_USERNAME, null); String password = getConfigurationString(context, ConfigurationConstants.CONFIG_PASSWORD, ConfigurationConstants.OLD_CONFIG_PASSWORD, null); String jdbcPropertiesFile = getConfigurationString(context, ConfigurationConstants.CONFIG_JDBC_PROPS_FILE, ConfigurationConstants.OLD_CONFIG_JDBC_PROPS_FILE, null); String dbTypeName = getConfigurationString(context, ConfigurationConstants.CONFIG_DATABASE_TYPE, ConfigurationConstants.OLD_CONFIG_DATABASE_TYPE, null); // If connect URL is not specified, use embedded Derby if (connectUrl == null || connectUrl.trim().length() == 0) { LOGGER.warn("No connection URL specified. " + "Using embedded derby database instance."); driverClassName = DEFAULT_DRIVER_CLASSNAME; userName = DEFAULT_USERNAME; password = DEFAULT_PASSWORD; dbTypeName = DEFAULT_DBTYPE; String homePath = System.getProperty("user.home").replace('\\', '/'); String defaultDbDir = homePath + "/.flume/jdbc-channel"; File dbDir = new File(defaultDbDir); String canonicalDbDirPath = null; try { canonicalDbDirPath = dbDir.getCanonicalPath(); } catch (IOException ex) { throw new JdbcChannelException("Unable to find canonical path of dir: " + defaultDbDir, ex); } if (!dbDir.exists()) { if (!dbDir.mkdirs()) { throw new JdbcChannelException("unable to create directory: " + canonicalDbDirPath); } } connectUrl = "jdbc:derby:" + canonicalDbDirPath + "/db;create=true"; // No jdbc properties file will be used jdbcPropertiesFile = null; LOGGER.warn("Overriding values for - driver: " + driverClassName + ", user: " + userName + "connectUrl: " + connectUrl + ", jdbc properties file: " + jdbcPropertiesFile + ", dbtype: " + dbTypeName); } // Right now only Derby and MySQL supported databaseType = DatabaseType.getByName(dbTypeName); switch (databaseType) { case DERBY: case MYSQL: break; default: throw new JdbcChannelException("Database " + databaseType + " not supported at this time"); } // Register driver if (driverClassName == null || driverClassName.trim().length() == 0) { throw new JdbcChannelException("No jdbc driver specified"); } try { Class.forName(driverClassName); } catch (ClassNotFoundException ex) { throw new JdbcChannelException("Unable to load driver: " + driverClassName, ex); } // JDBC Properties Properties jdbcProps = new Properties(); if (jdbcPropertiesFile != null && jdbcPropertiesFile.trim().length() > 0) { File jdbcPropsFile = new File(jdbcPropertiesFile.trim()); if (!jdbcPropsFile.exists()) { throw new JdbcChannelException("Jdbc properties file does not exist: " + jdbcPropertiesFile); } InputStream inStream = null; try { inStream = new FileInputStream(jdbcPropsFile); jdbcProps.load(inStream); } catch (IOException ex) { throw new JdbcChannelException( "Unable to load jdbc properties " + "from file: " + jdbcPropertiesFile, ex); } finally { if (inStream != null) { try { inStream.close(); } catch (IOException ex) { LOGGER.error("Unable to close file: " + jdbcPropertiesFile, ex); } } } } if (userName != null) { Object oldUser = jdbcProps.put("user", userName); if (oldUser != null) { LOGGER.warn("Overriding user from: " + oldUser + " to: " + userName); } } if (password != null) { Object oldPass = jdbcProps.put("password", password); if (oldPass != null) { LOGGER.warn( "Overriding password from the jdbc properties with " + " the one specified explicitly."); } } if (LOGGER.isDebugEnabled()) { StringBuilder sb = new StringBuilder("JDBC Properties {"); boolean first = true; Enumeration<?> propertyKeys = jdbcProps.propertyNames(); while (propertyKeys.hasMoreElements()) { if (first) { first = false; } else { sb.append(", "); } String key = (String) propertyKeys.nextElement(); sb.append(key).append("="); if (key.equalsIgnoreCase("password")) { sb.append("*******"); } else { sb.append(jdbcProps.get(key)); } } sb.append("}"); LOGGER.debug(sb.toString()); } // Transaction Isolation String txIsolation = getConfigurationString(context, ConfigurationConstants.CONFIG_TX_ISOLATION_LEVEL, ConfigurationConstants.OLD_CONFIG_TX_ISOLATION_LEVEL, TransactionIsolation.READ_COMMITTED.getName()); TransactionIsolation txIsolationLevel = TransactionIsolation.getByName(txIsolation); LOGGER.debug("Transaction isolation will be set to: " + txIsolationLevel); // Setup Datasource ConnectionFactory connFactory = new DriverManagerConnectionFactory(connectUrl, jdbcProps); connectionPool = new GenericObjectPool(); String maxActiveConnections = getConfigurationString(context, ConfigurationConstants.CONFIG_MAX_CONNECTIONS, ConfigurationConstants.OLD_CONFIG_MAX_CONNECTIONS, "10"); int maxActive = 10; if (maxActiveConnections != null && maxActiveConnections.length() > 0) { try { maxActive = Integer.parseInt(maxActiveConnections); } catch (NumberFormatException nfe) { LOGGER.warn("Max active connections has invalid value: " + maxActiveConnections + ", Using default: " + maxActive); } } LOGGER.debug("Max active connections for the pool: " + maxActive); connectionPool.setMaxActive(maxActive); statementPool = new GenericKeyedObjectPoolFactory(null); // Creating the factory instance automatically wires the connection pool new PoolableConnectionFactory(connFactory, connectionPool, statementPool, databaseType.getValidationQuery(), false, false, txIsolationLevel.getCode()); dataSource = new PoolingDataSource(connectionPool); txFactory = new JdbcTransactionFactory(dataSource, this); } /** * A callback method invoked from individual transaction instances after * a successful commit. The argument passed is the net number of events to * be added to the current size as tracked by the provider. * @param delta the net number of events to be added to reflect the current * size of the channel */ protected void updateCurrentChannelSize(long delta) { long currentSizeLong = currentSize.addAndGet(delta); LOGGER.debug("channel size updated to: " + currentSizeLong); } /** * Helper method to transition the configuration from the old long form * style configuration to the new short form. If the value is specified for * both the old and the new forms, the one associated with the new form * takes precedence. * * @param context * @param key the expected configuration key * @param oldKey the deprecated configuration key * @param default value, null if no default * @return the value associated with the key */ private String getConfigurationString(Context context, String key, String oldKey, String defaultValue) { String oldValue = context.getString(oldKey); if (oldValue != null && oldValue.length() > 0) { LOGGER.warn("Long form configuration key \"" + oldKey + "\" is deprecated. Please use the short form key \"" + key + "\" instead."); } String value = context.getString(key); if (value == null) { if (oldValue != null) { value = oldValue; } else { value = defaultValue; } } return value; } }