Java tutorial
/* * (C) 2007-2012 Alibaba Group Holding Limited. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * Authors: * wuhua <wq163@163.com> , boyan <killme2008@gmail.com> */ package com.b5m.raindrop.source.metaq; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import backtype.storm.spout.Scheme; import backtype.storm.spout.SpoutOutputCollector; import backtype.storm.task.TopologyContext; import backtype.storm.topology.IRichSpout; import backtype.storm.topology.OutputFieldsDeclarer; import com.taobao.gecko.core.util.LinkedTransferQueue; import com.taobao.metamorphosis.Message; import com.taobao.metamorphosis.client.MessageSessionFactory; import com.taobao.metamorphosis.client.MetaClientConfig; import com.taobao.metamorphosis.client.MetaMessageSessionFactory; import com.taobao.metamorphosis.client.consumer.ConsumerConfig; import com.taobao.metamorphosis.client.consumer.MessageConsumer; import com.taobao.metamorphosis.client.consumer.MessageListener; import com.taobao.metamorphosis.exception.MetaClientException; import com.taobao.metamorphosis.utils.ZkUtils.ZKConfig; /** * * * @author Jacky Liu, LiBin Jin * @date 2011-11-8 * */ public class MetaSpout implements IRichSpout { private static final long serialVersionUID = 4382748324382L; public static final String FETCH_MAX_SIZE = "meta.fetch.max_size"; public static final String TOPIC = "meta.topic"; public static final int DEFAULT_MAX_SIZE = 128 * 1024; private transient MessageConsumer messageConsumer; private transient MessageSessionFactory sessionFactory; private final MetaClientConfig metaClientConfig; private final ConsumerConfig consumerConfig; static final Log log = LogFactory.getLog(MetaSpout.class); private final Scheme scheme; /** * Time in milliseconds to wait for a message from the queue if there is no * message ready when the topology requests a tuple (via * {@link #nextTuple()}). */ public static final long WAIT_FOR_NEXT_MESSAGE = 1L; private transient ConcurrentHashMap<Long, MetaMessageWrapper> id2wrapperMap; private transient SpoutOutputCollector collector; private transient LinkedTransferQueue<MetaMessageWrapper> messageQueue; /** * ??? */ private AtomicLong receivedCount; public MetaSpout(final MetaClientConfig metaClientConfig, final ConsumerConfig consumerConfig, final Scheme scheme) { super(); this.metaClientConfig = metaClientConfig; this.consumerConfig = consumerConfig; this.scheme = scheme; } private void logMetaClientConfig(MetaClientConfig config) { if (!log.isDebugEnabled()) return; log.debug(new StringBuilder("Meta Client Config:{").append(config.getServerUrl()) .append(", diamondPartitionsDataId->").append(config.getDiamondPartitionsDataId()) .append(", diamondPartitionsGroup->").append(config.getDiamondPartitionsGroup()) .append(", recoverMessageIntervalInMills->").append(config.getRecoverMessageIntervalInMills()) .append(", recoverThreadCount->").append(config.getRecoverThreadCount()) .append(", partitionsInfo->").append(config.getPartitionsInfo()).append(", ") .append(logZkConfig(config.getZkConfig())).append("}").toString()); } private String logZkConfig(ZKConfig config) { if (null == config) return "zkConfig->null"; return new StringBuilder("zkConfig:{").append(config.zkConnect).append(", zkConnectionTimeoutMs->") .append(config.zkConnectionTimeoutMs).append(", zkEnable->").append(config.zkEnable) .append(", zkRoot->").append(config.zkRoot).append(", zkSessionTimeoutMs->") .append(config.zkSessionTimeoutMs).append(", zkConfig{zkConnect->").append(config.zkSyncTimeMs) .append("} ").toString(); } private void logConsumerConfig(ConsumerConfig consumerConfig) { if (!log.isDebugEnabled()) return; log.debug(new StringBuilder("Consumer Config:{consumerId->").append(consumerConfig.getConsumerId()) .append(", commitOffsetPeriodInMills->").append(consumerConfig.getCommitOffsetPeriodInMills()) .append(", diamondPartitionsDataId->").append(consumerConfig.getDiamondPartitionsDataId()) .append(", diamondPartitionsGroup->").append(consumerConfig.getDiamondPartitionsGroup()) .append(", fetchRunnerCount->").append(consumerConfig.getFetchRunnerCount()) .append(", fetchTimeoutInMills->").append(consumerConfig.getFetchTimeoutInMills()) .append(", group->").append(consumerConfig.getGroup()).append(", maxDelayFetchTimeInMills->") .append(consumerConfig.getMaxDelayFetchTimeInMills()).append(", maxFetchRetries->") .append(consumerConfig.getMaxFetchRetries()).append(", maxIncreaseFetchDataRetries->") .append(consumerConfig.getMaxIncreaseFetchDataRetries()).append(", offset->") .append(consumerConfig.getOffset()).append(", partition->").append(consumerConfig.getPartition()) .append(", recoverMessageIntervalInMills->") .append(consumerConfig.getRecoverMessageIntervalInMills()).append(", recoverThreadCount->") .append(consumerConfig.getRecoverThreadCount()).append(", serverUrl->") .append(consumerConfig.getServerUrl()).append(", loadBalanceStrategyType->") .append(consumerConfig.getLoadBalanceStrategyType()).append(", partitionsInfo->") .append(consumerConfig.getPartitionsInfo()).append(", ") .append(logZkConfig(consumerConfig.getZkConfig())).append("}").toString()); } public void open(final Map conf, final TopologyContext context, final SpoutOutputCollector collector) { final String topic = (String) conf.get(TOPIC); if (topic == null) { throw new IllegalArgumentException(TOPIC + " is null"); } Integer maxSize = (Integer) conf.get(FETCH_MAX_SIZE); if (maxSize == null) { log.warn("Using default FETCH_MAX_SIZE"); maxSize = DEFAULT_MAX_SIZE; } this.id2wrapperMap = new ConcurrentHashMap<Long, MetaMessageWrapper>(); this.messageQueue = new LinkedTransferQueue<MetaMessageWrapper>(); receivedCount = new AtomicLong(0); try { this.collector = collector; logConsumerConfig(this.consumerConfig); logMetaClientConfig(this.metaClientConfig); this.setUpMeta(topic, maxSize); } catch (final MetaClientException e) { log.error("Setup meta consumer failed", e); } } private void setUpMeta(final String topic, final Integer maxSize) throws MetaClientException { this.sessionFactory = new MetaMessageSessionFactory(this.metaClientConfig); this.messageConsumer = this.sessionFactory.createConsumer(this.consumerConfig); this.messageConsumer.subscribe(topic, maxSize, new MessageListener() { public void recieveMessages(final Message message) { final MetaMessageWrapper wrapper = new MetaMessageWrapper(message); MetaSpout.this.id2wrapperMap.put(message.getId(), wrapper); MetaSpout.this.messageQueue.offer(wrapper); if (log.isDebugEnabled()) { log.debug(new StringBuilder("InQueue : ").append(message.getId()).append(", partition:") .append(message.getPartition()).append(", total received count:") .append(receivedCount.incrementAndGet()).toString()); } try { wrapper.latch.await(); } catch (final InterruptedException e) { Thread.currentThread().interrupt(); } if (!wrapper.success) { throw new RuntimeException("Consume message failed"); } } public Executor getExecutor() { return null; } }).completeSubscribe(); } public void close() { try { this.messageConsumer.shutdown(); } catch (final MetaClientException e) { log.error("Shutdown consumer failed", e); } try { this.sessionFactory.shutdown(); } catch (final MetaClientException e) { log.error("Shutdown session factory failed", e); } } public void nextTuple() { if (this.messageConsumer != null) { try { final MetaMessageWrapper wrapper = this.messageQueue.poll(WAIT_FOR_NEXT_MESSAGE, TimeUnit.MILLISECONDS); if (wrapper == null) { return; } final Message message = wrapper.message; this.collector.emit(this.scheme.deserialize(message.getData()), message.getId()); } catch (final InterruptedException e) { // interrupted while waiting for message, big deal } } } public void ack(final Object msgId) { if (msgId instanceof Long) { final long id = (Long) msgId; final MetaMessageWrapper wrapper = this.id2wrapperMap.remove(id); if (wrapper == null) { log.warn(String.format("don't know how to ack(%s: %s)", msgId.getClass().getName(), msgId)); return; } wrapper.success = true; wrapper.latch.countDown(); } else { log.warn(String.format("don't know how to ack(%s: %s)", msgId.getClass().getName(), msgId)); } } public void fail(final Object msgId) { if (msgId instanceof Long) { final long id = (Long) msgId; final MetaMessageWrapper wrapper = this.id2wrapperMap.remove(id); if (wrapper == null) { log.warn(String.format("don't know how to reject(%s: %s)", msgId.getClass().getName(), msgId)); return; } wrapper.success = false; wrapper.latch.countDown(); } else { log.warn(String.format("don't know how to reject(%s: %s)", msgId.getClass().getName(), msgId)); } } public void declareOutputFields(final OutputFieldsDeclarer declarer) { declarer.declare(this.scheme.getOutputFields()); } public boolean isDistributed() { return true; } @Override public void activate() { // TODO Auto-generated method stub } @Override public void deactivate() { // TODO Auto-generated method stub } @Override public Map<String, Object> getComponentConfiguration() { // TODO Auto-generated method stub return null; } }