kafka.examples.Producer.java Source code

Introduction

Here is the source code for kafka.examples.Producer.java

Source

/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 * 
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package kafka.examples;

import java.io.IOException;
import java.io.StringReader;
import java.util.Properties;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.wltea.analyzer.lucene.IKAnalyzer;

import weibo4j.Timeline;
import weibo4j.examples.oauth2.Log;
import weibo4j.model.StatusWapper;
import weibo4j.model.WeiboException;
import kafka.producer.KeyedMessage;
import kafka.producer.ProducerConfig;

public class Producer extends Thread {
    private final kafka.javaapi.producer.Producer<Integer, String> producer;
    private final String topic;
    private final Properties props = new Properties();

    public Producer(String topic) {
        props.put("serializer.class", "kafka.serializer.StringEncoder");
        props.put("metadata.broker.list", "10.18.97.66:9092");
        // Use random partitioner. Don't need the key type. Just set it to Integer.
        // The message is of type String.
        producer = new kafka.javaapi.producer.Producer<Integer, String>(new ProducerConfig(props));
        this.topic = topic;
    }

    public void run() {
        while (true) {
            String access_token = "2.009F1d9BmHHChD7abcd6de0a0jui5Y";
            int count = 20;
            Timeline tm = new Timeline(access_token);
            Analyzer analyzer4 = new IKAnalyzer(false);// ?

            try {
                StatusWapper status = tm.getPublicTimeline(count, 0);
                //-------------------------------------------
                try {
                    TokenStream tokenstream = analyzer4.tokenStream("", new StringReader(status.toString()));
                    CharTermAttribute termAttribute = tokenstream.addAttribute(CharTermAttribute.class);// token

                    tokenstream.reset();// ?

                    while (tokenstream.incrementToken()) {// ??token
                        String prTxt = new String(termAttribute.buffer(), 0, termAttribute.length());
                        //producer.send(new KeyedMessage<Integer, String>(topic, ptTxt + " "));
                        System.out.print(prTxt + "  ");
                    }
                    //System.out.println();
                    tokenstream.close();//TokenStream
                } catch (IOException e) {
                    e.printStackTrace();
                }
                //-------------------------------------------
                producer.send(new KeyedMessage<Integer, String>(topic, status.toString()));
                Log.logInfo(status.toString());

            } catch (WeiboException e) {
                e.printStackTrace();
            }
        }
    }

}