Here you can find the source of mergeConsecutiveTokens(Iterable
public static Iterable<String> mergeConsecutiveTokens(Iterable<String> tokens)
//package com.java2s; /*//from w ww . j av a 2s . com * Copyright 2014 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ import java.util.Iterator; public class Main { public static Iterable<String> mergeConsecutiveTokens(Iterable<String> tokens) { return new Iterable<String>() { @Override public Iterator<String> iterator() { /* merge consecutive types if they are not words */ final Iterator<String> token_iter = tokens.iterator(); return new Iterator<String>() { Iterator<String> iter = token_iter; String _last_token = null; String _token; @Override public boolean hasNext() { if (!iter.hasNext()) { return false; } _token = iter.next(); if (_last_token == null) { _last_token = _token; return true; } while (_last_token.equals(_token) && iter.hasNext()) _token = iter.next(); if (_last_token.equals(_token)) return false; _last_token = _token; return true; } @Override public String next() { return _token; } }; } }; } }