org.apache.fluo.core.async.AsyncConditionalWriter.java Source code

Introduction

Here is the source code for org.apache.fluo.core.async.AsyncConditionalWriter.java

Source

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
 * agreements. See the NOTICE file distributed with this work for additional information regarding
 * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance with the License. You may obtain a
 * copy of the License at
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package org.apache.fluo.core.async;

import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.concurrent.Callable;
import java.util.concurrent.TimeUnit;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableList.Builder;
import com.google.common.util.concurrent.AsyncFunction;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import org.apache.accumulo.core.client.ConditionalWriter;
import org.apache.accumulo.core.client.ConditionalWriter.Result;
import org.apache.accumulo.core.data.ConditionalMutation;
import org.apache.fluo.core.impl.Environment;
import org.apache.fluo.core.impl.FluoConfigurationImpl;
import org.apache.fluo.core.util.FluoExecutors;
import org.apache.fluo.core.util.Limit;

public class AsyncConditionalWriter implements AsyncFunction<Collection<ConditionalMutation>, Iterator<Result>> {

    private final ConditionalWriter cw;
    private final ListeningExecutorService les;
    private final Limit semaphore;

    public AsyncConditionalWriter(Environment env, ConditionalWriter cw) {
        this.cw = cw;
        int numThreads = env.getConfiguration().getInt(FluoConfigurationImpl.ASYNC_CW_THREADS,
                FluoConfigurationImpl.ASYNC_CW_THREADS_DEFAULT);
        int permits = env.getConfiguration().getInt(FluoConfigurationImpl.ASYNC_CW_LIMIT,
                FluoConfigurationImpl.ASYNC_CW_LIMIT_DEFAULT);
        this.les = MoreExecutors.listeningDecorator(FluoExecutors.newFixedThreadPool(numThreads, "asyncCW"));
        // the conditional writer currently has not memory limits... give it too much and it blows out
        // memory.. need to fix this in conditional writer
        // for now this needs to be memory based
        this.semaphore = new Limit(permits);
    }

    private class IterTask implements Callable<Iterator<Result>> {

        private Iterator<Result> input;
        private int permitsAcquired;

        public IterTask(Iterator<Result> iter, int permitsAcquired) {
            this.input = iter;
            this.permitsAcquired = permitsAcquired;
        }

        @Override
        public Iterator<Result> call() throws Exception {
            try {
                Builder<Result> imlb = ImmutableList.builder();
                while (input.hasNext()) {
                    Result result = input.next();
                    imlb.add(result);
                }
                return imlb.build().iterator();
            } finally {
                semaphore.release(permitsAcquired);
            }
        }

    }

    @Override
    public ListenableFuture<Iterator<Result>> apply(Collection<ConditionalMutation> input) {
        if (input.size() == 0) {
            return Futures.immediateFuture(Collections.<Result>emptyList().iterator());
        }

        semaphore.acquire(input.size());
        Iterator<Result> iter = cw.write(input.iterator());
        return les.submit(new IterTask(iter, input.size()));
    }

    public void close() {
        les.shutdownNow();
        try {
            les.awaitTermination(5, TimeUnit.SECONDS);
        } catch (InterruptedException e) {
            throw new RuntimeException(e);
        }
    }
}