org.asimba.engine.tgt.jgroups.JGroupsTGTFactoryTest.java Source code

Java tutorial

Introduction

Here is the source code for org.asimba.engine.tgt.jgroups.JGroupsTGTFactoryTest.java

Source

/*
 * Asimba Server
 * 
 * Copyright (C) 2015 Asimba
 * 
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 * 
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 * 
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see www.gnu.org/licenses
 * 
 * Asimba - Serious Open Source SSO - More information on www.asimba.org
 * 
 */
package org.asimba.engine.tgt.jgroups;

import java.io.InputStream;
import java.math.BigInteger;
import java.security.SecureRandom;
import java.util.Arrays;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.Scanner;
import java.util.Set;

import org.apache.commons.lang3.SerializationUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.jgroups.JChannel;
import org.jgroups.blocks.ReplicatedHashMap;
import org.junit.After;
import org.junit.Before;
import org.junit.FixMethodOrder;
import org.junit.Test;
import org.junit.runners.MethodSorters;

import static org.hamcrest.core.IsEqual.equalTo;
import static org.hamcrest.core.IsNot.not;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertFalse;
import static org.mockito.Mockito.*;

import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.MockitoAnnotations;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
import org.w3c.dom.Element;

import com.alfaariss.oa.api.configuration.IConfigurationManager;
import com.alfaariss.oa.api.tgt.ITGT;
import com.alfaariss.oa.api.user.IUser;

import org.asimba.engine.cluster.JGroupsCluster;

import com.alfaariss.oa.engine.core.configuration.ConfigurationManager;
import com.alfaariss.oa.engine.core.tgt.factory.ITGTAliasStore;

@FixMethodOrder(MethodSorters.NAME_ASCENDING)
public class JGroupsTGTFactoryTest {
    private static final Log _oLogger = LogFactory.getLog(JGroupsTGTFactoryTest.class);

    private static final String FILENAME_CONFIG = "jgroupsfactory-config-ok.xml";
    private static final String FILENAME_CONFIG_NONBLOCKING = "jgroupsfactory-config-nonblocking.xml";

    private static final long EXPIRATION_FOR_TEST = 500000;

    // current implementation of setNextFillBytes supports up to 255 unique values :(
    private static final long MAX_FILLBYTES_VALUE = 255;
    private static long nextBytesFillValue = 0;

    private static final String IDP_TYPE = "IDP";
    private static final String SP_TYPE = "SP";
    private static final String SOME_REQUESTOR = "someRequestor";
    private static final String SOME_ALIAS = "someAlias";

    @Mock
    SecureRandom mockedSecureRandom;

    @Mock
    IUser mockedUser;

    // nodenames in AvailableNodeNames must also be configured in FILENAME_CONFIG
    String[] AvailableNodeNames = { "one", "two", "three", "four", "five" };
    JGroupsTGTFactory[] Factories = new JGroupsTGTFactory[AvailableNodeNames.length];
    ITGTAliasStore[] SpStores = new ITGTAliasStore[AvailableNodeNames.length];
    ITGTAliasStore[] IdpStores = new ITGTAliasStore[AvailableNodeNames.length];

    @Before
    public void before() {
        MockitoAnnotations.initMocks(this);
        nextBytesFillValue = 0l;

        mockedUser = Mockito.mock(IUser.class, withSettings().serializable());

        doAnswer(new Answer<Void>() {
            @Override
            public Void answer(InvocationOnMock invocation) {
                byte[] bytes = (byte[]) invocation.getArguments()[0];
                setNextBytesAnswer(bytes, ++nextBytesFillValue);
                return null;
            }
        }).when(mockedSecureRandom).nextBytes(any(byte[].class));

        when(mockedUser.getID()).thenReturn("mockedUserID");
        when(mockedUser.getOrganization()).thenReturn("mockedUserOrganization");

        for (int i = 0; i < AvailableNodeNames.length; ++i) {
            Factories[i] = null;
        }
    }

    public void setNextBytesAnswer(byte[] bytes, long value) {
        byte[] id = BigInteger.valueOf(value).toByteArray();
        System.arraycopy(id, 0, bytes, 0, (id.length < 30) ? id.length : 30);
    }

    @After
    public void after() throws Exception {
        for (int i = 0; i < AvailableNodeNames.length; ++i) {
            if (Factories[i] != null) {
                //_oLogger.debug("Cleaning factory " + i);
                //cleanTheFactory(Factories[i]);
                Factories[i].stop();
                Factories[i] = null;
                SpStores[i] = null;
                IdpStores[i] = null;
            }
        }
    }

    @Test
    public void test0_FillBytesUnique() {
        long firstValue = 1l;

        byte[] firstId = new byte[ITGT.TGT_LENGTH];
        setNextBytesAnswer(firstId, firstValue);

        byte[] nextId = new byte[ITGT.TGT_LENGTH];
        for (long nextValue = firstValue + 1; nextValue < MAX_FILLBYTES_VALUE; ++nextValue) {
            setNextBytesAnswer(nextId, nextValue);
            if (Arrays.equals(nextId, firstId)) {
                _oLogger.error("Id's are equal for nextValue " + nextValue);
            }
            assertThat(Arrays.equals(nextId, firstId), equalTo(false));
        }

    }

    /**
     * Isolates problems related to serializability of JGroupsTGT
     * @throws Exception
     */
    @Test
    public void test01_JGroupsTGTSerializable() throws Exception {
        JGroupsTGTFactory oTGTFactory = createJGroupsTGTFactory(0, EXPIRATION_FOR_TEST, FILENAME_CONFIG);
        JGroupsTGT oTGT = (JGroupsTGT) oTGTFactory.createTGT(mockedUser);

        try {
            SerializationUtils.serialize(oTGT);
        } catch (Exception e) {
            _oLogger.error("Object of class JGroupsTGT cannot be serialized", e);
            assertThat(true, equalTo(false)); // or the universe implodes
        }
    }

    @Test
    public void test01a_JGroupsTGTDefaultConfiguration() throws Exception {
        JGroupsTGTFactory oTGTFactory = createJGroupsTGTFactory(0, EXPIRATION_FOR_TEST, FILENAME_CONFIG);

        assertThat(oTGTFactory.isBlockingUpdates(), equalTo(true));
        assertThat(oTGTFactory.getTimeout(),
                equalTo(new ReplicatedHashMap<String, String>(new JChannel()).getTimeout()));
        assertThat(oTGTFactory.getAliasMapRetries(), equalTo(JGroupsTGTFactory.ALIASMAP_RETRIES_DEFAULT));
        assertThat(oTGTFactory.getAliasMapTimeout(), equalTo(JGroupsTGTFactory.ALIASMAP_TIMEOUT_DEFAULT));
        assertThat(oTGTFactory.isAliasMapFailureLogging(), equalTo(JGroupsTGTFactory.ALIASMAP_LOGGING_DEFAULT));
    }

    @Test
    public void test01b_JGroupsTGTNonBlockingConfiguration() throws Exception {
        JGroupsTGTFactory oTGTFactory = createJGroupsTGTFactory(0, EXPIRATION_FOR_TEST,
                FILENAME_CONFIG_NONBLOCKING);

        assertThat(oTGTFactory.isBlockingUpdates(), equalTo(false));
        assertThat(oTGTFactory.getTimeout(), equalTo(100l));
        assertThat(oTGTFactory.getAliasMapRetries(), equalTo(20));
        assertThat(oTGTFactory.getAliasMapTimeout(), equalTo(15l));
        assertThat(oTGTFactory.isAliasMapFailureLogging(), equalTo(true));
        // make sure we do not happen to test for the defaults
        assertThat(oTGTFactory.getAliasMapRetries(), not(equalTo(JGroupsTGTFactory.ALIASMAP_RETRIES_DEFAULT)));
        assertThat(oTGTFactory.getAliasMapTimeout(), not(equalTo(JGroupsTGTFactory.ALIASMAP_TIMEOUT_DEFAULT)));
        assertThat(oTGTFactory.isAliasMapFailureLogging(),
                not(equalTo(JGroupsTGTFactory.ALIASMAP_LOGGING_DEFAULT)));
    }

    /**
     * Isolates basic problems with using a very simple ReplicatedHasMap
     * @throws Exception
     */
    @Test
    public void test02_BasicReplicatedHashMapWithStringStringMap() throws Exception {
        //channel.connect("HashmapCluster");
        try (JChannel channel = createChannelFromConfig()) {
            //channel.connect("HashmapCluster");
            ReplicatedHashMap<String, String> map = new ReplicatedHashMap<>(channel);

            map.setBlockingUpdates(true);
            map.put("test", "test");
            assertThat(map.get("test"), not(equalTo(null)));
            assertThat(map.size(), equalTo(1));
            map.clear();
            map.stop();
        }
    }

    /**
     * Isolates basic problems with using a ReplicatedHasMap for JGroupsTGT
     * @throws Exception
     */
    @Test
    public void test03_BasicReplicatedHashMapWithStringTGTMap() throws Exception {
        //channel.connect("HashmapCluster");
        try (JChannel channel = createChannelFromConfig()) {
            //channel.connect("HashmapCluster");
            ReplicatedHashMap<String, JGroupsTGT> map = new ReplicatedHashMap<>(channel);
            JGroupsTGTFactory oTGTFactory = createJGroupsTGTFactory(0, EXPIRATION_FOR_TEST, FILENAME_CONFIG);
            JGroupsTGT oTGT = (JGroupsTGT) oTGTFactory.createTGT(mockedUser);

            map.setBlockingUpdates(true);
            map.put("test", oTGT);
            JGroupsTGT retrievedTGT = map.get("test");
            assertThat(retrievedTGT, not(equalTo(null)));

            assertThat(map.size(), equalTo(1));
            map.clear();
            map.stop();
        }
    }

    @Test
    public void test04_OneNodeOneTGT() throws Exception {
        _oLogger.debug("test04_OneNodeOneTGT");
        testNTGTFactories(1, 1);
    }

    @Test
    public void test05_TwoNodesOneTGT() throws Exception {
        testNTGTFactories(2, 1);
    }

    @Test
    public void test06_FiveNodesOneTGT() throws Exception {
        testNTGTFactories(5, 1);
    }

    @Test
    public void test07_TwoNodesManyTGTs() throws Exception {
        testNTGTFactories(2, 100);
    }

    @Test
    public void test08_MaxNodesMaxTGTs() throws Exception {
        testNTGTFactories(AvailableNodeNames.length, (int) MAX_FILLBYTES_VALUE);
    }

    @Test
    public void test08a_MaxNodesManyTGTs() throws Exception {
        testNTGTFactories(AvailableNodeNames.length, (int) 1000); // TODO: increase and analyze
        _oLogger.info("Total number of unique TGTs: " + Factories[0].size());
    }

    /**
     * keep this test switched off under normal circumstances, it is more a performance test
     * @throws Exception
     */
    //@Test
    public void test08b_TestHowManyItWillDo() throws Exception {
        reportMemory();
        testNTGTFactories(2, (int) 75000); // TODO: increase and analyze
        _oLogger.debug("Total number of unique TGTs: " + Factories[0].size());
        reportMemory();
    }

    @Test
    public void test09_RunTwoNodesAndAddOne() throws Exception {
        final int nTGTs = 100;
        final int expectedTGTs = nTGTs * 2;
        testNTGTFactories(2, nTGTs);
        assertThat(Factories[0].size(), equalTo(expectedTGTs));
        assertThat(Factories[1].size(), equalTo(expectedTGTs));
        assertThat(Factories[2], equalTo(null));
        createFactory(2);
        JGroupsTGTFactory addedFactory = Factories[2];
        assertThat(addedFactory, not(equalTo(null)));
        assertThat(addedFactory.size(), equalTo(expectedTGTs));
        JGroupsTGT newTGT = (JGroupsTGT) addedFactory.createTGT(mockedUser);
        addedFactory.persist(newTGT);
        assertThat(addedFactory.size(), equalTo(expectedTGTs + 1));
        assertThat(Factories[0].size(), equalTo(expectedTGTs + 1));
        assertThat(Factories[1].size(), equalTo(expectedTGTs + 1));
    }

    @Test
    public void test10_RunTwoNodesAddOneAndStopStartOne() throws Exception {
        final int nTGTs = 100;
        final int expectedTGTs = nTGTs * 2;
        testNTGTFactories(2, nTGTs);
        createFactory(2);
        assertThat(Factories[2].size(), equalTo(expectedTGTs));
        JGroupsTGTFactory restartFactory = Factories[1];
        restartFactory.stop();
        JGroupsTGT newTGT = (JGroupsTGT) Factories[0].createTGT(mockedUser);
        Factories[0].persist(newTGT);
        assertThat(Factories[2].size(), equalTo(expectedTGTs + 1));
        assertThat(restartFactory.size(), equalTo(expectedTGTs));
        restartFactory.start();
        assertThat(restartFactory.size(), equalTo(expectedTGTs + 1));
    }

    /**
     * Test removal of expired TGTs
     * @throws java.lang.Exception
     */
    @Test
    public void test11_RemoveExpiredTGT() throws Exception {
        JGroupsTGTFactory oTGTFactory = createJGroupsTGTFactory(0, 1000, FILENAME_CONFIG);
        JGroupsTGT oTGT = (JGroupsTGT) oTGTFactory.createTGT(mockedUser);

        oTGTFactory.persist(oTGT);
        assertTrue(oTGTFactory.exists(oTGT.getId()));

        final String alias = SOME_ALIAS + "-sp";
        ITGTAliasStore spStore = oTGTFactory.getAliasStoreSP();
        oTGTFactory.getAliasStoreSP().putAlias(SP_TYPE, SOME_REQUESTOR, oTGT.getId(), alias);
        assertThat(spStore.isAlias(SP_TYPE, SOME_REQUESTOR, alias), equalTo(true));

        oTGTFactory.removeExpired();
        assertTrue(oTGTFactory.exists(oTGT.getId()));
        assertThat(spStore.isAlias(SP_TYPE, SOME_REQUESTOR, alias), equalTo(true));

        Thread.sleep(1000);

        oTGTFactory.removeExpired();

        if (!oTGTFactory.isBlockingUpdates()) {
            Thread.sleep(1000);
        }
        assertFalse(oTGTFactory.exists(oTGT.getId()));
        assertThat(spStore.isAlias(SP_TYPE, SOME_REQUESTOR, alias), equalTo(false));
    }

    private void testNTGTFactories(int nNodes, int nTGTs) throws Exception {
        int firstFreeNode = getFirstUnusedNode();
        if (firstFreeNode + nNodes > AvailableNodeNames.length) {
            _oLogger.error("Not enough unused nodes left");
            throw new Exception("Not enough unused nodes left");
        }

        /*if (nTGTs < 1 || nTGTs > MAX_FILLBYTES_VALUE) {
           _oLogger.error("Invalid number of TGTs: " + nTGTs);
           throw new Exception("Invalid number of TGTs: " + nTGTs);
        }*/

        for (int i = firstFreeNode; i < nNodes; ++i) {
            createFactory(i);
        }

        RetrieveRepeater slowRepeater = new RetrieveRepeater(10, 1000);
        RetrieveRepeater quickRepeater = new RetrieveRepeater(10, 10);

        int persisted = 0;
        for (int i = 0; i < nNodes; ++i) {
            for (int j = 0; j < nTGTs; ++j) {
                JGroupsTGT tgt = (JGroupsTGT) Factories[i].createTGT(mockedUser);
                Factories[i].persist(tgt);
                String requestor = i + "-" + SOME_REQUESTOR;
                for (int k = 0; k < nNodes; ++k) {
                    //JGroupsTGT rTGT = Factories[k].retrieve(tgt.getId());
                    JGroupsTGT rTGT = quickRepeater.retrieve(Factories[k], tgt.getId());
                    if (rTGT == null) {
                        _oLogger.debug(
                                "No luck retrieving tgt '" + tgt.getId() + "', will wait for max 10 seconds");
                        rTGT = slowRepeater.retrieve(Factories[k], tgt.getId());
                    }
                    if (rTGT == null) {
                        _oLogger.error("=========== start of Factory dump due to upcoming error ==========");
                        Factories[i].stop();
                        _oLogger.error("=========== end of Factory dump due to upcoming error ==========");
                    } else {
                        assertThat("Assertion failed at (i,j,k): " + i + "," + j + "," + k, rTGT,
                                not(equalTo(null)));
                        assertThat(rTGT.getId(), equalTo(tgt.getId()));
                    }
                }
                if (++persisted % 10000 == 0) {
                    _oLogger.info("Persisted: " + persisted + "/" + (nNodes * nTGTs) + " (TGTs: "
                            + Factories[0].size() + ")");
                }
                String alias = SOME_ALIAS + "-sp-" + j;
                SpStores[i].putAlias(SP_TYPE, requestor, tgt.getId(), alias);
                assertThat(SpStores[i].getAlias(SP_TYPE, requestor, tgt.getId()), equalTo(alias));
                for (int l = 0; l < nNodes; ++l) {
                    String rAlias = SpStores[l].getAlias(SP_TYPE, requestor, tgt.getId());
                    //String rAlias = quickRepeater.getAlias(SpStores[l], SP_TYPE, requestor, tgt.getId());
                    if (rAlias == null) {
                        _oLogger.debug("No luck getting alias '" + SP_TYPE + "-" + requestor + "-" + tgt.getId()
                                + "', will wait for max 10 seconds");
                        rAlias = slowRepeater.getAlias(SpStores[l], SP_TYPE, requestor, tgt.getId());
                    }
                    if (rAlias == null) {
                        _oLogger.error("=========== start of Factory dump due to upcoming error ==========");
                        Factories[i].stop();
                        _oLogger.error("=========== end of Factory dump due to upcoming error ==========");
                    }
                    assertThat("Fails for node (i,j,l): " + i + "," + j + "," + l, rAlias, equalTo(alias));
                }
                alias = SOME_ALIAS + "-idp-" + j;
                IdpStores[i].putAlias(IDP_TYPE, requestor, tgt.getId(), alias);
                assertThat("Fails for node (i,j): " + i + "," + j,
                        IdpStores[i].getAlias(IDP_TYPE, requestor, tgt.getId()), equalTo(alias));
                for (int l = 0; l < nNodes; ++l) {
                    String rAlias = IdpStores[l].getAlias(IDP_TYPE, requestor, tgt.getId());
                    //String rAlias = quickRepeater.getAlias(IdpStores[l], IDP_TYPE, requestor, tgt.getId());
                    if (rAlias == null) {
                        _oLogger.debug("No luck getting alias '" + IDP_TYPE + "-" + requestor + "-" + tgt.getId()
                                + "', will wait for max 10 seconds");
                        rAlias = slowRepeater.getAlias(IdpStores[l], IDP_TYPE, requestor, tgt.getId());
                    }
                    if (rAlias == null) {
                        _oLogger.error("=========== start of Factory dump due to upcoming error ==========");
                        Factories[i].stop();
                        _oLogger.error("=========== end of Factory dump due to upcoming error ==========");
                    }
                    assertThat("Fails for node (i,j,l): " + i + "," + j + "," + l, rAlias, equalTo(alias));
                }
            }
        }
        for (int i = 0; i < nNodes; ++i) {
            assertThat(Factories[i].size(), equalTo(persisted));
        }

        //quickRepeater.logReport(_oLogger);
        //slowRepeater.logReport(_oLogger);
    }

    private void createFactory(int i) throws Exception {
        Factories[i] = createJGroupsTGTFactory(i, EXPIRATION_FOR_TEST, FILENAME_CONFIG);
        SpStores[i] = Factories[i].getAliasStoreSP();
        IdpStores[i] = Factories[i].getAliasStoreIDP();
    }

    private int getFirstUnusedNode() throws Exception {
        for (int i = 0; i < AvailableNodeNames.length; ++i) {
            if (Factories[i] == null) {
                return i;
            }
        }

        String message = "No unused nodes left";
        _oLogger.error(message);
        throw new Exception(message);
    }

    private void cleanTheFactory(JGroupsTGTFactory oTGTFactory) throws Exception {
        Set<Entry<String, JGroupsTGT>> entries = oTGTFactory.entrySet();
        for (Entry<String, JGroupsTGT> entry : entries) {
            oTGTFactory.clean(entry.getValue());
        }
    }

    private JGroupsTGTFactory createJGroupsTGTFactory(int n, long expiration, String configFile) throws Exception {
        String id = AvailableNodeNames[n];
        System.setProperty(JGroupsCluster.PROP_ASIMBA_NODE_ID, id);

        IConfigurationManager oConfigManager = readConfigElementFromResource(configFile);

        Element eClusterElement = oConfigManager.getSection(null, "cluster", "id=test");
        assertThat(eClusterElement, not(equalTo(null)));

        Element eAliasClusterElement = oConfigManager.getSection(null, "alias-cluster", "id=test-alias");
        assertThat(eAliasClusterElement, not(equalTo(null)));

        JGroupsCluster oCluster = new JGroupsCluster();
        oCluster.start(oConfigManager, eClusterElement);
        JChannel jChannel = (JChannel) oCluster.getChannel();
        jChannel.connect("Something");
        assertThat(jChannel, not(equalTo(null)));
        _oLogger.info("JCluster address:" + jChannel.getAddressAsString());
        JGroupsCluster oAliasCluster = new JGroupsCluster();
        oAliasCluster.start(oConfigManager, eAliasClusterElement);

        JGroupsTGTFactory oTGTFactory = Factories[n] = new JGroupsTGTFactory();
        oTGTFactory.startForTesting(oConfigManager, eClusterElement, oCluster, oAliasCluster, mockedSecureRandom,
                expiration != 0 ? expiration : EXPIRATION_FOR_TEST);
        //oTGTFactory.setAliasMapFailureLogging(true, JGroupsTGTAliasStoreTest.class.getName());

        return oTGTFactory;
    }

    private JChannel createChannelFromConfig() throws Exception {
        IConfigurationManager oConfigManager = readConfigElementFromResource(FILENAME_CONFIG);

        Element eClusterElement = oConfigManager.getSection(null, "cluster", "id=test");
        assertThat(eClusterElement, not(equalTo(null)));

        JGroupsCluster cluster = new JGroupsCluster();
        cluster.start(oConfigManager, eClusterElement);
        JChannel jChannel = (JChannel) cluster.getChannel();

        return jChannel;
    }

    private IConfigurationManager readConfigElementFromResource(String filename) throws Exception {

        InputStream oIS = JGroupsTGTFactory.class.getClassLoader().getResourceAsStream(filename);
        String sConfig;

        try (Scanner s = new Scanner(oIS, "UTF-8")) {
            s.useDelimiter("\\A");
            sConfig = s.next();
        }

        //_oLogger.info("XML Read: [" + sConfig + "]");

        Properties oProperties = new Properties();
        oProperties.put("configuration.handler.class",
                "com.alfaariss.oa.util.configuration.handler.text.PlainTextConfigurationHandler");
        oProperties.put("config", sConfig);

        ConfigurationManager oConfigManager = ConfigurationManager.getInstance();
        oConfigManager.start(oProperties);

        return oConfigManager;
    }

    private void reportMemory() {
        final int MegaBytes = 10241024;
        long freeMemory = Runtime.getRuntime().freeMemory() / MegaBytes;
        long totalMemory = Runtime.getRuntime().totalMemory() / MegaBytes;
        long maxMemory = Runtime.getRuntime().maxMemory() / MegaBytes;

        _oLogger.debug("Used Memory in JVM: " + (maxMemory - freeMemory));
        _oLogger.debug("freeMemory in JVM: " + freeMemory);
        _oLogger.debug("totalMemory in JVM shows current size of java heap : " + totalMemory);
        _oLogger.debug("maxMemory in JVM: " + maxMemory);

    }

    private interface Store {
        public Object get() throws Exception;
    }

    private class RetrieveRepeater {
        public final static int TYPE_TGT = 1;
        public final static int TYPE_IDP = 2;
        public final static int TYPE_SP = 3;
        private final int repeats;
        private final int sleep;
        private int invocations;
        private final int[] repetitions;
        private int failures;

        public RetrieveRepeater(int repeats, int sleep) {
            this.repeats = repeats;
            this.sleep = sleep;
            this.invocations = 0;
            this.failures = 0;
            this.repetitions = new int[repeats];
        }

        public String getAlias(final ITGTAliasStore store, final String type, final String entityId,
                final String tgtId) throws Exception {
            return (String) repeat(new Store() {
                @Override
                public Object get() throws Exception {
                    return store.getAlias(type, entityId, tgtId);
                }
            });
        }

        public JGroupsTGT retrieve(final JGroupsTGTFactory factory, final String key) throws Exception {
            return (JGroupsTGT) repeat(new Store() {
                @Override
                public Object get() throws Exception {
                    return factory.retrieve(key);
                }
            });
        }

        private Object repeat(Store store) throws Exception {
            if (repeats <= 0) {
                return store.get();
            }
            int cycle = 0;
            Object result;

            invocations++;
            while ((result = store.get()) == null && cycle < this.repeats) {
                Thread.sleep(cycle * this.sleep);
                ++cycle;
            }
            this.repetitions[cycle] += 1;

            if (result == null) {
                ++failures;
                _oLogger.debug("failure for type: ");
            }

            return result;
        }

        public void logReport(Log logger) {
            logger.info("");
            logger.info(RetrieveRepeater.class.toString());
            logger.info("Invocations: " + invocations);
            logger.info("Successes per cycle (0 means direct succes):");
            for (int i = 0; i < repetitions.length; ++i) {
                logger.info("  " + i + " after " + (i * sleep) + " msecs: " + repetitions[i]);
            }
            logger.info("Failures: " + failures);
            logger.info("");
        }
    }
}