Example usage for org.apache.hadoop.util.bloom CountingBloomFilter CountingBloomFilter

List of usage examples for org.apache.hadoop.util.bloom CountingBloomFilter CountingBloomFilter

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.util.bloom.*
  5. CountingBloomFilter
  6. CountingBloomFilter

Introduction

In this page you can find the example usage for org.apache.hadoop.util.bloom CountingBloomFilter CountingBloomFilter.

Prototype

public CountingBloomFilter(int vectorSize, int nbHash, int hashType)

Source Link

Document

Constructor

Usage

From source file:com.eincs.athens.handler.AthensBlockFilter.java

License:Apache License

@Inject
public AthensBlockFilter(BlockDB blockDB) {
    this.blockDB = blockDB;
    this.bloomFilter = new CountingBloomFilter(BLOOM_FILTER_VECTOR_SIZE, BLOOM_FILTER_NUMBER_HASH,
            Hash.MURMUR_HASH);//  w  ww  .ja  va  2s . com
}

From source file:org.wso2.extension.siddhi.eventtable.rdbms.BloomFilterImpl.java

License:Open Source License

public void buildBloomFilters(ResultSet results) {
    CountingBloomFilter[] bloomFilters = new CountingBloomFilter[attributeList.size()];
    for (int i = 0; i < bloomFilters.length; i++) {
        bloomFilters[i] = new CountingBloomFilter(bloomFilterSize, bloomFilterHashFunction, Hash.MURMUR_HASH);
    }/*w  w w. j  a va2  s.  c  o m*/

    try {
        while (results.next()) {
            for (int i = 0; i < bloomFilters.length; i++) {
                switch (attributeList.get(i).getType()) {
                case INT:
                    bloomFilters[i].add(new Key(Integer.toString(results.getInt(i + 1)).getBytes()));
                    break;
                case LONG:
                    bloomFilters[i].add(new Key(Long.toString(results.getLong(i + 1)).getBytes()));
                    break;
                case FLOAT:
                    bloomFilters[i].add(new Key(Float.toString(results.getFloat(i + 1)).getBytes()));
                    break;
                case DOUBLE:
                    bloomFilters[i].add(new Key(Double.toString(results.getDouble(i + 1)).getBytes()));
                    break;
                case STRING:
                    String attributeValue = results.getString(i + 1);
                    if (attributeValue != null) {
                        bloomFilters[i].add(new Key(attributeValue.getBytes()));
                    }
                    break;
                case BOOL:
                    bloomFilters[i].add(new Key(Boolean.toString(results.getBoolean(i + 1)).getBytes()));
                    break;

                }
            }
        }
        results.close();
        this.bloomFilters = bloomFilters;
    } catch (SQLException ex) {
        throw new ExecutionPlanRuntimeException(
                "Error while initiating blooms filter with db data, " + ex.getMessage(), ex);
    }
}

From source file:org.wso2.siddhi.bloomfilter.CBFBasedWindow.java

License:Open Source License

/**
 * Counting bloom filter will be created with the windowSize and with
 * specifying value of the filtering attribute for the window and for the
 * bloom filter//from  ww w  .  jav a  2  s  . c o  m
 * 
 * @param windowSize
 *            No of events in the window
 * @param windowFilterAttributeId
 *            Filtering attribute location of the object array
 * @param filterValue
 *            Filtering condition value
 * @param joinAttributeId
 *            Joining attribute location of the object array
 */
public CBFBasedWindow(int windowSize, double falsePositiveRate, int windowFilterAttributeId, double filterValue,
        int joinAttributeId) {
    super(windowSize, windowFilterAttributeId, filterValue);
    this.joinAttributeId = joinAttributeId;

    eventQueue = new LinkedList<Event>();

    this.falsePositiveRate = falsePositiveRate;

    int optimalBloomFilterSize = BloomFilterUtil.optimalBloomFilterSize(windowSize, falsePositiveRate);

    filter = new CountingBloomFilter(optimalBloomFilterSize,
            (int) BloomFilterUtil.optimalNoOfHash(optimalBloomFilterSize, windowSize), Hash.MURMUR_HASH);
}

From source file:org.wso2.siddhi.core.table.RDBMSEventTable.java

License:Open Source License

private synchronized void buildBloomFilters() {
    this.bloomFilters = new CountingBloomFilter[tableDefinition.getAttributeList().size()];
    for (int i = 0; i < bloomFilters.length; i++) {
        // number of hashes: 4
        bloomFilters[i] = new CountingBloomFilter(BLOOM_FILTER_SIZE, BLOOM_FILTER_HASH_FUNCTIONS,
                Hash.MURMUR_HASH);//from w  ww  .  j a v a  2s  .  com
    }
    Connection con = null;
    Statement stmt = null;
    try {
        con = dataSource.getConnection();
        stmt = con.createStatement();
        ResultSet results = stmt.executeQuery("SELECT * FROM " + fullTableName);
        int count = 0;
        while (results.next()) {
            count++;
            for (int i = 0; i < bloomFilters.length; i++) {
                switch (tableDefinition.getAttributeList().get(i).getType()) {
                case INT:
                    bloomFilters[i].add(new Key(Integer.toString(results.getInt(i + 1)).getBytes()));
                    break;
                case LONG:
                    bloomFilters[i].add(new Key(Long.toString(results.getLong(i + 1)).getBytes()));
                    break;
                case FLOAT:
                    bloomFilters[i].add(new Key(Float.toString(results.getFloat(i + 1)).getBytes()));
                    break;
                case DOUBLE:
                    bloomFilters[i].add(new Key(Double.toString(results.getDouble(i + 1)).getBytes()));
                    break;
                case STRING:
                    bloomFilters[i].add(new Key(results.getString(i + 1).getBytes()));
                    break;
                case BOOL:
                    bloomFilters[i].add(new Key(Boolean.toString(results.getBoolean(i + 1)).getBytes()));
                    break;

                }
            }
        }
        results.close();

    } catch (Exception ex) {
        log.error(ex);
    } finally {
        cleanUpConnections(stmt, con);
    }
}

From source file:org.wso2.siddhi.core.util.collection.bloomfilter.CountingBloomFilterBasedWindow.java

License:Open Source License

/**
 * Bloom filters will be created with the given expectedNoOfEventsPerWindow
 * (windowSize). Initialize the join attribute according to the expression.
 * /*from   w  w w .ja  v a2  s.c  o m*/
 * @param expressions
 *            [0] expected No Of Events in the window
 * @param expressions
 *            [1] if value is 0 - beforeWindowData if value is 1 -
 *            onOrbeforeWindowData if value is 2 - outputDataWindowData
 * @param expressions
 *            [2] value is the array position of the above three arrays
 * @param expressions
 *            [3] bloomFilterSize - Size of the bloom filter
 */
public void init(Expression[] expressions) throws OperationNotSupportedException {

    eventQueue = new LinkedBlockingQueue<StreamEvent>();

    joinAttributeId = new int[4];
    joinAttributeId[0] = 0;
    joinAttributeId[1] = 0;

    if (expressions.length == 3) {
        expectedNoOfEventsPerWindow = ((IntConstant) expressions[0]).getValue();
        joinAttributeId[2] = ((IntConstant) expressions[1]).getValue();
        joinAttributeId[3] = ((IntConstant) expressions[2]).getValue();
        bloomFilterSize = BloomFilterUtil.optimalBloomFilterSize(expectedNoOfEventsPerWindow, 0.0005);
    } else if (expressions.length == 4) {
        expectedNoOfEventsPerWindow = ((IntConstant) expressions[0]).getValue();
        joinAttributeId[2] = ((IntConstant) expressions[1]).getValue();
        joinAttributeId[3] = ((IntConstant) expressions[2]).getValue();
        bloomFilterSize = ((IntConstant) expressions[3]).getValue();
    } else {
        throw new OperationNotSupportedException(
                "Parameters count is not matching, There should be three or four parameters ");
    }

    if (bloomFilterSize < BloomFilterUtil.optimalBloomFilterSize(expectedNoOfEventsPerWindow, 0.001)
            || bloomFilterSize > BloomFilterUtil.optimalBloomFilterSize(expectedNoOfEventsPerWindow, 0.00001)) {
        throw new OperationNotSupportedException("Bloomfilter size not suitable");
    }

    int noOfHash = BloomFilterUtil.optimalNoOfHash(bloomFilterSize, expectedNoOfEventsPerWindow);

    filter = new CountingBloomFilter(bloomFilterSize, noOfHash, Hash.MURMUR_HASH);

}

From source file:org.wso2.siddhi.extension.eventtable.rdbms.DBHandler.java

License:Open Source License

public void buildBloomFilters() {
    CountingBloomFilter[] bloomFilters = new CountingBloomFilter[tableDefinition.getAttributeList().size()];

    for (int i = 0; i < bloomFilters.length; i++) {
        bloomFilters[i] = new CountingBloomFilter(bloomFilterSize, bloomFilterHashFunction, Hash.MURMUR_HASH);
    }//from  w  w w  . j a  v a 2  s. c  om
    Connection con = null;
    Statement stmt = null;
    try {
        con = dataSource.getConnection();
        stmt = con.createStatement();
        String selectTableRowQuery = constructQuery(tableName,
                elementMappings.get(RDBMSEventTableConstants.EVENT_TABLE_GENERIC_RDBMS_SELECT_TABLE), null,
                null, null, null, null);
        ResultSet results = stmt.executeQuery(selectTableRowQuery);
        while (results.next()) {
            for (int i = 0; i < bloomFilters.length; i++) {
                switch (tableDefinition.getAttributeList().get(i).getType()) {
                case INT:
                    bloomFilters[i].add(new Key(Integer.toString(results.getInt(i + 1)).getBytes()));
                    break;
                case LONG:
                    bloomFilters[i].add(new Key(Long.toString(results.getLong(i + 1)).getBytes()));
                    break;
                case FLOAT:
                    bloomFilters[i].add(new Key(Float.toString(results.getFloat(i + 1)).getBytes()));
                    break;
                case DOUBLE:
                    bloomFilters[i].add(new Key(Double.toString(results.getDouble(i + 1)).getBytes()));
                    break;
                case STRING:
                    bloomFilters[i].add(new Key(results.getString(i + 1).getBytes()));
                    break;
                case BOOL:
                    bloomFilters[i].add(new Key(Boolean.toString(results.getBoolean(i + 1)).getBytes()));
                    break;

                }
            }
        }
        results.close();
        this.bloomFilters = bloomFilters;
        this.isBloomFilterEnabled = true;
    } catch (SQLException ex) {
        throw new ExecutionPlanRuntimeException(
                "Error while initiating blooms filter with db data, " + ex.getMessage(), ex);
    } finally {
        cleanUpConnections(stmt, con);
    }
}