Example usage for org.apache.hadoop.io IntWritable get

List of usage examples for org.apache.hadoop.io IntWritable get

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.io.*
  5. IntWritable
  6. get

Introduction

In this page you can find the example usage for org.apache.hadoop.io IntWritable get.

Prototype

public int get()

Source Link

Document

Return the value of this IntWritable.

Usage

From source file:it.uniroma1.bdc.tesi.piccioli.giraphstandalone.checkintegrity.undirect.Check.java

License:Apache License

/**
 * Prima fase composta da i primi 2 superstep 1 superstep - calcolo del
 * degree di ogni nodo e invio info a nodi vicino 2 superstep - elimino
 * archi fuori ordinamento//from   w w w.  j  a v a 2  s.c  o  m
 *
 * @param vertex
 * @param messages
 * @throws java.io.IOException
 *
 */
@Override
public void compute(Vertex<IntWritable, DoubleWritable, NullWritable> vertex, Iterable<IntWritable> messages)
        throws IOException {

    long superstep = this.getSuperstep();
    Iterable<Edge<IntWritable, NullWritable>> edges = vertex.getEdges();

    if (superstep == 0) {
        //invio messaggi per controllo esistenza arco inverso
        this.sendMessageToAllEdges(vertex, vertex.getId());

        //controllo unicit edge
        Set<Integer> edgeSet = new HashSet<>();
        for (Edge<IntWritable, NullWritable> edge : edges) {
            if (!edgeSet.contains(edge.getTargetVertexId().get())) {
                edgeSet.add(edge.getTargetVertexId().get());
            } else {
                //Segnalo errore edge doppio
                System.out.println(vertex.getId() + "-->" + edge.getTargetVertexId() + " doppio");
            }

            //controllo se sono presenti archi verso su stesso vertice
            if (edge.getTargetVertexId().get() == vertex.getId().get()) {
                System.out.println("su vertice " + vertex.getId() + " presente arco su se stesso");
            }
        }

    }
    if (superstep == 1) {
        //controllo esistenza edge inverso
        Set<Integer> edgeSet = new HashSet<>();
        for (Edge<IntWritable, NullWritable> edge : edges) {
            if (!edgeSet.contains(edge.getTargetVertexId().get())) {
                edgeSet.add(edge.getTargetVertexId().get());
            }
        }
        for (IntWritable msg : messages) {
            if (!edgeSet.contains(msg.get())) {
                //Segnalo errore mancanza edge inverso
                System.out.println("manca " + vertex.getId() + "-->" + msg);
            }
        }
        vertex.voteToHalt();
    }

}

From source file:it.uniroma1.bdc.tesi.piccioli.giraphstandalone.densesubgraph.undirect.intwritable.DenseSubgraphUndirectMasterCompute.java

License:Apache License

@Override
public void compute() {

    long superstep = getSuperstep();
    IntWritable removedEdges = this.getAggregatedValue(REMOVEDEDGES);//superstep precedente
    IntWritable removedVertex = this.getAggregatedValue(REMOVEDVERTICIES);//superstep precedente

    if (isEven(superstep)) {//0,2,4....

        LOG.info("confronto \t" + prevStepRemovedVertex + "\t" + removedEdges);

        //con rimozione effettiva dei vertici ci vogliono 2 step per startup
        if ((prevStepRemovedVertex.equals(removedEdges.get())) && superstep > 2) {
            LOG.info("NO CHANGES - HALT COMPUTATION");
            LOG.info("BEST DENSITY\t" + bestlDensity + " at " + bestDensitySuperstep);
            this.haltComputation();
            return;
        }/*from  w  w w.j a  v  a2s  .  c o m*/

        //Aggiorno variabile vertici rimossi per step successivo
        prevStepRemovedVertex = removedEdges.get();

        //DENSITY UNDIRECT ?(S) = (|E(S)| / 2 ) / |S|
        // |E(S)| / 2 perch giraph rappresenta edge non diretto con 2 edge diretti
        Long edges = this.getTotalNumEdges() - removedEdges.get();
        Long vertices = this.getTotalNumVertices() - removedVertex.get();
        Double currDensity = (edges.doubleValue() / 2) / vertices.doubleValue();

        LOG.info("superstep\t" + superstep + "\tedge\t" + edges + "\tvertices\t" + vertices + "\tdensity\t"
                + currDensity);

        if (currDensity > bestlDensity) {
            bestlDensity = currDensity;
            bestDensitySuperstep = superstep - 2;//Densit calcolata sul supertep pari precedente
            this.getConf().setLong(OPTIMALSUPERSTEP, superstep);
        }

        //soglia = 2(1 + epsilon) ?(S)
        Double soglia = 2 * (1 + epsilon) * currDensity;
        LOG.info("soglia = " + soglia);

        this.getContext().getConfiguration().setDouble(SOGLIA, soglia);

    }
    //        else {//1,3,5...
    //
    //        }

}

From source file:it.uniroma1.bdc.tesi.piccioli.giraphstandalone.sssp.intwritable.SimpleShortestPathsComputationTextValue.java

License:Apache License

@Override
public void compute(Vertex<IntWritable, IntWritable, NullWritable> vertex, Iterable<IntWritable> messages)
        throws IOException {

    if (getSuperstep() == 0) {
        vertex.setValue(new IntWritable(Integer.MAX_VALUE));
    }/*w  w w.j  a  v  a2s. com*/
    int minDist = isSource(vertex) ? 0 : Integer.MAX_VALUE;
    for (IntWritable message : messages) {
        minDist = Math.min(minDist, message.get());
    }
    if (LOG.isDebugEnabled()) {
        System.out.println("Vertex " + vertex.getId() + " got minDist = " + minDist + " vertex value = "
                + vertex.getValue());
    }
    if (minDist < new Long(vertex.getValue().toString())) {
        vertex.setValue(new IntWritable(minDist));
        for (Edge<IntWritable, NullWritable> edge : vertex.getEdges()) {

            //                long distance = minDist + edge.getValue().get();
            int distance = minDist + 1;

            if (LOG.isDebugEnabled()) {
                System.out.println(
                        "Vertex " + vertex.getId() + " sent to " + edge.getTargetVertexId() + " = " + distance);
            }
            sendMessage(edge.getTargetVertexId(), new IntWritable(distance));
        }
    }
    vertex.voteToHalt();
}

From source file:it.uniroma1.bdc.tesi.piccioli.giraphstandalone.trianglecount.intwritable.TriangleCount.java

License:Apache License

@Override
public void compute(Vertex<IntWritable, DoubleWritable, NullWritable> vertex, Iterable<IntWritable> messages)
        throws IOException {

    Iterable<Edge<IntWritable, NullWritable>> edges = vertex.getEdges();

    if (getSuperstep() == 0) {

        for (Edge<IntWritable, NullWritable> edge : edges) {
            this.sendMessageToAllEdges(vertex, edge.getTargetVertexId());
        }//w w  w. jav  a  2s. c o  m

    } else if (getSuperstep() == 1) {

        Double T = 0.0;
        Set<Integer> edgeMap = Sets.<Integer>newHashSet();

        for (Edge<IntWritable, NullWritable> edge : edges) {
            edgeMap.add(edge.getTargetVertexId().get());
        }
        for (IntWritable message : messages) {
            if (edgeMap.contains(message.get())) {
                T++;
            }
        }

        T = T / 6;

        vertex.setValue(new DoubleWritable(T));
        vertex.voteToHalt();

        aggregate(SOMMA + getSuperstep(), new DoubleWritable(T));

    }

}

From source file:it.uniroma1.bdc.tesi.piccioli.giraphstandalone.trianglecount.intwritable.TriangleCountPlusPlusPhase2.java

License:Apache License

@Override
public void compute(Vertex<IntWritable, IntWritable, NullWritable> vertex, Iterable<IntWritable> messages)
        throws IOException {

    Iterable<Edge<IntWritable, NullWritable>> edges = vertex.getEdges();

    if (getSuperstep() == 2) {
        //triangle count
        for (Edge<IntWritable, NullWritable> edge : edges) {
            this.sendMessageToAllEdges(vertex, edge.getTargetVertexId());
        }//w  w  w.j a v a  2 s  .  c om
    }
    if (getSuperstep() == 3) {
        Integer T = 0;
        Set<Integer> edgeMap = Sets.<Integer>newHashSet();

        for (Edge<IntWritable, NullWritable> edge : edges) {
            edgeMap.add(edge.getTargetVertexId().get());
        }

        for (IntWritable message : messages) {
            if (edgeMap.contains(message.get())) {
                T++;
            }
        }
        vertex.setValue(new IntWritable(T));
        aggregate(SOMMA + getSuperstep(), new LongWritable(T));
        vertex.voteToHalt();
    }
}

From source file:ivory.core.data.document.IntDocVectorsForwardIndex.java

License:Apache License

/**
 * Returns the document vector given a docno.
 *
 * @return {@code IntDocVector} for the appropriate docno
 *//*from w  w w  .  ja v a 2 s  .  c o m*/
public IntDocVector getDocVector(int docno) throws IOException {
    Preconditions.checkArgument(!(docno > collectionDocumentCount || docno < 1));

    long pos = positions[docno - docnoOffset - 1];

    int fileNo = (int) (pos / BigNumber);
    pos = pos % BigNumber;

    SequenceFile.Reader reader = null;
    try {
        reader = new SequenceFile.Reader(fs, new Path(path + "/part-m-" + FORMAT.format(fileNo)), conf);
    } catch (IOException e) {
        // Try alternative naming scheme for the old API.
        reader = new SequenceFile.Reader(fs, new Path(path + "/part-" + FORMAT.format(fileNo)), conf);
    }

    IntWritable key = new IntWritable();
    IntDocVector value;

    try {
        value = (IntDocVector) reader.getValueClass().newInstance();
    } catch (Exception e) {
        throw new RuntimeException("Unable to instantiate key/value pair!");
    }

    reader.seek(pos);
    reader.next(key, value);

    if (key.get() != docno) {
        LOG.error("unable to doc vector for docno " + docno + ": found docno " + key + " instead");
        return null;
    }

    reader.close();
    return value;
}

From source file:ivory.core.data.document.TermDocVectorsForwardIndex.java

License:Apache License

/**
 * Returns the document vector given a docno.
 *//*w  w  w .  j a  v a2 s  .co m*/
public TermDocVector getDocVector(int docno) throws IOException {
    // TODO: This method re-opens the SequenceFile on every access.
    // Would be more efficient to cache the file handles.
    if (docno > collectionDocumentCount || docno < 1) {
        return null;
    }

    long pos = positions[docno - docnoOffset - 1];

    int fileNo = (int) (pos / BigNumber);
    pos = pos % BigNumber;

    SequenceFile.Reader reader = null;
    try {
        reader = new SequenceFile.Reader(fs, new Path(path + "/part-m-" + FORMAT.format(fileNo)), conf);
    } catch (IOException e) {
        // Try alternative naming scheme for output of old API.
        reader = new SequenceFile.Reader(fs, new Path(path + "/part-" + FORMAT.format(fileNo)), conf);
    }

    IntWritable key = new IntWritable();
    TermDocVector value;

    try {
        value = (TermDocVector) reader.getValueClass().newInstance();
    } catch (Exception e) {
        throw new RuntimeException("Unable to instantiate key/value pair!");
    }

    reader.seek(pos);
    reader.next(key, value);

    if (key.get() != docno) {
        LOG.error(String.format("Unable to doc vector for docno %d: found docno %d instead.", docno, key));
        return null;
    }

    reader.close();
    return value;
}

From source file:ivory.core.data.index.IntPostingsForwardIndex.java

License:Apache License

public PostingsList getPostingsList(int termid) throws IOException {
    // TODO: This method re-opens the SequenceFile on every access. Would be more efficient to cache
    // the file handles.

    long pos = positions[termid - 1];

    int fileNo = (int) (pos / BigNumber);
    pos = pos % BigNumber;//from w w  w  . j  a  v a 2  s .com

    // Open up the SequenceFile.
    SequenceFile.Reader reader = null;
    try {
        reader = new SequenceFile.Reader(fs, new Path(postingsPath + "/part-" + FORMAT.format(fileNo)), conf);
    } catch (IOException e) {
        // Try alternative naming scheme for output of new API.
        reader = new SequenceFile.Reader(fs, new Path(postingsPath + "/part-r-" + FORMAT.format(fileNo)), conf);
    }

    IntWritable key = new IntWritable();
    PostingsList value = null;
    try {
        value = (PostingsList) Class.forName(reader.getValueClassName()).newInstance();
    } catch (Exception e) {
        e.printStackTrace();
        return null;
    }

    reader.seek(pos);
    reader.next(key, value);

    if (key.get() != termid) {
        LOG.error("unable to fetch postings for term \"" + termid + "\": found key \"" + key + "\" instead");
        return null;
    }

    reader.close();
    return value;
}

From source file:ivory.data.IntDocVectorsForwardIndex.java

License:Apache License

/**
 * Returns the document vector given a docno.
 */// ww w.ja  v  a  2 s.  c o  m
public IntDocVector getDocVector(int docno) throws IOException {
    if (docno > mCollectionDocumentCount || docno < 1)
        return null;

    long pos = mPositions[docno - mDocnoOffset - 1];

    int fileNo = (int) (pos / BuildIntDocVectorsForwardIndex.BigNumber);
    pos = pos % BuildIntDocVectorsForwardIndex.BigNumber;

    SequenceFile.Reader reader = new SequenceFile.Reader(mFs,
            new Path(mPath + "/part-" + sFormatW5.format(fileNo)), mConf);

    IntWritable key = new IntWritable();
    IntDocVector value;

    try {
        value = (IntDocVector) reader.getValueClass().newInstance();
    } catch (Exception e) {
        throw new RuntimeException("Unable to instantiate key/value pair!");
    }

    reader.seek(pos);
    reader.next(key, value);

    if (key.get() != docno) {
        sLogger.error("unable to doc vector for docno " + docno + ": found docno " + key + " instead");
        return null;
    }

    reader.close();
    return value;
}

From source file:ivory.data.IntPostingsForwardIndex.java

License:Apache License

public PostingsList getPostingsList(int termid) throws IOException {
    // TODO: This method re-opens the SequenceFile on every access. Would be more efficient to cache
    // the file handles.

    //sLogger.info("getPostingsList("+termid+")");
    long pos = positions[termid - 1];

    IntWritable key = new IntWritable();

    PostingsList value = null;//from   w  w  w. j a  v  a 2 s. co  m
    try {
        value = (PostingsList) Class.forName(postingsType).newInstance();
    } catch (InstantiationException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    } catch (IllegalAccessException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    } catch (ClassNotFoundException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    }
    //sLogger.info("type: "+postingsType);
    //sLogger.info("stored pos: " + pos);
    int fileNo = (int) (pos / BIG_LONG_NUMBER);

    pos = pos % BIG_LONG_NUMBER;

    String fileNoStr = fileNo + "";
    String padd = "";
    for (int i = 5; i > fileNoStr.length(); i--)
        padd += "0";
    fileNoStr = padd + fileNoStr;

    // open up the SequenceFile

    //sLogger.info("file no: " + fileNoStr);
    //sLogger.info("file pos: " + pos);

    SequenceFile.Reader reader = new SequenceFile.Reader(mFs, new Path(postingsPath + "/part-" + fileNoStr),
            conf);

    reader.seek(pos);
    reader.next(key, value);

    if (key.get() != termid) {
        sLogger.error(
                "unable to fetch postings for term \"" + termid + "\": found key \"" + key + "\" instead");
        return null;
    }

    reader.close();
    return value;
}