Example usage for org.apache.hadoop.io WritableUtils clone

Introduction

In this page you can find the example usage for org.apache.hadoop.io WritableUtils clone.

Prototype

public static <T extends Writable> T clone(T orig, Configuration conf)

Source Link

Document

Make a copy of a writable object using serialization to a buffer.

Usage

From source file:TaggedMapOutput.java

License:Apache License

public TaggedMapOutput clone(JobConf job) {
    return (TaggedMapOutput) WritableUtils.clone(this, job);
}

From source file:com.citic.zxyjs.zwlscx.mapreduce.join.api.TaggedMapOutput.java

License:Apache License

public TaggedMapOutput clone(Configuration conf) {
    return (TaggedMapOutput) WritableUtils.clone(this, conf);
}

From source file:com.manning.hip.ch4.joins.improved.impl.OutputValue.java

License:Apache License

public OutputValue clone(JobConf job) {
    return WritableUtils.clone(this, job);
}

From source file:main.okapi.common.computation.SendFriends.java

License:Apache License

@Override
public void compute(Vertex<I, V, E> vertex, Iterable<M> messages) throws IOException {

    final Vertex<I, V, E> _vertex = vertex;

    final ArrayListWritable friends = new ArrayListWritable() {
        @Override/*from   www  .  j av  a 2 s.c o m*/
        public void setClass() {
            setClass(_vertex.getId().getClass());
        }
    };

    for (Edge<I, E> edge : vertex.getEdges()) {
        friends.add(WritableUtils.clone(edge.getTargetVertexId(), getConf()));
    }

    MessageWrapper<I, ArrayListWritable<I>> msg = new MessageWrapper<I, ArrayListWritable<I>>() {

        @Override
        public Class getVertexIdClass() {
            return _vertex.getClass();
        }

        @Override
        public Class getMessageClass() {
            return friends.getClass();
        }
    };

    msg.setSourceId(vertex.getId());
    msg.setMessage(friends);
    sendMessageToAllEdges(vertex, (M) msg);
}

From source file:org.apache.giraph.comm.messages.with_source.SimpleMessageWithSourceStore.java

License:Apache License

/**
 * YH: If there is already a map of sources related to the destination
 * vertex id, then return that map. Otherwise create a new one, put it
 * in the partition map, and return it.//from   w ww  . j  a v  a 2s  . c om
 *
 * @param partitionMap Message map for this partition
 * @param dstId Id of the destination vertex (receiver)
 * @param cloneDstId True if dstId must be cloned/copied before storage
 * @return Source map for the destination vertex
 */
protected ConcurrentMap<I, T> getOrCreateSourceMap(ConcurrentMap<I, ConcurrentMap<I, T>> partitionMap, I dstId,
        boolean cloneDstId) {
    ConcurrentMap<I, T> srcMap = partitionMap.get(dstId);

    if (srcMap == null) {
        ConcurrentMap<I, T> tmpMap = new MapMaker()
                .concurrencyLevel(config.getNettyServerExecutionConcurrency()).makeMap();

        // YH: if needed, clone the dest vertex id. This is needed when the
        // original object is user-accessible and/or can be invalidated on
        // some iterator's next() call up the call chain.
        I safeDstId = cloneDstId ? WritableUtils.clone(dstId, config) : dstId;

        srcMap = partitionMap.putIfAbsent(safeDstId, tmpMap);
        if (srcMap == null) {
            srcMap = tmpMap;
        }
    }

    return srcMap;
}

From source file:org.apache.giraph.comm.requests.SendVertexDLForkRequest.java

License:Apache License

/**
 * Constructor./*from w w w .j a va 2  s.  c  om*/
 *
 * @param senderId Sender vertex id
 * @param receiverId Receiver vertex id
 * @param conf ImmutableClassesGiraphConfiguration
 */
public SendVertexDLForkRequest(I senderId, I receiverId, ImmutableClassesGiraphConfiguration conf) {
    setConf(conf); // getConf() is null until properly set
    this.senderId = WritableUtils.clone(senderId, getConf());
    this.receiverId = WritableUtils.clone(receiverId, getConf());
}

From source file:org.apache.giraph.comm.requests.SendVertexDLTokenRequest.java

License:Apache License

/**
 * Constructor./* w w w  . ja va  2 s . c  o m*/
 *
 * @param senderId Sender vertex id
 * @param receiverId Receiver vertex id
 * @param conf ImmutableClassesGiraphConfiguration
 */
public SendVertexDLTokenRequest(I senderId, I receiverId, ImmutableClassesGiraphConfiguration<I, V, E> conf) {
    setConf(conf); // getConf() is null until properly set
    this.senderId = WritableUtils.clone(senderId, getConf());
    this.receiverId = WritableUtils.clone(receiverId, getConf());
}

From source file:org.apache.giraph.examples.okapi.common.computation.SendFriends.java

License:Apache License

@Override
public void compute(Vertex<I, V, E> vertex, Iterable<M> messages) throws IOException {

    final Vertex<I, V, E> finalVertex = vertex;

    final ArrayListWritable friends = new ArrayListWritable() {
        @Override/*w  w  w . j  a  va  2s  .co  m*/
        public void setClass() {
            setClass(finalVertex.getId().getClass());
        }
    };

    for (Edge<I, E> edge : vertex.getEdges()) {
        friends.add(WritableUtils.clone(edge.getTargetVertexId(), getConf()));
    }

    MessageWrapper<I, ArrayListWritable<I>> msg = new MessageWrapper<I, ArrayListWritable<I>>() {
        @Override
        public Class getVertexIdClass() {
            return finalVertex.getClass();
        }

        @Override
        public Class getMessageClass() {
            return friends.getClass();
        }
    };

    msg.setSourceId(vertex.getId());
    msg.setMessage(friends);
    sendMessageToAllEdges(vertex, (M) msg);
}

From source file:org.apache.giraph.partition.VertexTypeStore.java

License:Apache License

/**
 * Adds vertexId to set, synchronizing on set.
 *
 * @param set Set to add to//from w  w w.j av a  2 s .c  o m
 * @param vertexId Vertex id to add
 */
private void synchronizedAdd(Set set, I vertexId) {
    Class<I> vertexIdClass = conf.getVertexIdClass();
    synchronized (set) {
        if (vertexIdClass.equals(IntWritable.class)) {
            set.add(((IntWritable) vertexId).get());
        } else if (vertexIdClass.equals(LongWritable.class)) {
            set.add(((LongWritable) vertexId).get());
        } else {
            // need to clone id when not primitive
            set.add(WritableUtils.clone(vertexId, conf));
        }
    }
}

From source file:org.apache.giraph.tools.graphanalytics.semiclustering.SemiClusteringVertex.java

License:Apache License

/**
 * The user overrides the Compute() method, which will be executed at each
 * active vertex in every superstep//from   w  w w .  ja  v  a2 s  .  c o  m
 */
@Override
public void compute(Iterable<SemiClusterMessage> messages) throws IOException {
    if (this.getSuperstep() == 0) {
        initClusters();
    }

    if (this.getSuperstep() >= 1) {
        TreeSet<SemiClusterMessage> candidates = new TreeSet<SemiClusterMessage>();

        for (SemiClusterMessage msg : messages) {
            candidates.add(msg);

            if (!msg.contains(this.getId()) && msg.size() == semiClusterMaximumVertexCount) {
                SemiClusterMessage msgNew = WritableUtils.clone(msg, this.getConf());
                msgNew.addVertex(this);
                msgNew.setSemiClusterId("C" + createNewSemiClusterName(msgNew.getVertexList()));
                msgNew.setScore(semiClusterScoreCalcuation(msgNew));

                candidates.add(msgNew);
            }
        }

        Iterator<SemiClusterMessage> bestCandidates = candidates.descendingIterator();
        int count = 0;

        while (bestCandidates.hasNext() && count < graphJobMessageSentCount) {
            SemiClusterMessage candidate = bestCandidates.next();
            this.sendMessageToAllEdges(candidate);
            count++;
        }

        // Update candidates
        SemiClusterMessage value = this.getValue();
        Set<SemiClusterDetails> clusters = value.getSemiClusterContainThis();
        for (SemiClusterMessage msg : candidates) {
            if (clusters.size() > graphJobVertexMaxClusterCount) {
                break;
            } else {
                clusters.add(new SemiClusterDetails(msg.getSemiClusterId(), msg.getScore()));
            }
        }

        value.setClusters(clusters, graphJobVertexMaxClusterCount);
        this.setValue(value);
    }
}