List of usage examples for org.apache.hadoop.io LongWritable set
public void set(long value)
From source file:org.apache.giraph.mapping.DefaultEmbeddedLongByteOps.java
License:Apache License
@Override public void embedTargetInfo(LongWritable id) { if ((id.get() & MASK) != 0) { throw new IllegalStateException("Expected first 9 bits of long " + " to be empty"); }//from w w w . java 2 s .c om byte target = mappingStore.getByteTarget(id); // first bit = 0 & rest 8 bits set to target // add 1 to distinguish between not set and assignment to worker-0 // (prefix bits = 0 can mean one of two things : // no entry in the mapping, in which case target = -1, so -1 + 1 = 0 // vertex is created later during computation, so prefix bits are 0 anyway) long maskValue = ((1L + target) & 0xFF) << 55; id.set(id.get() | maskValue); }
From source file:org.apache.giraph.mapping.DefaultEmbeddedLongByteOps.java
License:Apache License
@Override public void removeTargetInfo(LongWritable id) { id.set(id.get() & IMASK); }
From source file:org.apache.giraph.mapping.translate.LongByteTranslateEdge.java
License:Apache License
@Override public LongWritable translateId(LongWritable targetId) { LongWritable translatedId = new LongWritable(); translatedId.set(targetId.get()); localData.getMappingStoreOps().embedTargetInfo(translatedId); return translatedId; }
From source file:org.apache.giraph.partition.SimpleRangePartitionFactoryTest.java
License:Apache License
private void testRange(int numWorkers, int keySpaceSize, int allowedWorkerDiff, boolean emptyWorkers) { Configuration conf = new Configuration(); conf.setLong(GiraphConstants.PARTITION_VERTEX_KEY_SPACE_SIZE, keySpaceSize); SimpleLongRangePartitionerFactory<Writable, Writable> factory = new SimpleLongRangePartitionerFactory<Writable, Writable>(); factory.setConf(new ImmutableClassesGiraphConfiguration(conf)); ArrayList<WorkerInfo> infos = new ArrayList<WorkerInfo>(); for (int i = 0; i < numWorkers; i++) { WorkerInfo info = new WorkerInfo(); info.setInetSocketAddress(new InetSocketAddress(8080)); info.setTaskId(i);// w w w. j a v a2 s . c o m infos.add(info); } Collection<PartitionOwner> owners = factory.createMasterGraphPartitioner() .createInitialPartitionOwners(infos, -1); int[] tasks = new int[owners.size()]; for (PartitionOwner owner : owners) { WorkerInfo worker = owner.getWorkerInfo(); assertEquals(0, tasks[owner.getPartitionId()]); tasks[owner.getPartitionId()] = worker.getTaskId() + 1; } checkMapping(tasks, allowedWorkerDiff, emptyWorkers); WorkerGraphPartitioner<LongWritable, Writable, Writable> workerPartitioner = factory .createWorkerGraphPartitioner(); workerPartitioner.updatePartitionOwners(null, owners); LongWritable longWritable = new LongWritable(); int[] partitions = new int[keySpaceSize]; for (int i = 0; i < keySpaceSize; i++) { longWritable.set(i); PartitionOwner owner = workerPartitioner.getPartitionOwner(longWritable); partitions[i] = owner.getPartitionId(); } checkMapping(partitions, 1, emptyWorkers); }
From source file:org.apache.giraph.partition.VertexPhilosophersTable.java
License:Apache License
/** * Send our philosopher's dependencies to their partners. * Required in the case of directed graphs (neighbouring * philosopher will otherwise fail to see dependencies). *//*from w ww . ja va 2s. c o m*/ public void sendDependencies() { LongWritable vertexId = new LongWritable(); // reused LongWritable depVertexId = new LongWritable(); // reused // when to flush cache/send message to particular worker int maxMessagesSizePerWorker = GiraphConfiguration.MAX_MSG_REQUEST_SIZE.get(conf); // cache for messages that will be sent to neighbours // (much more efficient than using SendDataCache b/c // we don't need to store/know dst partition ids) Int2ObjectOpenHashMap<VertexIdData<I, I>> msgCache = new Int2ObjectOpenHashMap<VertexIdData<I, I>>(); // Get cached copy of keyset to ignore future modifications // (future modifications are always *received* dependencies). // keySet() is backed by collection, so need to get "safe" copy. long[] pKeySet; synchronized (pMap) { pKeySet = pMap.keySet().toLongArray(); } Long2ByteOpenHashMap neighbours; long[] neighboursKeySet; for (long pId : pKeySet) { // must synchronize on pMap and neighbours separately, due to // race with synchronize-on-neighbours in receiveDependency() synchronized (pMap) { neighbours = pMap.get(pId); } synchronized (neighbours) { neighboursKeySet = neighbours.keySet().toLongArray(); } // we store pId to neighbourId mapping, so we want every // neighbourId to know that they need a "to pId" mapping for (long neighbourId : neighboursKeySet) { vertexId.set(neighbourId); depVertexId.set(pId); // this will work even if vertex doesn't exist yet int dstTaskId = serviceWorker.getVertexPartitionOwner((I) vertexId).getWorkerInfo().getTaskId(); //LOG.info("[[PTABLE]] send dependency to taskId " + dstTaskId); // skip sending message for local request if (serviceWorker.getWorkerInfo().getTaskId() == dstTaskId) { receiveDependency((I) vertexId, (I) depVertexId); } else { // no need to sync, this is executed by single thread VertexIdData<I, I> messages = msgCache.get(dstTaskId); if (messages == null) { messages = new ByteArrayVertexIdVertexId<I>(); messages.setConf(conf); messages.initialize(); msgCache.put(dstTaskId, messages); } // note that, unlike regular data messages, first vertex id // is also data---i.e., this msg is not being sent to that id messages.add((I) vertexId, (I) depVertexId); if (messages.getSize() > maxMessagesSizePerWorker) { msgCache.remove(dstTaskId); serviceWorker.getWorkerClient().sendWritableRequest(dstTaskId, new SendVertexDLDepRequest(messages)); } } } } // send remaining messages for (int dstTaskId : msgCache.keySet()) { // no need to remove, map will be trashed entirely VertexIdData<I, I> messages = msgCache.get(dstTaskId); serviceWorker.getWorkerClient().sendWritableRequest(dstTaskId, new SendVertexDLDepRequest(messages)); } // flush network serviceWorker.getWorkerClient().waitAllRequests(); }
From source file:org.apache.giraph.partition.VertexPhilosophersTable.java
License:Apache License
/** * Blocking call that returns when all forks are acquired and * the philosopher starts to eat./*from w w w .ja v a 2 s .com*/ * * @param vertexId Vertex id (philosopher) to acquire forks for */ public void acquireForks(I vertexId) { //LOG.info("[[PTABLE]] " + vertexId + ": acquiring forks"); LongWritable dstId = new LongWritable(); // reused long pId = ((LongWritable) vertexId).get(); Long2ByteOpenHashMap neighbours = pMap.get(pId); byte oldForkInfo; boolean needFlush = false; // ALWAYS lock neighbours before pHungry/pEating synchronized (neighbours) { synchronized (pHungry) { pHungry.add(pId); } } // for loop NOT synchronized b/c mutations must never occur // while compute threads are running, so keyset is fixed // (if NEW items were added, this will fail catastrophically!) for (long neighbourId : neighbours.keySet()) { synchronized (neighbours) { byte forkInfo = neighbours.get(neighbourId); oldForkInfo = forkInfo; // Note that missing fork does NOT imply holding token, // b/c we may have already sent off our token if (!haveFork(forkInfo) && haveToken(forkInfo)) { forkInfo &= ~MASK_HAVE_TOKEN; neighbours.put(neighbourId, forkInfo); //LOG.info("[[PTABLE]] " + pId + ": request fork " + // neighbourId + " " + toString(forkInfo)); } // otherwise, we already sent our token or already hold // our fork (can be clean or dirty) } // We're working with stale forkInfo here, so that request can // be sent outside of synchronization block. This has to be // done to avoid deadlocks due to local sendToken getting blocked // due to local conflicts. // // Also note that we've applied forkInfo update before sending. // Important b/c local requests will immediately modify forkInfo. if (!haveFork(oldForkInfo) && haveToken(oldForkInfo)) { dstId.set(neighbourId); needFlush |= sendToken(vertexId, (I) dstId); } } if (needFlush) { //LOG.info("[[PTABLE]] " + vertexId + ": flushing"); serviceWorker.getWorkerClient().waitAllRequests(); //LOG.info("[[PTABLE]] " + vertexId + ": done flush"); } boolean done; cvLock.lock(); // lock early to avoid missing signals try { while (true) { done = true; // Recheck forks. This must be done in a single block // to get fully consistent view of ALL forks. synchronized (neighbours) { ObjectIterator<Long2ByteMap.Entry> itr = neighbours.long2ByteEntrySet().fastIterator(); while (itr.hasNext()) { byte forkInfo = itr.next().getByteValue(); if (!haveFork(forkInfo)) { //LOG.info("[[PTABLE]] " + vertexId + ": still missing fork"); done = false; break; } } if (done) { // have all forks, now eating // must do this while synchronized on neighbours synchronized (pHungry) { pHungry.remove(pId); } synchronized (pEating) { pEating.add(pId); } //LOG.info("[[PTABLE]] " + vertexId + ": got all forks"); return; } } //LOG.info("[[PTABLE]] " + vertexId + ": wait for forks"); newFork.await(); //LOG.info("[[PTABLE]] " + vertexId + ": woke up!"); } } catch (InterruptedException e) { throw new IllegalStateException("acquireForks: got interrupted!"); } finally { cvLock.unlock(); } }
From source file:org.apache.giraph.partition.VertexPhilosophersTable.java
License:Apache License
/** * Dirties used forks and satisfies any pending requests * for such forks. Equivalent to "stop eating". * * @param vertexId Vertex id (philosopher) that finished eating *//*from ww w . java 2 s .co m*/ public void releaseForks(I vertexId) { //LOG.info("[[PTABLE]] " + vertexId + ": releasing forks"); LongWritable dstId = new LongWritable(); // reused long pId = ((LongWritable) vertexId).get(); Long2ByteOpenHashMap neighbours = pMap.get(pId); byte oldForkInfo; boolean needFlush = false; // ALWAYS lock neighbours before pHungry/pEating synchronized (neighbours) { synchronized (pEating) { pEating.remove(pId); } } // for loop NOT synchronized b/c mutations must never occur // while compute threads are running, so keyset is fixed // (if NEW items were added, this will fail catastrophically!) // // all held forks are (possibly implicitly) dirty for (long neighbourId : neighbours.keySet()) { // must synchronize since we may receive token synchronized (neighbours) { byte forkInfo = neighbours.get(neighbourId); oldForkInfo = forkInfo; // we stop eating right away, so a previously dirtied fork // (dirty forks can be reused) CAN be taken from under us if (!haveFork(forkInfo)) { //LOG.info("[[PTABLE]] " + vertexId + ": already lost fork " + // neighbourId + " " + toString(forkInfo)); continue; } else { if (haveToken(forkInfo)) { // fork will be sent outside of sync block forkInfo &= ~MASK_IS_DIRTY; forkInfo &= ~MASK_HAVE_FORK; //LOG.info("[[PTABLE]] " + vertexId + ": sending clean fork to " + // neighbourId + " " + toString(forkInfo)); } else { // otherwise, explicitly dirty the fork // (so that fork is released immediately on token receipt) forkInfo |= MASK_IS_DIRTY; //LOG.info("[[PTABLE]] " + vertexId + ": dirty fork " + // neighbourId + " " + toString(forkInfo)); } neighbours.put(neighbourId, forkInfo); } } if (haveFork(oldForkInfo) && haveToken(oldForkInfo)) { dstId.set(neighbourId); needFlush |= sendFork(vertexId, (I) dstId); } } // NOTE: this is also what flushes pending messages to the network, // so that other philosophers will see up-to-date messages (required // for serializability). If nobody needs a fork, messages will // get flushed only when that fork is requested. if (needFlush) { //LOG.info("[[PTABLE]] " + vertexId + ": flushing"); serviceWorker.getWorkerClient().waitAllRequests(); //LOG.info("[[PTABLE]] " + vertexId + ": done flush"); } }
From source file:org.apache.giraph.reducers.impl.LongXorReduce.java
License:Apache License
@Override public LongWritable reduce(LongWritable curValue, LongWritable valueToReduce) { curValue.set(curValue.get() ^ valueToReduce.get()); return curValue; }
From source file:org.apache.giraph.types.ByteToLongWritableWrapper.java
License:Apache License
@Override public void wrap(Byte javaValue, LongWritable writableValue) { writableValue.set(javaValue.longValue()); }
From source file:org.apache.giraph.types.IntToLongWritableWrapper.java
License:Apache License
@Override public void wrap(Integer javaValue, LongWritable writableValue) { writableValue.set(javaValue.longValue()); }