List of usage examples for org.apache.hadoop.io Text writeString
public static int writeString(DataOutput out, String s) throws IOException
From source file:ml.shifu.guagua.yarn.GuaguaSplitWriter.java
License:Apache License
private static SplitMetaInfo[] writeOldSplits(org.apache.hadoop.mapred.InputSplit[] splits, FSDataOutputStream out, Configuration conf) throws IOException { SplitMetaInfo[] info = new SplitMetaInfo[splits.length]; if (splits.length != 0) { int i = 0; long offset = out.getPos(); for (org.apache.hadoop.mapred.InputSplit split : splits) { long prevLen = out.getPos(); Text.writeString(out, split.getClass().getName()); split.write(out);//from ww w . ja v a 2 s. c o m long currLen = out.getPos(); String[] locations = split.getLocations(); final int max_loc = conf.getInt(MAX_SPLIT_LOCATIONS, 10); if (locations.length > max_loc) { LOG.warn("Max block location exceeded for split: " + split + " splitsize: " + locations.length + " maxsize: " + max_loc); locations = Arrays.copyOf(locations, max_loc); } info[i++] = new JobSplit.SplitMetaInfo(locations, offset, split.getLength()); offset += currLen - prevLen; } } return info; }
From source file:net.thevis.groovyhadoop.backport.CombineFileSplit.java
License:Apache License
public void write(DataOutput out) throws IOException { out.writeLong(totLength);//from w ww .ja v a 2 s . com out.writeInt(lengths.length); for (long length : lengths) { out.writeLong(length); } out.writeInt(paths.length); for (Path p : paths) { Text.writeString(out, p.toString()); } out.writeInt(startoffset.length); for (long length : startoffset) { out.writeLong(length); } }
From source file:oracle.kv.hadoop.KVInputSplit.java
License:Open Source License
/** * Serialize the fields of this object to <code>out</code>. * * @param out <code>DataOuput</code> to serialize this object into. * @throws IOException//from w w w . j av a 2 s .c o m */ @Override public void write(DataOutput out) throws IOException { out.writeInt(kvHelperHosts.length); for (int i = 0; i < kvHelperHosts.length; i++) { Text.writeString(out, kvHelperHosts[i]); } Text.writeString(out, kvStore); Text.writeString(out, "" + kvPart); /* If kvPart == 0, then this is a multi-partition split. */ if (kvPart == 0) { assert partitionSets != null; out.writeInt(partitionSets.size()); for (Set<Integer> partitions : partitionSets) { out.writeInt(partitions.size()); for (Integer p : partitions) { out.writeInt(p); } } } Text.writeString(out, (direction == null ? "" : direction.name())); out.writeInt(batchSize); writeBytes(out, (parentKey == null ? null : parentKey.toByteArray())); writeBytes(out, (subRange == null ? null : subRange.toByteArray())); Text.writeString(out, (depth == null ? "" : depth.name())); writeBytes(out, (consistency == null ? null : consistency.toByteArray())); out.writeLong(timeout); Text.writeString(out, (timeoutUnit == null ? "" : timeoutUnit.name())); out.writeInt(locations.length); for (int i = 0; i < locations.length; i++) { Text.writeString(out, locations[i]); } Text.writeString(out, (formatterClassName == null ? "" : formatterClassName)); Text.writeString(out, (kvStoreSecurityFile == null ? "" : kvStoreSecurityFile)); }
From source file:oracle.kv.hadoop.table.TableInputSplit.java
License:Open Source License
/** * Serialize the fields of this object to <code>out</code>. * * @param out <code>DataOuput</code> to serialize this object into. * @throws IOException//from ww w. j av a 2 s. c o m */ @Override public void write(DataOutput out) throws IOException { out.writeInt(locations.length); for (int i = 0; i < locations.length; i++) { Text.writeString(out, locations[i]); } out.writeInt(kvHelperHosts.length); for (int i = 0; i < kvHelperHosts.length; i++) { Text.writeString(out, kvHelperHosts[i]); } Text.writeString(out, kvStore); Text.writeString(out, tableName); Text.writeString(out, (primaryKeyProperty == null ? EMPTY_STR : primaryKeyProperty)); /* For MultiRowOptions */ Text.writeString(out, (fieldRangeProperty == null ? EMPTY_STR : fieldRangeProperty)); /* For TableIteratorOptions */ Text.writeString(out, (direction == null ? EMPTY_STR : direction.name())); writeBytes(out, (consistency == null ? null : consistency.toByteArray())); out.writeLong(timeout); Text.writeString(out, (timeoutUnit == null ? EMPTY_STR : timeoutUnit.name())); out.writeInt(maxRequests); out.writeInt(batchSize); out.writeInt(maxBatches); out.writeInt(partitionSets.size()); for (Set<Integer> partitions : partitionSets) { out.writeInt(partitions.size()); for (Integer p : partitions) { out.writeInt(p); } } out.writeInt(queryBy); out.writeInt(shardSet.size()); for (RepGroupId repGroupId : shardSet) { out.writeInt(repGroupId.getGroupId()); } /* Serialize empty string rather than null to avoid NPE from Hadoop. */ Text.writeString(out, (onqlWhereClause == null ? EMPTY_STR : onqlWhereClause)); /* * Write the name of the login file, the user credentials (username * and password), and the name of the trust file. If name of the * login file is null, leave the others null too. */ if (loginFlnm == null) { Text.writeString(out, EMPTY_STR); /* login file name */ Text.writeString(out, EMPTY_STR); /* username */ Text.writeString(out, EMPTY_STR); /* password */ Text.writeString(out, EMPTY_STR); /* trust file name */ } else { Text.writeString(out, loginFlnm); Text.writeString(out, pwdCredentials.getUsername()); Text.writeString(out, String.copyValueOf(pwdCredentials.getPassword())); Text.writeString(out, trustFlnm); } }
From source file:org.apache.avro.mapred.TaggedInputSplit.java
License:Apache License
public void write(DataOutput out) throws IOException { Text.writeString(out, inputSplitClass.getName()); inputSplit.write(out);// w w w . java2s .c o m Text.writeString(out, inputFormatClass.getName()); Text.writeString(out, mapperClass.getName()); Text.writeString(out, schema.toString()); }
From source file:org.apache.crunch.impl.mr.run.CrunchInputSplit.java
License:Apache License
public void write(DataOutput out) throws IOException { out.writeInt(nodeIndex);/* w ww . j a v a2s .c om*/ bundle.write(out); Text.writeString(out, inputSplit.getClass().getName()); SerializationFactory factory = new SerializationFactory(conf); Serializer serializer = factory.getSerializer(inputSplit.getClass()); serializer.open((DataOutputStream) out); serializer.serialize(inputSplit); }
From source file:org.apache.crunch.io.FormatBundle.java
License:Apache License
@Override public void write(DataOutput out) throws IOException { Text.writeString(out, formatClass.getName()); out.writeInt(extraConf.size());//w w w . j av a 2 s . co m for (Map.Entry<String, String> e : extraConf.entrySet()) { Text.writeString(out, e.getKey()); Text.writeString(out, e.getValue()); } }
From source file:org.apache.crunch.types.writable.GenericArrayWritable.java
License:Apache License
public void write(DataOutput out) throws IOException { WritableUtils.writeVInt(out, values.length); if (values.length > 0) { int nulls = 0; for (int i = 0; i < values.length; i++) { if (values[i] == null) { nulls++;/*w ww . j a v a2s . co m*/ } } WritableUtils.writeVInt(out, nulls); if (values.length - nulls > 0) { if (valueClass == null) { throw new IllegalStateException("Value class not set by constructor or read"); } Text.writeString(out, valueClass.getName()); for (int i = 0; i < values.length; i++) { if (values[i] != null) { values[i].write(out); } } } } }
From source file:org.apache.giraph.comm.requests.SendWorkerToWorkerMessageRequest.java
License:Apache License
@Override void writeRequest(DataOutput output) throws IOException { Text.writeString(output, message.getClass().getName()); message.write(output); }
From source file:org.apache.giraph.graph.BspServiceMaster.java
License:Apache License
@Override public int createInputSplits() { // Only the 'master' should be doing this. Wait until the number of // processes that have reported health exceeds the minimum percentage. // If the minimum percentage is not met, fail the job. Otherwise // generate the input splits try {/* w ww . j a va 2 s .c o m*/ if (getZkExt().exists(inputSplitsPath, false) != null) { LOG.info(inputSplitsPath + " already exists, no need to create"); return Integer.parseInt(new String(getZkExt().getData(inputSplitsPath, false, null))); } } catch (KeeperException.NoNodeException e) { if (LOG.isInfoEnabled()) { LOG.info("createInputSplits: Need to create the " + "input splits at " + inputSplitsPath); } } catch (KeeperException e) { throw new IllegalStateException("createInputSplits: KeeperException", e); } catch (InterruptedException e) { throw new IllegalStateException("createInputSplits: InterrtupedException", e); } // When creating znodes, in case the master has already run, resume // where it left off. List<WorkerInfo> healthyWorkerInfoList = checkWorkers(); if (healthyWorkerInfoList == null) { setJobState(ApplicationState.FAILED, -1, -1); return -1; } // Note that the input splits may only be a sample if // INPUT_SPLIT_SAMPLE_PERCENT is set to something other than 100 List<InputSplit> splitList = generateInputSplits(healthyWorkerInfoList.size()); if (healthyWorkerInfoList.size() > splitList.size()) { LOG.warn("createInputSplits: Number of inputSplits=" + splitList.size() + " < " + healthyWorkerInfoList.size() + "=number of healthy processes, " + "some workers will be not used"); } String inputSplitPath = null; for (int i = 0; i < splitList.size(); ++i) { try { ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); DataOutput outputStream = new DataOutputStream(byteArrayOutputStream); InputSplit inputSplit = splitList.get(i); Text.writeString(outputStream, inputSplit.getClass().getName()); ((Writable) inputSplit).write(outputStream); inputSplitPath = inputSplitsPath + "/" + i; getZkExt().createExt(inputSplitPath, byteArrayOutputStream.toByteArray(), Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT, true); if (LOG.isDebugEnabled()) { LOG.debug("createInputSplits: Created input split " + "with index " + i + " serialized as " + byteArrayOutputStream.toString()); } } catch (KeeperException.NodeExistsException e) { if (LOG.isInfoEnabled()) { LOG.info("createInputSplits: Node " + inputSplitPath + " already exists."); } } catch (KeeperException e) { throw new IllegalStateException("createInputSplits: KeeperException", e); } catch (InterruptedException e) { throw new IllegalStateException("createInputSplits: IllegalStateException", e); } catch (IOException e) { throw new IllegalStateException("createInputSplits: IOException", e); } } // Let workers know they can start trying to load the input splits try { getZkExt().create(inputSplitsAllReadyPath, null, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); } catch (KeeperException.NodeExistsException e) { LOG.info("createInputSplits: Node " + inputSplitsAllReadyPath + " already exists."); } catch (KeeperException e) { throw new IllegalStateException("createInputSplits: KeeperException", e); } catch (InterruptedException e) { throw new IllegalStateException("createInputSplits: IllegalStateException", e); } return splitList.size(); }