List of usage examples for org.apache.hadoop.io MapWritable MapWritable
public MapWritable()
From source file:hbasemath.AbstractVector.java
License:Apache License
public void initMap(Result rs) { this.entries = new MapWritable(); NavigableMap<byte[], byte[]> map = rs.getFamilyMap(Constants.COLUMNFAMILY); for (Map.Entry<byte[], byte[]> e : map.entrySet()) { if (e != null) { this.entries.put(new IntWritable(BytesUtil.bytesToInt(e.getKey())), new DoubleWritable(Bytes.toDouble(e.getValue()))); }/*from w ww . ja v a 2 s . co m*/ } }
From source file:hbasemath.SparseVector.java
License:Apache License
public SparseVector() { this(new MapWritable()); }
From source file:hbasemath.SparseVector.java
License:Apache License
/** * Sets the value of index//from w w w. j ava 2 s .co m * * @param index * @param value */ public void set(int index, double value) { // If entries are null, create new object if (this.entries == null) { this.entries = new MapWritable(); } if (value != 0) // only stores non-zero element this.entries.put(new IntWritable(index), new DoubleWritable(value)); }
From source file:in.dream_lab.goffish.LongMapJSONReader.java
License:Apache License
@SuppressWarnings("unchecked") Vertex<V, E, LongWritable, LongWritable> createVertex(String JSONString) { JSONArray JSONInput = (JSONArray) JSONValue.parse(JSONString); LongWritable sourceID = new LongWritable(Long.valueOf(JSONInput.get(0).toString())); assert (vertexMap.get(sourceID) == null); Vertex<V, E, LongWritable, LongWritable> vertex = new Vertex<V, E, LongWritable, LongWritable>(sourceID); //fix this/*from ww w.j a v a2 s . c o m*/ //assumed value of jsonMap= "key1:type1:value1$ key2:type2:value2$....." //type could be Long or String or Double String jsonMap = JSONInput.get(1).toString(); String[] vprop = jsonMap.split(Pattern.quote("$")); //key,value property pairs for a vertex MapWritable vertexMap = new MapWritable(); for (int i = 0; i < vprop.length; i++) { String[] map = vprop[i].split(Pattern.quote(":")); Text key = new Text(map[0]); //FIXME:assuming String values for now Text value = new Text(map[2]); vertexMap.put(key, value); } V vertexValue = (V) vertexMap; vertex.setValue(vertexValue); JSONArray edgeList = (JSONArray) JSONInput.get(2); for (Object edgeInfo : edgeList) { Object edgeValues[] = ((JSONArray) edgeInfo).toArray(); LongWritable sinkID = new LongWritable(Long.valueOf(edgeValues[0].toString())); LongWritable edgeID = new LongWritable(Long.valueOf(edgeValues[1].toString())); //fix this //same format as vertex String[] eprop = edgeValues[2].toString().split(Pattern.quote("$")); MapWritable edgeMap = new MapWritable(); for (int i = 0; i < eprop.length; i++) { String[] map = eprop[i].split(Pattern.quote(":")); Text key = new Text(map[0]); //FIXME:assuming String values for now Text value = new Text(map[2]); edgeMap.put(key, value); } Edge<E, LongWritable, LongWritable> edge = new Edge<E, LongWritable, LongWritable>(edgeID, sinkID); E edgeValue = (E) edgeMap; edge.setValue(edgeValue); vertex.addEdge(edge); } return vertex; }
From source file:io.aos.hdfs.MapWritableTest.java
License:Apache License
@Test public void mapWritable() throws IOException { // vv MapWritableTest MapWritable src = new MapWritable(); src.put(new IntWritable(1), new Text("cat")); src.put(new VIntWritable(2), new LongWritable(163)); MapWritable dest = new MapWritable(); WritableUtils.cloneInto(dest, src);/*w ww .j a va 2s . c o m*/ assertThat((Text) dest.get(new IntWritable(1)), is(new Text("cat"))); assertThat((LongWritable) dest.get(new VIntWritable(2)), is(new LongWritable(163))); // ^^ MapWritableTest }
From source file:io.aos.hdfs.MapWritableTest.java
License:Apache License
@Test public void setWritableEmulation() throws IOException { MapWritable src = new MapWritable(); src.put(new IntWritable(1), NullWritable.get()); src.put(new IntWritable(2), NullWritable.get()); MapWritable dest = new MapWritable(); WritableUtils.cloneInto(dest, src);/* w w w .java 2s.c om*/ assertThat(dest.containsKey(new IntWritable(1)), is(true)); }
From source file:io.druid.indexer.InputRowSerde.java
License:Apache License
public static final byte[] toBytes(final InputRow row, AggregatorFactory[] aggs) { try {/*from w w w . j av a 2s. c om*/ ByteArrayDataOutput out = ByteStreams.newDataOutput(); //write timestamp out.writeLong(row.getTimestampFromEpoch()); //writing all dimensions List<String> dimList = row.getDimensions(); Text[] dims = EMPTY_TEXT_ARRAY; if (dimList != null) { dims = new Text[dimList.size()]; for (int i = 0; i < dims.length; i++) { dims[i] = new Text(dimList.get(i)); } } StringArrayWritable sw = new StringArrayWritable(dims); sw.write(out); MapWritable mw = new MapWritable(); if (dimList != null) { for (String dim : dimList) { List<String> dimValue = row.getDimension(dim); if (dimValue == null || dimValue.size() == 0) { continue; } if (dimValue.size() == 1) { mw.put(new Text(dim), new Text(dimValue.get(0))); } else { Text[] dimValueArr = new Text[dimValue.size()]; for (int i = 0; i < dimValueArr.length; i++) { dimValueArr[i] = new Text(dimValue.get(i)); } mw.put(new Text(dim), new StringArrayWritable(dimValueArr)); } } } //writing all metrics Supplier<InputRow> supplier = new Supplier<InputRow>() { @Override public InputRow get() { return row; } }; for (AggregatorFactory aggFactory : aggs) { String k = aggFactory.getName(); Aggregator agg = aggFactory .factorize(IncrementalIndex.makeColumnSelectorFactory(aggFactory, supplier, true)); agg.aggregate(); String t = aggFactory.getTypeName(); if (t.equals("float")) { mw.put(new Text(k), new FloatWritable(agg.getFloat())); } else if (t.equals("long")) { mw.put(new Text(k), new LongWritable(agg.getLong())); } else { //its a complex metric Object val = agg.get(); ComplexMetricSerde serde = getComplexMetricSerde(t); mw.put(new Text(k), new BytesWritable(serde.toBytes(val))); } } mw.write(out); return out.toByteArray(); } catch (IOException ex) { throw Throwables.propagate(ex); } }
From source file:io.druid.indexer.InputRowSerde.java
License:Apache License
public static final InputRow fromBytes(byte[] data, AggregatorFactory[] aggs) { try {//from ww w . j a va2 s . c om DataInput in = ByteStreams.newDataInput(data); //Read timestamp long timestamp = in.readLong(); //Read dimensions StringArrayWritable sw = new StringArrayWritable(); sw.readFields(in); List<String> dimensions = Arrays.asList(sw.toStrings()); MapWritable mw = new MapWritable(); mw.readFields(in); Map<String, Object> event = Maps.newHashMap(); for (String d : dimensions) { Writable v = mw.get(new Text(d)); if (v == null) { continue; } if (v instanceof Text) { event.put(d, ((Text) v).toString()); } else if (v instanceof StringArrayWritable) { event.put(d, Arrays.asList(((StringArrayWritable) v).toStrings())); } else { throw new ISE("unknown dim value type %s", v.getClass().getName()); } } //Read metrics for (AggregatorFactory aggFactory : aggs) { String k = aggFactory.getName(); Writable v = mw.get(new Text(k)); if (v == null) { continue; } String t = aggFactory.getTypeName(); if (t.equals("float")) { event.put(k, ((FloatWritable) v).get()); } else if (t.equals("long")) { event.put(k, ((LongWritable) v).get()); } else { //its a complex metric ComplexMetricSerde serde = getComplexMetricSerde(t); BytesWritable bw = (BytesWritable) v; event.put(k, serde.fromBytes(bw.getBytes(), 0, bw.getLength())); } } return new MapBasedInputRow(timestamp, dimensions, event); } catch (IOException ex) { throw Throwables.propagate(ex); } }
From source file:jadoop.HadoopGridJob.java
License:Open Source License
/** * Process the results that were returned by the Hadoop job. Each result * will be a key value pair with the format specified in the * HadoopGridTaskRunner class. The results for each key should be parsed and * placed into the HadoopGridTask object with the same key. * /*w w w . ja va2 s .c o m*/ * @see HadoopGridTaskRunner * * @throws IOException * if there is a problem reading the results. */ private void processResults(FileSystem fs, Path outDir) throws IOException { FileStatus[] fileStatus = fs.listStatus(outDir); /* * Process the results for all of the tasks that have completed. Any * task that did not complete will be included in any file. */ for (FileStatus file : fileStatus) { String fileName = file.getPath().getName(); if (fileName.contains("part-m-")) { Path filePath = new Path(outDir + "/" + fileName); SequenceFile.Reader reader = new SequenceFile.Reader(job.getConfiguration(), SequenceFile.Reader.file(filePath)); Text mapperOutputKey = new Text(); MapWritable mapperOutputVal = new MapWritable(); /* * If multiple tasks are sent to the same node then the response * file will contain multiple entries. Be sure to process each * one of them. */ while (reader.next(mapperOutputKey, mapperOutputVal)) { // Get the value returned from the HadoopGridTaskRunner. byte exitValue = ((ByteWritable) mapperOutputVal.get(new Text("EV"))).get(); boolean taskTO = ((BooleanWritable) mapperOutputVal.get(new Text("TO"))).get(); String stdOut = ((Text) mapperOutputVal.get(new Text("SO"))).toString(); String stdErr = ((Text) mapperOutputVal.get(new Text("SE"))).toString(); HadoopGridTask task = getTask(mapperOutputKey.toString()); if (taskTO) { task.markAsTimedout(); } else { // change the task's exit value. task.markAsFinished(exitValue); } if (task.captureStandardOutput()) { task.setStandardOutput(stdOut); } if (task.captureStandardError()) { task.setStandardError(stdErr); } } reader.close(); } } }
From source file:jadoop.HadoopGridTaskRunner.java
License:Open Source License
/** * The map function that executes the command line task and outputs the * return code, standard output and standard error. * /* w w w . j av a 2 s .co m*/ * The value provided as the input to this method will be a key and an array * of Text objects. The key matches the key provided when the HadoopGridTask * was added to the HadoopGridJob. The entries in the array are as follows: * <UL> * <LI>[0] - capStdOut: [true or false] If true, then all output to standard * output by the execution of the command should be captured and returned as * discussed below. If false, then standard output is not captured. This * will be the first element of the ArrayWritable. * <LI>[1] - capStdErr: [true or false] If true, then all output to standard * error by the execution of the command should be captured and returned as * discussed below. If false, then standard error is not captured. This will * be the second element in ArrayWritable. * <LI>[2] - timeout: The amount of time (in ms) that the task has to * execute. If the command does not complete before the timeout it is * terminated. * <LI>[3] - command: The command to be executed. Any arguments to the * command will be contained in subsequent entries. * <LI>[4]... - arguments to the command. These would be the individual * command line arguments if typed in at the command prompt. * </UL> * * <p> * For example: if standard output of an execution of the cal command for * June 2015 were to be captured the array entries would be: * <UL> * <LI>[0] - true * <LI>[1] - false * <LI>[2] - 1000 * <LI>[3] - cal * <LI>[4] - 6 * <LI>[5] - 2015 * </UL> * * <p> * Entries 0-2 are used as flags, the remaining entries are converted to an * array of Strings and used as the argument in a call to * Runtime.getRuntime().exec() to run the command. * * <p> * The key generated for the Mapper result will be the same key passed into * the mapper. The results generated for the Mapper will be a MapWritable * object with the following key/value pairs: * <UL> * <LI>EV,value : the value is a IntWritable containing the exit value * generated by the process created by the call to the Runtime.exec method. * <LI>TO,value : the value will be a BooleanWriteable indicating if the * task timed out (true) or not (false). * <LI>SO,value : the value is a Text containing the output written to * standard output by the executed program. * <LI>SE,value : the value is a Text containing the output written to * standard error by the executed program * </UL> * * @param key * a key that identifies this task. This will match the key * provided in the HadoopGridTask object. * @param value * the flags and command line as described above. * @param context * a Hadoop Context object provided by the Hadoop system. * * @throws InterruptedException * if there is a problem writing the task result to the context. * @throws IOException * if there is a problem writing the task result to the context. */ public void map(Text key, TextArrayWritable value, Context context) throws IOException, InterruptedException { String[] mapVal = value.toStrings(); boolean capStdOutput = Boolean.parseBoolean(mapVal[0]); boolean capStdErr = Boolean.parseBoolean(mapVal[1]); long timeout = Long.parseLong(mapVal[2]); // Build the command. String[] cmdInput = new String[mapVal.length - 3]; for (int i = 3; i < mapVal.length; i++) { cmdInput[i - 3] = mapVal[i]; } StringBuffer stdOutputStr = new StringBuffer(); StringBuffer errOutputStr = new StringBuffer(); byte exitValue = 0; boolean timedout = false; try { // Executes the command. Process p = Runtime.getRuntime().exec(cmdInput); long start = System.currentTimeMillis(); long cur = System.currentTimeMillis(); boolean done = false; while (!timedout && !done) { Thread.sleep(PROCSSESS_POLL_DELAY); /* * Check if the process has finished. If it has, the exit value * will come back, if not it throws an exception. */ try { exitValue = (byte) p.exitValue(); done = true; } catch (IllegalThreadStateException e) { // process not done yet, keep going... } cur = System.currentTimeMillis(); long elapsedTime = (cur - start); timedout = (elapsedTime >= timeout); // Keep long running tasks alive with hadoop. context.setStatus("Running for: " + elapsedTime + " ms."); } // Capture standard output generated by the command. if (capStdOutput) { BufferedReader stdOutputPrg = new BufferedReader(new InputStreamReader(p.getInputStream())); while (stdOutputPrg.ready()) { stdOutputStr.append(stdOutputPrg.readLine()); stdOutputStr.append("\n"); } } // Capture standard error generated by the command if (capStdErr) { BufferedReader stdErrPrg = new BufferedReader(new InputStreamReader(p.getErrorStream())); while (stdErrPrg.ready()) { errOutputStr.append(stdErrPrg.readLine()); errOutputStr.append("\n"); } } if (timedout) { p.destroy(); // kill the process. exitValue = -1; } } catch (Exception e) { /* * If an exception occurs put the message and stack trace on the end * of the standard error returned. */ errOutputStr.append("\n" + e.getMessage() + "\n" + e.getStackTrace()); exitValue = -1; } // Put the results into the context that is returned from this mapper. Text evKey = new Text("EV"); Text toKey = new Text("TO"); Text soKey = new Text("SO"); Text seKey = new Text("SE"); ByteWritable bwExitVal = new ByteWritable(exitValue); BooleanWritable bwTimeout = new BooleanWritable(timedout); Text tStdOutputStr = new Text(stdOutputStr.toString().trim()); Text tErrOutputStr = new Text(errOutputStr.toString().trim()); MapWritable mw = new MapWritable(); mw.put(evKey, bwExitVal); mw.put(toKey, bwTimeout); mw.put(soKey, tStdOutputStr); mw.put(seKey, tErrOutputStr); context.write(key, mw); }