List of usage examples for org.apache.hadoop.io MapWritable put
@Override
public Writable put(Writable key, Writable value)
From source file:io.aos.hdfs.MapWritableTest.java
License:Apache License
@Test public void mapWritable() throws IOException { // vv MapWritableTest MapWritable src = new MapWritable(); src.put(new IntWritable(1), new Text("cat")); src.put(new VIntWritable(2), new LongWritable(163)); MapWritable dest = new MapWritable(); WritableUtils.cloneInto(dest, src);//www. j a v a2 s . c o m assertThat((Text) dest.get(new IntWritable(1)), is(new Text("cat"))); assertThat((LongWritable) dest.get(new VIntWritable(2)), is(new LongWritable(163))); // ^^ MapWritableTest }
From source file:io.aos.hdfs.MapWritableTest.java
License:Apache License
@Test public void setWritableEmulation() throws IOException { MapWritable src = new MapWritable(); src.put(new IntWritable(1), NullWritable.get()); src.put(new IntWritable(2), NullWritable.get()); MapWritable dest = new MapWritable(); WritableUtils.cloneInto(dest, src);//from w ww .ja v a 2 s. com assertThat(dest.containsKey(new IntWritable(1)), is(true)); }
From source file:io.druid.indexer.InputRowSerde.java
License:Apache License
public static final byte[] toBytes(final InputRow row, AggregatorFactory[] aggs) { try {//from ww w.ja v a2 s . com ByteArrayDataOutput out = ByteStreams.newDataOutput(); //write timestamp out.writeLong(row.getTimestampFromEpoch()); //writing all dimensions List<String> dimList = row.getDimensions(); Text[] dims = EMPTY_TEXT_ARRAY; if (dimList != null) { dims = new Text[dimList.size()]; for (int i = 0; i < dims.length; i++) { dims[i] = new Text(dimList.get(i)); } } StringArrayWritable sw = new StringArrayWritable(dims); sw.write(out); MapWritable mw = new MapWritable(); if (dimList != null) { for (String dim : dimList) { List<String> dimValue = row.getDimension(dim); if (dimValue == null || dimValue.size() == 0) { continue; } if (dimValue.size() == 1) { mw.put(new Text(dim), new Text(dimValue.get(0))); } else { Text[] dimValueArr = new Text[dimValue.size()]; for (int i = 0; i < dimValueArr.length; i++) { dimValueArr[i] = new Text(dimValue.get(i)); } mw.put(new Text(dim), new StringArrayWritable(dimValueArr)); } } } //writing all metrics Supplier<InputRow> supplier = new Supplier<InputRow>() { @Override public InputRow get() { return row; } }; for (AggregatorFactory aggFactory : aggs) { String k = aggFactory.getName(); Aggregator agg = aggFactory .factorize(IncrementalIndex.makeColumnSelectorFactory(aggFactory, supplier, true)); agg.aggregate(); String t = aggFactory.getTypeName(); if (t.equals("float")) { mw.put(new Text(k), new FloatWritable(agg.getFloat())); } else if (t.equals("long")) { mw.put(new Text(k), new LongWritable(agg.getLong())); } else { //its a complex metric Object val = agg.get(); ComplexMetricSerde serde = getComplexMetricSerde(t); mw.put(new Text(k), new BytesWritable(serde.toBytes(val))); } } mw.write(out); return out.toByteArray(); } catch (IOException ex) { throw Throwables.propagate(ex); } }
From source file:jadoop.HadoopGridTaskRunner.java
License:Open Source License
/** * The map function that executes the command line task and outputs the * return code, standard output and standard error. * //from w w w .j a va 2 s . c o m * The value provided as the input to this method will be a key and an array * of Text objects. The key matches the key provided when the HadoopGridTask * was added to the HadoopGridJob. The entries in the array are as follows: * <UL> * <LI>[0] - capStdOut: [true or false] If true, then all output to standard * output by the execution of the command should be captured and returned as * discussed below. If false, then standard output is not captured. This * will be the first element of the ArrayWritable. * <LI>[1] - capStdErr: [true or false] If true, then all output to standard * error by the execution of the command should be captured and returned as * discussed below. If false, then standard error is not captured. This will * be the second element in ArrayWritable. * <LI>[2] - timeout: The amount of time (in ms) that the task has to * execute. If the command does not complete before the timeout it is * terminated. * <LI>[3] - command: The command to be executed. Any arguments to the * command will be contained in subsequent entries. * <LI>[4]... - arguments to the command. These would be the individual * command line arguments if typed in at the command prompt. * </UL> * * <p> * For example: if standard output of an execution of the cal command for * June 2015 were to be captured the array entries would be: * <UL> * <LI>[0] - true * <LI>[1] - false * <LI>[2] - 1000 * <LI>[3] - cal * <LI>[4] - 6 * <LI>[5] - 2015 * </UL> * * <p> * Entries 0-2 are used as flags, the remaining entries are converted to an * array of Strings and used as the argument in a call to * Runtime.getRuntime().exec() to run the command. * * <p> * The key generated for the Mapper result will be the same key passed into * the mapper. The results generated for the Mapper will be a MapWritable * object with the following key/value pairs: * <UL> * <LI>EV,value : the value is a IntWritable containing the exit value * generated by the process created by the call to the Runtime.exec method. * <LI>TO,value : the value will be a BooleanWriteable indicating if the * task timed out (true) or not (false). * <LI>SO,value : the value is a Text containing the output written to * standard output by the executed program. * <LI>SE,value : the value is a Text containing the output written to * standard error by the executed program * </UL> * * @param key * a key that identifies this task. This will match the key * provided in the HadoopGridTask object. * @param value * the flags and command line as described above. * @param context * a Hadoop Context object provided by the Hadoop system. * * @throws InterruptedException * if there is a problem writing the task result to the context. * @throws IOException * if there is a problem writing the task result to the context. */ public void map(Text key, TextArrayWritable value, Context context) throws IOException, InterruptedException { String[] mapVal = value.toStrings(); boolean capStdOutput = Boolean.parseBoolean(mapVal[0]); boolean capStdErr = Boolean.parseBoolean(mapVal[1]); long timeout = Long.parseLong(mapVal[2]); // Build the command. String[] cmdInput = new String[mapVal.length - 3]; for (int i = 3; i < mapVal.length; i++) { cmdInput[i - 3] = mapVal[i]; } StringBuffer stdOutputStr = new StringBuffer(); StringBuffer errOutputStr = new StringBuffer(); byte exitValue = 0; boolean timedout = false; try { // Executes the command. Process p = Runtime.getRuntime().exec(cmdInput); long start = System.currentTimeMillis(); long cur = System.currentTimeMillis(); boolean done = false; while (!timedout && !done) { Thread.sleep(PROCSSESS_POLL_DELAY); /* * Check if the process has finished. If it has, the exit value * will come back, if not it throws an exception. */ try { exitValue = (byte) p.exitValue(); done = true; } catch (IllegalThreadStateException e) { // process not done yet, keep going... } cur = System.currentTimeMillis(); long elapsedTime = (cur - start); timedout = (elapsedTime >= timeout); // Keep long running tasks alive with hadoop. context.setStatus("Running for: " + elapsedTime + " ms."); } // Capture standard output generated by the command. if (capStdOutput) { BufferedReader stdOutputPrg = new BufferedReader(new InputStreamReader(p.getInputStream())); while (stdOutputPrg.ready()) { stdOutputStr.append(stdOutputPrg.readLine()); stdOutputStr.append("\n"); } } // Capture standard error generated by the command if (capStdErr) { BufferedReader stdErrPrg = new BufferedReader(new InputStreamReader(p.getErrorStream())); while (stdErrPrg.ready()) { errOutputStr.append(stdErrPrg.readLine()); errOutputStr.append("\n"); } } if (timedout) { p.destroy(); // kill the process. exitValue = -1; } } catch (Exception e) { /* * If an exception occurs put the message and stack trace on the end * of the standard error returned. */ errOutputStr.append("\n" + e.getMessage() + "\n" + e.getStackTrace()); exitValue = -1; } // Put the results into the context that is returned from this mapper. Text evKey = new Text("EV"); Text toKey = new Text("TO"); Text soKey = new Text("SO"); Text seKey = new Text("SE"); ByteWritable bwExitVal = new ByteWritable(exitValue); BooleanWritable bwTimeout = new BooleanWritable(timedout); Text tStdOutputStr = new Text(stdOutputStr.toString().trim()); Text tErrOutputStr = new Text(errOutputStr.toString().trim()); MapWritable mw = new MapWritable(); mw.put(evKey, bwExitVal); mw.put(toKey, bwTimeout); mw.put(soKey, tStdOutputStr); mw.put(seKey, tErrOutputStr); context.write(key, mw); }
From source file:net.sf.katta.lib.lucene.LuceneServer.java
License:Apache License
@Override public MapWritable getDetails(final String[] shards, final int docId, final String[] fieldNames) throws IOException { final SearcherHandle handle = getSearcherHandleByShard(shards[0]); IndexSearcher searcher = handle.getSearcher(); IndexReader ir = searcher.getIndexReader(); final MapWritable result = new MapWritable(); final Document doc = doc(shards[0], docId, fieldNames); final List<Fieldable> fields = doc.getFields(); for (final Fieldable field : fields) { final String name = field.name(); if (field.isBinary()) { final byte[] binaryValue = field.getBinaryValue(); result.put(new Text(name), new BytesWritable(binaryValue)); } else {//from w w w.j a v a2s . co m final String stringValue = field.stringValue(); result.put(new Text(name), new Text(stringValue)); } TermFreqVector tfv = ir.getTermFreqVector(docId, name); String terms[] = tfv.getTerms(); int freqs[] = tfv.getTermFrequencies(); MapWritable returnTerms = new MapWritable(); for (int t = 0; t < tfv.size(); t++) { returnTerms.put(new Text(terms[t]), new IntWritable(freqs[t])); } result.put(new Text(name + "_freqs"), returnTerms); } return result; }
From source file:org.apache.accumulo.core.client.mapreduce.lib.impl.InputConfigurator.java
License:Apache License
/** * Sets configurations for multiple tables at a time. * * @param implementingClass/*from www . j a v a2s. c o m*/ * the class whose name will be used as a prefix for the property configuration key * @param conf * the Hadoop configuration object to configure * @param configs * an array of {@link InputTableConfig} objects to associate with the job * @since 1.6.0 */ public static void setInputTableConfigs(Class<?> implementingClass, Configuration conf, Map<String, InputTableConfig> configs) { MapWritable mapWritable = new MapWritable(); for (Map.Entry<String, InputTableConfig> tableConfig : configs.entrySet()) mapWritable.put(new Text(tableConfig.getKey()), tableConfig.getValue()); ByteArrayOutputStream baos = new ByteArrayOutputStream(); try { mapWritable.write(new DataOutputStream(baos)); } catch (IOException e) { throw new IllegalStateException("Table configuration could not be serialized."); } String confKey = enumToConfKey(implementingClass, ScanOpts.TABLE_CONFIGS); conf.set(confKey, Base64.getEncoder().encodeToString(baos.toByteArray())); }
From source file:org.apache.accumulo.core.clientImpl.mapreduce.lib.InputConfigurator.java
License:Apache License
/** * Sets configurations for multiple tables at a time. * * @param implementingClass//from w ww . j a va 2s.c om * the class whose name will be used as a prefix for the property configuration key * @param conf * the Hadoop configuration object to configure * @param configs * an array of InputTableConfig objects to associate with the job * @since 1.6.0 */ public static void setInputTableConfigs(Class<?> implementingClass, Configuration conf, Map<String, org.apache.accumulo.core.client.mapreduce.InputTableConfig> configs) { MapWritable mapWritable = new MapWritable(); for (Map.Entry<String, org.apache.accumulo.core.client.mapreduce.InputTableConfig> tableConfig : configs .entrySet()) mapWritable.put(new Text(tableConfig.getKey()), tableConfig.getValue()); ByteArrayOutputStream baos = new ByteArrayOutputStream(); try { mapWritable.write(new DataOutputStream(baos)); } catch (IOException e) { throw new IllegalStateException("Table configuration could not be serialized."); } String confKey = enumToConfKey(implementingClass, ScanOpts.TABLE_CONFIGS); conf.set(confKey, Base64.getEncoder().encodeToString(baos.toByteArray())); }
From source file:org.apache.accumulo.hadoopImpl.mapreduce.lib.InputConfigurator.java
License:Apache License
public static void setExecutionHints(Class<?> implementingClass, Configuration conf, Map<String, String> hints) { MapWritable mapWritable = new MapWritable(); hints.forEach((k, v) -> mapWritable.put(new Text(k), new Text(v))); String key = enumToConfKey(implementingClass, ScanOpts.EXECUTION_HINTS); String val = toBase64(mapWritable); conf.set(key, val); }
From source file:org.apache.flume.channel.file.FlumeEvent.java
License:Apache License
private MapWritable toMapWritable(Map<String, String> map) { MapWritable result = new MapWritable(); if (map != null) { for (Map.Entry<String, String> entry : map.entrySet()) { result.put(new Text(entry.getKey()), new Text(entry.getValue())); }//from w w w .j a v a 2s . c om } return result; }
From source file:org.apache.gora.util.WritableUtils.java
License:Apache License
public static final void writeProperties(DataOutput out, Properties props) throws IOException { MapWritable propsWritable = new MapWritable(); for (Entry<Object, Object> prop : props.entrySet()) { Writable key = new Text(prop.getKey().toString()); Writable value = new Text(prop.getValue().toString()); propsWritable.put(key, value); }// w w w . j a va 2 s .co m propsWritable.write(out); }