Example usage for org.apache.hadoop.io MapWritable MapWritable

List of usage examples for org.apache.hadoop.io MapWritable MapWritable

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.io.*
  5. MapWritable
  6. MapWritable

Introduction

In this page you can find the example usage for org.apache.hadoop.io MapWritable MapWritable.

Prototype

public MapWritable()

Source Link

Document

Default constructor.

Usage

From source file:net.sf.katta.lib.lucene.LuceneServer.java

License:Apache License

@Override
public MapWritable getDetails(final String[] shards, final int docId, final String[] fieldNames)
        throws IOException {
    final SearcherHandle handle = getSearcherHandleByShard(shards[0]);
    IndexSearcher searcher = handle.getSearcher();
    IndexReader ir = searcher.getIndexReader();

    final MapWritable result = new MapWritable();
    final Document doc = doc(shards[0], docId, fieldNames);
    final List<Fieldable> fields = doc.getFields();
    for (final Fieldable field : fields) {
        final String name = field.name();
        if (field.isBinary()) {
            final byte[] binaryValue = field.getBinaryValue();
            result.put(new Text(name), new BytesWritable(binaryValue));
        } else {//from   www. ja  va 2  s. c  om
            final String stringValue = field.stringValue();
            result.put(new Text(name), new Text(stringValue));
        }

        TermFreqVector tfv = ir.getTermFreqVector(docId, name);
        String terms[] = tfv.getTerms();
        int freqs[] = tfv.getTermFrequencies();
        MapWritable returnTerms = new MapWritable();
        for (int t = 0; t < tfv.size(); t++) {
            returnTerms.put(new Text(terms[t]), new IntWritable(freqs[t]));
        }
        result.put(new Text(name + "_freqs"), returnTerms);
    }
    return result;
}

From source file:nutchIndexer.NutchMap.java

License:Open Source License

@Override
public void map(LongWritable key, Text value, Mapper.Context context) throws IOException, InterruptedException {
    TrecOLParser document = new TrecOLParser(value.toString());
    documentAnalyzed = new MapWritable();
    if (document.isParsed()) {
        this.tokenizer.tokenize(document.getDocContent());
        while (this.tokenizer.hasMoreTokens()) {
            IntWritable counter = CastingTypes.zero;
            String newTerm = this.tokenizer.nextToken();
            Text term = new Text(newTerm);
            if (documentAnalyzed.containsKey(term)) {
                counter = CastingTypes.strToIntWr(documentAnalyzed.get(term).toString());
            }//from ww w . jav a 2s . co  m
            documentAnalyzed.put(term, CastingTypes.intToIntWr(counter.get() + 1));
        }
        if (!documentAnalyzed.isEmpty()) {
            context.write(CastingTypes.strToIntWr(document.getDocId()), documentAnalyzed);
        }
    }
}

From source file:oracle.kv.hadoop.hive.table.TableSerDeBase.java

License:Open Source License

@Override
public void initialize(Configuration job, Properties tbl) throws SerDeException {
    initKvStoreParams(tbl);/*  w w w.  j a v a2s.  com*/
    serdeParams = initSerdeParams(job, tbl);
    /* For degugging. */
    displayInitParams(serdeParams);
    validateParams(tbl);
    objInspector = createObjectInspector();
    hiveRow = new ArrayList<Object>();
    kvMapWritable = new MapWritable();
}

From source file:org.apache.accumulo.core.client.mapreduce.lib.impl.InputConfigurator.java

License:Apache License

/**
 * Sets configurations for multiple tables at a time.
 *
 * @param implementingClass/*from  www .j av a 2s.c  om*/
 *          the class whose name will be used as a prefix for the property configuration key
 * @param conf
 *          the Hadoop configuration object to configure
 * @param configs
 *          an array of {@link InputTableConfig} objects to associate with the job
 * @since 1.6.0
 */
public static void setInputTableConfigs(Class<?> implementingClass, Configuration conf,
        Map<String, InputTableConfig> configs) {
    MapWritable mapWritable = new MapWritable();
    for (Map.Entry<String, InputTableConfig> tableConfig : configs.entrySet())
        mapWritable.put(new Text(tableConfig.getKey()), tableConfig.getValue());

    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    try {
        mapWritable.write(new DataOutputStream(baos));
    } catch (IOException e) {
        throw new IllegalStateException("Table configuration could not be serialized.");
    }

    String confKey = enumToConfKey(implementingClass, ScanOpts.TABLE_CONFIGS);
    conf.set(confKey, Base64.getEncoder().encodeToString(baos.toByteArray()));
}

From source file:org.apache.accumulo.core.client.mapreduce.lib.impl.InputConfigurator.java

License:Apache License

/**
 * Returns all {@link InputTableConfig} objects associated with this job.
 *
 * @param implementingClass/*from   w w  w. j  a v a 2  s.  com*/
 *          the class whose name will be used as a prefix for the property configuration key
 * @param conf
 *          the Hadoop configuration object to configure
 * @return all of the table query configs for the job
 * @since 1.6.0
 */
public static Map<String, InputTableConfig> getInputTableConfigs(Class<?> implementingClass,
        Configuration conf) {
    Map<String, InputTableConfig> configs = new HashMap<>();
    Map.Entry<String, InputTableConfig> defaultConfig = getDefaultInputTableConfig(implementingClass, conf);
    if (defaultConfig != null)
        configs.put(defaultConfig.getKey(), defaultConfig.getValue());
    String configString = conf.get(enumToConfKey(implementingClass, ScanOpts.TABLE_CONFIGS));
    MapWritable mapWritable = new MapWritable();
    if (configString != null) {
        try {
            byte[] bytes = Base64.getDecoder().decode(configString);
            ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
            mapWritable.readFields(new DataInputStream(bais));
            bais.close();
        } catch (IOException e) {
            throw new IllegalStateException(
                    "The table query configurations could not be deserialized from the given configuration");
        }
    }
    for (Map.Entry<Writable, Writable> entry : mapWritable.entrySet())
        configs.put(((Text) entry.getKey()).toString(), (InputTableConfig) entry.getValue());

    return configs;
}

From source file:org.apache.accumulo.core.clientImpl.mapreduce.lib.InputConfigurator.java

License:Apache License

/**
 * Sets configurations for multiple tables at a time.
 *
 * @param implementingClass// w w  w .j av  a 2 s  .  c  o  m
 *          the class whose name will be used as a prefix for the property configuration key
 * @param conf
 *          the Hadoop configuration object to configure
 * @param configs
 *          an array of InputTableConfig objects to associate with the job
 * @since 1.6.0
 */
public static void setInputTableConfigs(Class<?> implementingClass, Configuration conf,
        Map<String, org.apache.accumulo.core.client.mapreduce.InputTableConfig> configs) {
    MapWritable mapWritable = new MapWritable();
    for (Map.Entry<String, org.apache.accumulo.core.client.mapreduce.InputTableConfig> tableConfig : configs
            .entrySet())
        mapWritable.put(new Text(tableConfig.getKey()), tableConfig.getValue());

    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    try {
        mapWritable.write(new DataOutputStream(baos));
    } catch (IOException e) {
        throw new IllegalStateException("Table configuration could not be serialized.");
    }

    String confKey = enumToConfKey(implementingClass, ScanOpts.TABLE_CONFIGS);
    conf.set(confKey, Base64.getEncoder().encodeToString(baos.toByteArray()));
}

From source file:org.apache.accumulo.core.clientImpl.mapreduce.lib.InputConfigurator.java

License:Apache License

/**
 * Returns all InputTableConfig objects associated with this job.
 *
 * @param implementingClass/*from   w ww . ja v  a  2 s .c o  m*/
 *          the class whose name will be used as a prefix for the property configuration key
 * @param conf
 *          the Hadoop configuration object to configure
 * @return all of the table query configs for the job
 * @since 1.6.0
 */
public static Map<String, org.apache.accumulo.core.client.mapreduce.InputTableConfig> getInputTableConfigs(
        Class<?> implementingClass, Configuration conf) {
    Map<String, org.apache.accumulo.core.client.mapreduce.InputTableConfig> configs = new HashMap<>();
    Map.Entry<String, org.apache.accumulo.core.client.mapreduce.InputTableConfig> defaultConfig = getDefaultInputTableConfig(
            implementingClass, conf);
    if (defaultConfig != null)
        configs.put(defaultConfig.getKey(), defaultConfig.getValue());
    String configString = conf.get(enumToConfKey(implementingClass, ScanOpts.TABLE_CONFIGS));
    MapWritable mapWritable = new MapWritable();
    if (configString != null) {
        try {
            byte[] bytes = Base64.getDecoder().decode(configString);
            ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
            mapWritable.readFields(new DataInputStream(bais));
            bais.close();
        } catch (IOException e) {
            throw new IllegalStateException("The table query configurations could not be deserialized"
                    + " from the given configuration");
        }
    }
    for (Map.Entry<Writable, Writable> entry : mapWritable.entrySet())
        configs.put(entry.getKey().toString(),
                (org.apache.accumulo.core.client.mapreduce.InputTableConfig) entry.getValue());

    return configs;
}

From source file:org.apache.accumulo.hadoopImpl.mapreduce.lib.InputConfigurator.java

License:Apache License

/**
 * Returns all {@link InputTableConfig} objects associated with this job.
 *
 * @param implementingClass/*  ww  w .j a  v a 2  s. c  om*/
 *          the class whose name will be used as a prefix for the property configuration key
 * @param conf
 *          the Hadoop configuration object to configure
 * @param tableName
 *          the table name for which to retrieve the configuration
 * @return all of the table query configs for the job
 * @since 1.6.0
 */
private static Map<String, InputTableConfig> getInputTableConfigs(Class<?> implementingClass,
        Configuration conf, String tableName) {
    Map<String, InputTableConfig> configs = new HashMap<>();
    Map.Entry<String, InputTableConfig> defaultConfig = getDefaultInputTableConfig(implementingClass, conf,
            tableName);
    if (defaultConfig != null)
        configs.put(defaultConfig.getKey(), defaultConfig.getValue());
    String configString = conf.get(enumToConfKey(implementingClass, ScanOpts.TABLE_CONFIGS));
    MapWritable mapWritable = new MapWritable();
    if (configString != null) {
        try {
            byte[] bytes = Base64.getDecoder().decode(configString);
            ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
            mapWritable.readFields(new DataInputStream(bais));
            bais.close();
        } catch (IOException e) {
            throw new IllegalStateException("The table query configurations could not be deserialized"
                    + " from the given configuration");
        }
    }
    for (Map.Entry<Writable, Writable> entry : mapWritable.entrySet())
        configs.put(entry.getKey().toString(), (InputTableConfig) entry.getValue());

    return configs;
}

From source file:org.apache.accumulo.hadoopImpl.mapreduce.lib.InputConfigurator.java

License:Apache License

public static void setExecutionHints(Class<?> implementingClass, Configuration conf,
        Map<String, String> hints) {
    MapWritable mapWritable = new MapWritable();
    hints.forEach((k, v) -> mapWritable.put(new Text(k), new Text(v)));

    String key = enumToConfKey(implementingClass, ScanOpts.EXECUTION_HINTS);
    String val = toBase64(mapWritable);

    conf.set(key, val);
}

From source file:org.apache.accumulo.hadoopImpl.mapreduce.lib.InputConfigurator.java

License:Apache License

public static Map<String, String> getExecutionHints(Class<?> implementingClass, Configuration conf) {
    String key = enumToConfKey(implementingClass, ScanOpts.EXECUTION_HINTS);
    String encodedEH = conf.get(key);
    if (encodedEH == null) {
        return Collections.emptyMap();
    }/*from   ww  w  . j ava  2  s  . co  m*/

    MapWritable mapWritable = new MapWritable();
    fromBase64(mapWritable, encodedEH);

    HashMap<String, String> hints = new HashMap<>();
    mapWritable.forEach((k, v) -> hints.put(k.toString(), v.toString()));
    return hints;
}