Example usage for org.apache.hadoop.conf Configuration set

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration set.

Prototype

public void set(String name, String value) 

Source Link

Document

Set the value of the name property.

Usage

From source file:com.chinamobile.bcbsp.io.BSPFileOutputFormat.java

License:Apache License

/**
 * Set the {@link Path} of the output directory for the BC-BSP job.
 *
 * @param job/*from   w  w w. j  ava 2  s. co  m*/
 *        the current BSPJob job.
 * @param outputDir
 *        the {@link Path} of the output directory for the BC-BSP job.
 */
public static void setOutputPath(BSPJob job, Path outputDir) {
    Configuration conf = job.getConf();
    checkOutputSpecs(job, outputDir);
    conf.set(Constants.USER_BC_BSP_JOB_OUTPUT_DIR, outputDir.toString());
}

---

From source file:com.citic.zxyjs.zwlscx.mapreduce.lib.input.HFileOutputFormatBase.java

License:Apache License

private static void configureBlockSize(HTable table, Configuration conf) throws IOException {
    StringBuilder blockSizeConfigValue = new StringBuilder();
    HTableDescriptor tableDescriptor = table.getTableDescriptor();
    if (tableDescriptor == null) {
        // could happen with mock table instance
        return;/*w w w . ja va  2s. com*/
    }
    Collection<HColumnDescriptor> families = tableDescriptor.getFamilies();
    int i = 0;
    for (HColumnDescriptor familyDescriptor : families) {
        if (i++ > 0) {
            blockSizeConfigValue.append('&');
        }
        blockSizeConfigValue.append(URLEncoder.encode(familyDescriptor.getNameAsString(), "UTF-8"));
        blockSizeConfigValue.append('=');
        blockSizeConfigValue
                .append(URLEncoder.encode(String.valueOf(familyDescriptor.getBlocksize()), "UTF-8"));
    }
    // Get rid of the last ampersand
    conf.set(BLOCK_SIZE_CONF_KEY, blockSizeConfigValue.toString());
}

---

From source file:com.citic.zxyjs.zwlscx.mapreduce.lib.input.HFileOutputFormatBase.java

License:Apache License

/**
 * Serialize column family to compression algorithm map to configuration.
 * Invoked while configuring the MR job for incremental load.
 * Package-private for unit tests only.//  w w w .  j  a  v a2s.  c om
 * 
 * @throws IOException
 *             on failure to read column family descriptors
 */
@edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "RCN_REDUNDANT_NULLCHECK_OF_NONNULL_VALUE")
static void configureCompression(HTable table, Configuration conf) throws IOException {
    StringBuilder compressionConfigValue = new StringBuilder();
    HTableDescriptor tableDescriptor = table.getTableDescriptor();
    if (tableDescriptor == null) {
        // could happen with mock table instance
        return;
    }
    Collection<HColumnDescriptor> families = tableDescriptor.getFamilies();
    int i = 0;
    for (HColumnDescriptor familyDescriptor : families) {
        if (i++ > 0) {
            compressionConfigValue.append('&');
        }
        compressionConfigValue.append(URLEncoder.encode(familyDescriptor.getNameAsString(), "UTF-8"));
        compressionConfigValue.append('=');
        compressionConfigValue.append(URLEncoder.encode(familyDescriptor.getCompression().getName(), "UTF-8"));
    }
    // Get rid of the last ampersand
    conf.set(COMPRESSION_CONF_KEY, compressionConfigValue.toString());
}

---

From source file:com.citic.zxyjs.zwlscx.mapreduce.lib.input.HFileOutputFormatBase.java

License:Apache License

/**
 * Serialize column family to bloom type map to configuration. Invoked while
 * configuring the MR job for incremental load.
 * /*from  w w w.  jav a 2  s. c o m*/
 * @throws IOException
 *             on failure to read column family descriptors
 */
static void configureBloomType(HTable table, Configuration conf) throws IOException {
    HTableDescriptor tableDescriptor = table.getTableDescriptor();
    if (tableDescriptor == null) {
        // could happen with mock table instance
        return;
    }
    StringBuilder bloomTypeConfigValue = new StringBuilder();
    Collection<HColumnDescriptor> families = tableDescriptor.getFamilies();
    int i = 0;
    for (HColumnDescriptor familyDescriptor : families) {
        if (i++ > 0) {
            bloomTypeConfigValue.append('&');
        }
        bloomTypeConfigValue.append(URLEncoder.encode(familyDescriptor.getNameAsString(), "UTF-8"));
        bloomTypeConfigValue.append('=');
        String bloomType = familyDescriptor.getBloomFilterType().toString();
        if (bloomType == null) {
            bloomType = HColumnDescriptor.DEFAULT_BLOOMFILTER;
        }
        bloomTypeConfigValue.append(URLEncoder.encode(bloomType, "UTF-8"));
    }
    conf.set(BLOOM_TYPE_CONF_KEY, bloomTypeConfigValue.toString());
}

---

From source file:com.ckelsel.hadoop.MaxTemperature.App.java

License:Open Source License

public static void main(String[] args) {
    if (args.length != 2) {
        System.err.println("Usage: MaxTemperature <input path> <output path>");
        System.exit(-1);/*  w  w  w  . ja  v  a  2 s .co m*/
    }

    System.out.println(args[0]);
    System.out.println(args[1]);

    try {
        Configuration conf = new Configuration();
        conf.set("mapred.job.tracker", "localhost:9001");

        Job job = Job.getInstance(conf);

        job.setJarByClass(App.class);
        job.setJobName("Max temperature");

        FileInputFormat.addInputPath(job, new Path(args[0]));

        // delete output if exists
        Path outPath = new Path(args[1]);
        outPath.getFileSystem(conf).delete(outPath, true);

        FileOutputFormat.setOutputPath(job, outPath);

        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);

        System.exit(job.waitForCompletion(true) ? 0 : -1);
    } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    } catch (ClassNotFoundException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    } catch (InterruptedException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    }
}

---

From source file:com.cloudera.castagna.logparser.mr.StatusCodesStats.java

License:Apache License

@Override
public int run(String[] args) throws Exception {
    if (args.length != 2) {
        System.err.printf("Usage: %s [generic options] <input> <output>\n", getClass().getName());
        ToolRunner.printGenericCommandUsage(System.err);
        return -1;
    }//from  w ww  .j  a va 2 s .c o  m

    Configuration configuration = getConf();
    boolean useCompression = configuration.getBoolean(Constants.OPTION_USE_COMPRESSION,
            Constants.OPTION_USE_COMPRESSION_DEFAULT);

    if (useCompression) {
        configuration.setBoolean("mapred.compress.map.output", true);
        configuration.set("mapred.output.compression.type", "BLOCK");
        configuration.set("mapred.map.output.compression.codec", "org.apache.hadoop.io.compress.GzipCodec");
    }

    boolean overrideOutput = configuration.getBoolean(Constants.OPTION_OVERWRITE_OUTPUT,
            Constants.OPTION_OVERWRITE_OUTPUT_DEFAULT);
    FileSystem fs = FileSystem.get(new Path(args[1]).toUri(), configuration);
    if (overrideOutput) {
        fs.delete(new Path(args[1]), true);
    }

    Job job = Job.getInstance(configuration);
    job.setJobName(Constants.STATUS_CODES_STATS);
    job.setJarByClass(getClass());

    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));

    job.setInputFormatClass(TextInputFormat.class);

    job.setMapperClass(StatusCodesStatsMapper.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(Text.class);

    job.setCombinerClass(StatusCodesStatsCombiner.class);

    job.setReducerClass(StatusCodesStatsReducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    Utils.setReducers(job, configuration, log);

    job.setOutputFormatClass(TextOutputFormat.class);

    if (log.isDebugEnabled())
        Utils.log(job, log);

    return job.waitForCompletion(true) ? 0 : 1;
}

---

From source file:com.cloudera.cdk.data.hbase.impl.Loader.java

License:Apache License

@Override
public void load() {
    Accessor.getDefault().registerDatasetRepository(new URIPattern(URI.create("hbase:*zk")),
            new OptionBuilder<DatasetRepository>() {
                @Override//  ww w  .jav a  2 s  .com
                public DatasetRepository getFromOptions(Map<String, String> options) {
                    Configuration conf = HBaseConfiguration.create();
                    String[] hostsAndPort = parseHostsAndPort(options.get("zk"));
                    conf.set(HConstants.ZOOKEEPER_QUORUM, hostsAndPort[0]);
                    String port = hostsAndPort[1];
                    if (port != null) {
                        conf.set(HConstants.ZOOKEEPER_CLIENT_PORT, port);
                    }
                    return new HBaseDatasetRepository.Builder().configuration(conf).build();
                }
            });
}

---

From source file:com.cloudera.cdk.data.hcatalog.impl.Loader.java

License:Apache License

/**
 * Sets the MetaStore URI in the given Configuration, if there is a host in
 * the match arguments. If there is no host, then the conf is not changed.
 *
 * @param conf a Configuration that will be used to connect to the MetaStore
 * @param match URIPattern match results
 *//*w ww. j a  v  a2s.  c  o m*/
private static void setMetaStoreURI(Configuration conf, Map<String, String> match) {
    try {
        int port = UNSPECIFIED_PORT;
        if (match.containsKey("port")) {
            try {
                port = Integer.parseInt(match.get("port"));
            } catch (NumberFormatException e) {
                port = UNSPECIFIED_PORT;
            }
        }
        // if either the host or the port is set, construct a new MetaStore URI
        // and set the property in the Configuration. otherwise, this will not
        // change the connection URI.
        if (match.containsKey("host")) {
            conf.set(HIVE_METASTORE_URI_PROP,
                    new URI("thrift", null, match.get("host"), port, "/", null, null).toString());
        }
    } catch (URISyntaxException ex) {
        throw new DatasetRepositoryException("Could not build metastore URI", ex);
    }
}

---

From source file:com.cloudera.cdk.maven.plugins.AbstractDatasetMojo.java

License:Apache License

private Configuration getConf() {
    Configuration conf = new Configuration(false);
    for (String key : hadoopConfiguration.stringPropertyNames()) {
        String value = hadoopConfiguration.getProperty(key);
        conf.set(key, value);
    }//from   w  ww. j av  a 2 s.  c o  m
    return conf;
}

---

From source file:com.cloudera.cdk.morphline.hadoop.rcfile.ReadRCFileTest.java

License:Apache License

private void createRCFile(final String fileName, final int numRecords, final int maxColumns)
        throws IOException {
    // Write the sequence file
    SequenceFile.Metadata metadata = getMetadataForRCFile();
    Configuration conf = new Configuration();
    conf.set(RCFile.COLUMN_NUMBER_CONF_STR, String.valueOf(maxColumns));
    Path inputFile = dfs.makeQualified(new Path(testDirectory, fileName));
    RCFile.Writer rcFileWriter = new RCFile.Writer(dfs, conf, inputFile, null, metadata, null);
    for (int row = 0; row < numRecords; row++) {
        BytesRefArrayWritable dataWrite = new BytesRefArrayWritable(maxColumns);
        dataWrite.resetValid(maxColumns);
        for (int column = 0; column < maxColumns; column++) {
            Text sampleText = new Text("ROW-NUM:" + row + ", COLUMN-NUM:" + column);
            ByteArrayDataOutput dataOutput = ByteStreams.newDataOutput();
            sampleText.write(dataOutput);
            dataWrite.set(column, new BytesRefWritable(dataOutput.toByteArray()));
        }//from   ww  w.  ja  v a  2 s .c  om
        rcFileWriter.append(dataWrite);
    }
    rcFileWriter.close();
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »