List of usage examples for org.apache.hadoop.conf Configuration getEnum
public <T extends Enum<T>> T getEnum(String name, T defaultValue)
From source file:com.datatorrent.demos.ads.Application.java
License:Open Source License
private void configure(DAG dag, Configuration conf) { if (StreamingApplication.Environment.CLUSTER == conf.getEnum(StreamingApplication.ENVIRONMENT, StreamingApplication.Environment.LOCAL)) { setLocalMode();/*w w w . j a va 2 s .c o m*/ // settings only affect distributed mode AttributeMap attributes = dag.getAttributes(); if (attributes.get(DAGContext.CONTAINER_MEMORY_MB) == null) { attributes.put(DAGContext.CONTAINER_MEMORY_MB, 2048); } if (attributes.get(DAGContext.MASTER_MEMORY_MB) == null) { attributes.put(DAGContext.MASTER_MEMORY_MB, 1024); } if (attributes.get(DAGContext.CONTAINERS_MAX_COUNT) == null) { attributes.put(DAGContext.CONTAINERS_MAX_COUNT, 1); } } else if (StreamingApplication.Environment.LOCAL == conf.getEnum(StreamingApplication.ENVIRONMENT, StreamingApplication.Environment.CLUSTER)) { setLocalMode(); } this.generatorVTuplesBlast = conf.getInt(P_generatorVTuplesBlast, this.generatorVTuplesBlast); this.generatorMaxWindowsCount = conf.getInt(P_generatorMaxWindowsCount, this.generatorMaxWindowsCount); this.locality = conf.getBoolean(P_allInline, false) ? Locality.CONTAINER_LOCAL : null; this.numGenerators = conf.getInt(P_numGenerators, this.numGenerators); }
From source file:com.yahoo.glimmer.indexing.RDFDocumentFactory.java
License:Open Source License
public static IndexType getIndexType(Configuration conf) { IndexType indexType = conf.getEnum(CONF_INDEX_TYPE_KEY, IndexType.UNDEFINED); if (indexType == IndexType.UNDEFINED) { throw new IllegalStateException("Index type not set in config."); }/*from w w w .j a va2 s .c om*/ return indexType; }
From source file:org.apache.crunch.types.avro.AvroMode.java
License:Apache License
public static AvroMode fromConfiguration(Configuration conf) { AvroMode mode = conf.getEnum(AVRO_MODE_PROPERTY, REFLECT); mode.setFromConfiguration(conf);// w ww.ja v a 2s .c o m return mode; }
From source file:org.apache.crunch.types.avro.AvroMode.java
License:Apache License
public static AvroMode fromShuffleConfiguration(Configuration conf) { AvroMode mode = conf.getEnum(AVRO_SHUFFLE_MODE_PROPERTY, REFLECT); mode.setFromConfiguration(conf);/*from ww w.java2s . co m*/ return mode; }
From source file:org.apache.nutch.mapreduce.FetchJob.java
License:Apache License
@Override protected void setup(Map<String, Object> args) throws Exception { super.setup(args); Params params = new Params(args); Configuration conf = getConf(); checkConfiguration(conf);//from ww w .j av a 2 s. c o m String crawlId = params.get(ARG_CRAWL, conf.get(Nutch.PARAM_CRAWL_ID)); FetchMode fetchMode = params.getEnum(ARG_FETCH_MODE, conf.getEnum(PARAM_FETCH_MODE, FetchMode.NATIVE)); batchId = params.get(ARG_BATCH, ALL_BATCH_ID_STR); int threads = params.getInt(ARG_THREADS, 5); boolean resume = params.getBoolean(ARG_RESUME, false); int limit = params.getInt(ARG_LIMIT, -1); numTasks = params.getInt(ARG_NUMTASKS, conf.getInt(PARAM_MAPREDUCE_JOB_REDUCES, 2)); boolean index = params.getBoolean(ARG_INDEX, false); /** Solr */ String solrUrl = params.get(ARG_SOLR_URL, conf.get(PARAM_SOLR_SERVER_URL)); String zkHostString = params.get(ARG_ZK, conf.get(PARAM_SOLR_ZK)); String solrCollection = params.get(ARG_COLLECTION, conf.get(PARAM_SOLR_COLLECTION)); /** Set re-computed config variables */ NutchConfiguration.setIfNotNull(conf, PARAM_CRAWL_ID, crawlId); conf.setEnum(PARAM_FETCH_MODE, fetchMode); NutchConfiguration.setIfNotNull(conf, PARAM_BATCH_ID, batchId); conf.setInt(PARAM_THREADS, threads); conf.setBoolean(PARAM_RESUME, resume); conf.setInt(PARAM_MAPPER_LIMIT, limit); conf.setInt(PARAM_MAPREDUCE_JOB_REDUCES, numTasks); conf.setBoolean(PARAM_INDEX_JUST_IN_TIME, index); NutchConfiguration.setIfNotNull(conf, PARAM_SOLR_SERVER_URL, solrUrl); NutchConfiguration.setIfNotNull(conf, PARAM_SOLR_ZK, zkHostString); NutchConfiguration.setIfNotNull(conf, PARAM_SOLR_COLLECTION, solrCollection); LOG.info(Params.format("className", this.getClass().getSimpleName(), "crawlId", crawlId, "batchId", batchId, "fetchMode", fetchMode, "numTasks", numTasks, "threads", threads, "resume", resume, "limit", limit, "index", index, "solrUrl", solrUrl, "zkHostString", zkHostString, "solrCollection", solrCollection)); }
From source file:org.apache.nutch.mapreduce.GenerateJob.java
License:Apache License
@Override protected void setup(Map<String, Object> args) throws Exception { super.setup(args); Params params = new Params(args); Configuration conf = getConf(); String crawlId = params.get(ARG_CRAWL, conf.get(PARAM_CRAWL_ID)); String batchId = params.get(ARG_BATCH, NutchUtil.generateBatchId()); boolean reGenerate = params.getBoolean(ARG_REGENERATE, false); long topN = params.getLong(ARG_TOPN, Long.MAX_VALUE); boolean filter = params.getBoolean(ARG_FILTER, true); boolean norm = params.getBoolean(ARG_NORMALIZE, true); long pseudoCurrTime = params.getLong(ARG_CURTIME, startTime); String nutchTmpDir = conf.get(PARAM_NUTCH_TMP_DIR, PATH_NUTCH_TMP_DIR); conf.set(PARAM_CRAWL_ID, crawlId);//w ww . j a va2 s .c o m conf.set(PARAM_BATCH_ID, batchId); conf.setLong(GENERATE_TIME_KEY, startTime); // seems not used, (or pseudoCurrTime used?) conf.setLong(PARAM_GENERATOR_CUR_TIME, pseudoCurrTime); conf.setBoolean(PARAM_GENERATE_REGENERATE, reGenerate); conf.setLong(PARAM_GENERATOR_TOP_N, topN); conf.setBoolean(PARAM_GENERATE_FILTER, filter); conf.setBoolean(PARAM_GENERATE_NORMALISE, norm); URLUtil.HostGroupMode hostGroupMode = conf.getEnum(PARAM_GENERATOR_COUNT_MODE, URLUtil.HostGroupMode.BY_HOST); conf.setEnum(PARTITION_MODE_KEY, hostGroupMode); LOG.info(Params.format("className", this.getClass().getSimpleName(), "crawlId", crawlId, "batchId", batchId, "filter", filter, "norm", norm, "pseudoCurrTime", DateTimeUtil.format(pseudoCurrTime), "topN", topN, "reGenerate", reGenerate, PARAM_GENERATOR_COUNT_MODE, hostGroupMode, PARTITION_MODE_KEY, hostGroupMode, "nutchTmpDir", nutchTmpDir)); Files.write(Paths.get(PATH_LAST_BATCH_ID), (batchId + "\n").getBytes(), StandardOpenOption.CREATE, StandardOpenOption.WRITE); }
From source file:org.apache.rya.indexing.OptionalConfigUtils.java
License:Apache License
/** * Retrieves the value for the geo indexer type from the config. * @param conf the {@link Configuration}. * @return the {@link GeoIndexerType} found in the config or * {@code null} if it doesn't exist./* w ww . java2 s. c om*/ */ public static GeoIndexerType getGeoIndexerType(final Configuration conf) { return conf.getEnum(GEO_INDEXER_TYPE, null); }
From source file:org.kitesdk.data.mapreduce.DatasetKeyOutputFormat.java
License:Apache License
@Override public void checkOutputSpecs(JobContext jobContext) { // The committer setup will fail if the output dataset does not exist View<E> target = load(jobContext); Configuration conf = Hadoop.JobContext.getConfiguration.invoke(jobContext); switch (conf.getEnum(KITE_WRITE_MODE, WriteMode.DEFAULT)) { case APPEND:/*from ww w .j ava 2 s. c o m*/ break; case OVERWRITE: // if the merge won't use replace, then delete the existing data if (!canReplace(target)) { target.deleteAll(); } break; default: case DEFAULT: boolean isReady = false; if (target instanceof Signalable) { isReady = ((Signalable) target).isReady(); } if (isReady || !target.isEmpty()) { throw new DatasetException("View is not empty or has been signaled as ready: " + target); } break; } }