Example usage for org.apache.hadoop.mapred JobConf getBoolean

List of usage examples for org.apache.hadoop.mapred JobConf getBoolean

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.mapred.*
  5. JobConf
  6. getBoolean

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred JobConf getBoolean.

Prototype

public boolean getBoolean(String name, boolean defaultValue)

Source Link

Document

Get the value of the name property as a boolean.

Usage

From source file:com.intel.hadoop.graphbuilder.preprocess.mapreduce.CreateGraphReducer.java

License:Open Source License

@Override
public void configure(JobConf job) {
    super.configure(job);
    this.valClass = job.getMapOutputValueClass();
    this.noBidir = job.getBoolean("noBidir", false);
    try {/*from  w ww. j  a  v  a 2 s. c o m*/
        if (job.get("EdgeFunc") != null) {
            this.EdgeFunc = (Functional) Class.forName(job.get("EdgeFunc")).newInstance();
            this.EdgeFunc.configure(job);
        }
        if (job.get("VertexFunc") != null) {
            this.VertexFunc = (Functional) Class.forName(job.get("VertexFunc")).newInstance();
            this.VertexFunc.configure(job);
        }
    } catch (InstantiationException e) {
        e.printStackTrace();
    } catch (IllegalAccessException e) {
        e.printStackTrace();
    } catch (ClassNotFoundException e) {
        e.printStackTrace();
    } catch (Exception e) {
        e.printStackTrace();
    }
}

From source file:com.intel.hadoop.graphbuilder.preprocess.mapreduce.EdgeTransformMapper.java

License:Open Source License

@Override
public void configure(JobConf job) {
    super.configure(job);
    this.reduceEndPoint = job.getBoolean("reduceEndPoint", EdgeTransformMR.SOURCE);
    try {/*from  ww w .j a  va  2 s. c  om*/
        this.graphparser = (GraphParser) Class.forName(job.get("GraphParser")).newInstance();
        this.vidparser = (FieldParser) Class.forName(job.get("VidParser")).newInstance();
        this.edataparser = (FieldParser) Class.forName(job.get("EdataParser")).newInstance();
        this.valClass = job.getMapOutputValueClass();
        val = (PairListType) valClass.newInstance();
    } catch (InstantiationException e) {
        e.printStackTrace();
    } catch (IllegalAccessException e) {
        e.printStackTrace();
    } catch (ClassNotFoundException e) {
        e.printStackTrace();
    }
}

From source file:com.intel.hadoop.graphbuilder.preprocess.mapreduce.EdgeTransformReducer.java

License:Open Source License

@Override
public void configure(JobConf job) {
    super.configure(job);
    this.reduceEndPoint = job.getBoolean("reduceEndPoint", EdgeTransformMR.SOURCE);
    try {/*from   w w w . j a  v a 2 s  . c om*/
        this.reduceFunc = (Functional) Class.forName(job.get("ReduceFunc")).newInstance();
        this.applyFunc = (Functional) Class.forName(job.get("ApplyFunc")).newInstance();
        this.reduceFunc.configure(job);
        this.applyFunc.configure(job);
    } catch (InstantiationException e) {
        e.printStackTrace();
    } catch (IllegalAccessException e) {
        e.printStackTrace();
    } catch (ClassNotFoundException e) {
        e.printStackTrace();
    } catch (Exception e) {
        e.printStackTrace();
    }
}

From source file:com.ricemap.spateDB.mapred.GridOutputFormat.java

License:Apache License

@Override
public RecordWriter<IntWritable, S> getRecordWriter(FileSystem ignored, JobConf job, String name,
        Progressable progress) throws IOException {
    // Get grid info
    CellInfo[] cellsInfo = SpatialSite.getCells(job);
    boolean pack = job.getBoolean(SpatialSite.PACK_CELLS, false);
    boolean expand = job.getBoolean(SpatialSite.EXPAND_CELLS, false);
    GridRecordWriter<S> writer = new GridRecordWriter<S>(job, name, cellsInfo, pack, expand);
    return writer;
}

From source file:com.ricemap.spateDB.mapred.GridOutputFormat2.java

License:Apache License

@Override
public RecordWriter<NullWritable, S> getRecordWriter(FileSystem ignored, JobConf job, String name,
        Progressable progress) throws IOException {
    // Get grid info
    CellInfo[] cellsInfo = SpatialSite.getCells(job);
    boolean pack = job.getBoolean(SpatialSite.PACK_CELLS, false);
    boolean expand = job.getBoolean(SpatialSite.EXPAND_CELLS, false);
    GridRecordWriter2<S> writer = new GridRecordWriter2<S>(job, name, cellsInfo, pack, expand);
    return writer;
}

From source file:com.ricemap.spateDB.mapred.RTreeGridOutputFormat.java

License:Apache License

@Override
public RecordWriter<IntWritable, S> getRecordWriter(FileSystem ignored, JobConf job, String name,
        Progressable progress) throws IOException {
    // Get grid info
    CellInfo[] cellsInfo = SpatialSite.getCells(job);
    boolean pack = job.getBoolean(SpatialSite.PACK_CELLS, false);
    boolean expand = job.getBoolean(SpatialSite.EXPAND_CELLS, false);
    RTreeGridRecordWriter<S> writer = new RTreeGridRecordWriter<S>(job, name, cellsInfo, pack, expand);
    writer.setStockObject((S) SpatialSite.createStockShape(job));
    return writer;
}

From source file:com.taobao.data.hive.hook.automapjoin.AutoMapJoinPreJobHook.java

License:Apache License

@Override
public void run(SessionState session, QueryPlan queryPlan, JobConf job, Integer taskId) {
    if (!job.getBoolean(HiveConf.ConfVars.HIVECONVERTJOIN.varname, false)
            || null == job.get(AUTO_JOIN_STATICS_DIR) || null == queryPlan) {
        return;/* w  w w .j ava 2  s  . co  m*/
    }
    List<Task<? extends Serializable>> rootTasks = queryPlan.getRootTasks();
    String taskIdStr = "Stage-" + taskId.toString();
    dfsTaskTag(rootTasks, taskIdStr); // dfs, fill the member variable 'taskTag'

    StringBuilder sb = new StringBuilder();
    sb.append(queryPlan.getQueryId());
    sb.append('\u0001');
    sb.append(taskIdStr);
    sb.append('\u0001');
    switch (taskTag) {
    case Task.NO_TAG:
        return;
    case Task.BACKUP_COMMON_JOIN:
        sb.append("Backup Common Join");
        break;
    case Task.COMMON_JOIN:
        sb.append("Common Join");
        break;
    case Task.CONVERTED_LOCAL_MAPJOIN:
        sb.append("Converted Local Map Join");
        break;
    case Task.CONVERTED_MAPJOIN:
        sb.append("Converted Map Join");
        break;
    case Task.LOCAL_MAPJOIN:
        sb.append("Local Map Join");
        break;
    }

    String sessionId = session.getSessionId();
    File dir = new File(job.get(AUTO_JOIN_STATICS_DIR));
    if (!dir.exists()) {
        dir.mkdir();
    }
    File file = new File(dir, sessionId + ".stat");
    synchronized (lock) {
        FileOutputStream fout = null;
        PrintStream pout = null;
        try {
            if (!file.exists()) {
                file.createNewFile();
            }
            fout = new FileOutputStream(file, true);
            pout = new PrintStream(fout);
            pout.println(sb.toString());
        } catch (FileNotFoundException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            if (null != pout) {
                pout.close();
            }
        }
    }

    // reset dfs results
    taskTag = Task.NO_TAG;
    found = false;
}

From source file:com.TCG.Nutch_DNS.Generator.java

License:Apache License

public Path[] generate(Path dbDir, Path segments, int numLists, long topN, long curTime) throws IOException {

    JobConf job = new NutchJob(getConf());
    boolean filter = job.getBoolean(GENERATOR_FILTER, true);
    boolean normalise = job.getBoolean(GENERATOR_NORMALISE, true);
    return generate(dbDir, segments, numLists, topN, curTime, filter, normalise, false, 1);
}

From source file:com.TCG.Nutch_DNS.HostDb.java

License:Apache License

public void update(Path crawlDb, Path[] segments, boolean normalize, boolean filter, boolean additionsAllowed,
        boolean force) throws IOException {
    FileSystem fs = FileSystem.get(getConf());
    Path lock = new Path(crawlDb, LOCK_NAME);
    LockUtil.createLockFile(fs, lock, force);
    SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
    long start = System.currentTimeMillis();

    JobConf job = HostDb.createJob(getConf(), crawlDb);
    job.setBoolean(CRAWLDB_ADDITIONS_ALLOWED, additionsAllowed);
    job.setBoolean(HostDbFilter.URL_FILTERING, filter);
    job.setBoolean(HostDbFilter.URL_NORMALIZING, normalize);

    boolean url404Purging = job.getBoolean(CRAWLDB_PURGE_404, false);

    if (LOG.isInfoEnabled()) {
        LOG.info("CrawlDb update: starting at " + sdf.format(start));
        LOG.info("CrawlDb update: db: " + crawlDb);
        LOG.info("CrawlDb update: segments: " + Arrays.asList(segments));
        LOG.info("CrawlDb update: additions allowed: " + additionsAllowed);
        LOG.info("CrawlDb update: URL normalizing: " + normalize);
        LOG.info("CrawlDb update: URL filtering: " + filter);
        LOG.info("CrawlDb update: 404 purging: " + url404Purging);
    }/*from  w ww .j  av  a2s . c o m*/

    for (int i = 0; i < segments.length; i++) {
        Path fetch = new Path(segments[i], CrawlDatum.FETCH_DIR_NAME);
        Path parse = new Path(segments[i], CrawlDatum.PARSE_DIR_NAME);
        if (fs.exists(fetch) && fs.exists(parse)) {
            FileInputFormat.addInputPath(job, fetch);
            FileInputFormat.addInputPath(job, parse);
        } else {
            LOG.info(" - skipping invalid segment " + segments[i]);
        }
    }

    if (LOG.isInfoEnabled()) {
        LOG.info("CrawlDb update: Merging segment data into db.");
    }
    try {
        JobClient.runJob(job);
    } catch (IOException e) {
        LockUtil.removeLockFile(fs, lock);
        Path outPath = FileOutputFormat.getOutputPath(job);
        if (fs.exists(outPath))
            fs.delete(outPath, true);
        throw e;
    }

    HostDb.install(job, crawlDb);
    long end = System.currentTimeMillis();
    LOG.info("CrawlDb update: finished at " + sdf.format(end) + ", elapsed: "
            + TimingUtil.elapsedTime(start, end));
}

From source file:com.TCG.Nutch_DNS.HostDb.java

License:Apache License

public static void install(JobConf job, Path crawlDb) throws IOException {
    boolean preserveBackup = job.getBoolean("db.preserve.backup", true);

    Path newCrawlDb = FileOutputFormat.getOutputPath(job);
    FileSystem fs = new JobClient(job).getFs();
    Path old = new Path(crawlDb, "old");
    Path current = new Path(crawlDb, CURRENT_NAME);
    if (fs.exists(current)) {
        if (fs.exists(old))
            fs.delete(old, true);// www  .  ja v  a 2s  . c o  m
        fs.rename(current, old);
    }
    fs.mkdirs(crawlDb);
    fs.rename(newCrawlDb, current);
    if (!preserveBackup && fs.exists(old))
        fs.delete(old, true);
    Path lock = new Path(crawlDb, LOCK_NAME);
    LockUtil.removeLockFile(fs, lock);
}