Example usage for org.apache.hadoop.mapreduce JobContext getConfiguration

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce JobContext getConfiguration.

Prototype

public Configuration getConfiguration();

Source Link

Document

Return the configuration for the job.

Usage

From source file:ml.shifu.shifu.guagua.ShifuInputFormat.java

License:Apache License

public static Path[] getInputPaths(JobContext context) {
    String dirs = context.getConfiguration().get(CommonConstants.CROSS_VALIDATION_DIR, "");
    LOG.info("crossValidation_dir:" + dirs);
    String[] list = StringUtils.split(dirs);
    Path[] result = new Path[list.length];
    for (int i = 0; i < list.length; i++) {
        result[i] = new Path(StringUtils.unEscapeString(list[i]));
    }/*from  w  ww. j  a va  2 s .  c om*/
    return result;
}

From source file:mvm.rya.accumulo.mr.RyaOutputFormat.java

License:Apache License

@Override
public void checkOutputSpecs(JobContext jobContext) throws IOException, InterruptedException {
    Configuration conf = jobContext.getConfiguration();

    // make sure that all of the indexers can connect
    getGeoIndexer(conf);//from   ww  w  . j a  v a 2  s . c  o m
    getFreeTextIndexer(conf);
    getTemporalIndexer(conf);
    getRyaIndexer(conf);
}

From source file:mvm.rya.accumulo.mr.utils.AccumuloHDFSFileInputFormat.java

License:Apache License

@Override
public List<InputSplit> getSplits(JobContext jobContext) throws IOException {
    //read the params from AccumuloInputFormat
    Configuration conf = jobContext.getConfiguration();
    Instance instance = AccumuloProps.getInstance(jobContext);
    String user = AccumuloProps.getUsername(jobContext);
    AuthenticationToken password = AccumuloProps.getPassword(jobContext);
    String table = AccumuloProps.getTablename(jobContext);
    ArgumentChecker.notNull(instance);/*  w  w  w  .  j a  va2 s  .  co m*/
    ArgumentChecker.notNull(table);

    //find the files necessary
    try {
        AccumuloConfiguration acconf = instance.getConfiguration();
        FileSystem fs = FileSystem.get(conf);
        Connector connector = instance.getConnector(user, password);
        TableOperations tos = connector.tableOperations();
        String tableId = tos.tableIdMap().get(table);
        String filePrefix = acconf.get(Property.INSTANCE_DFS_DIR) + "/tables/" + tableId;
        System.out.println(filePrefix);

        Scanner scanner = connector.createScanner("!METADATA", Constants.NO_AUTHS); //TODO: auths?
        scanner.setRange(new Range(new Text(tableId + "\u0000"), new Text(tableId + "\uFFFD")));
        scanner.fetchColumnFamily(new Text("file"));
        List<String> files = new ArrayList<String>();
        List<InputSplit> fileSplits = new ArrayList<InputSplit>();
        Job job = new Job(conf);
        for (Map.Entry<Key, Value> entry : scanner) {
            String file = filePrefix + entry.getKey().getColumnQualifier().toString();
            files.add(file);
            Path path = new Path(file);
            FileStatus fileStatus = fs.getFileStatus(path);
            long len = fileStatus.getLen();
            BlockLocation[] fileBlockLocations = fs.getFileBlockLocations(fileStatus, 0, len);
            fileSplits.add(new FileSplit(path, 0, len, fileBlockLocations[0].getHosts()));
            //                FileInputFormat.addInputPath(job, path);
        }
        System.out.println(files);
        return fileSplits;
        //            return super.getSplits(job);
    } catch (Exception e) {
        throw new IOException(e);
    }
}

From source file:mvm.rya.indexing.accumulo.ConfigUtils.java

License:Apache License

public static String getUsername(JobContext job) {
    return getUsername(job.getConfiguration());
}

From source file:mvm.rya.indexing.accumulo.ConfigUtils.java

License:Apache License

public static Authorizations getAuthorizations(JobContext job) {
    return getAuthorizations(job.getConfiguration());
}

From source file:mvm.rya.indexing.accumulo.ConfigUtils.java

License:Apache License

public static Instance getInstance(JobContext job) {
    return getInstance(job.getConfiguration());
}

From source file:mvm.rya.indexing.accumulo.ConfigUtils.java

License:Apache License

public static String getPassword(JobContext job) {
    return getPassword(job.getConfiguration());
}

From source file:mvm.rya.indexing.accumulo.ConfigUtils.java

License:Apache License

public static Connector getConnector(JobContext job) throws AccumuloException, AccumuloSecurityException {
    return getConnector(job.getConfiguration());
}

From source file:net.jarcec.sqoop.data.gen.mr.GeneratorInputFormat.java

License:Apache License

@Override
public List<InputSplit> getSplits(JobContext jobContext) throws IOException, InterruptedException {
    Configuration configuration = jobContext.getConfiguration();

    long files = configuration.getLong(Constants.FILES_COUNT, 0);
    long records = configuration.getLong(Constants.RECORD_COUNT, 0);

    List<InputSplit> splits = new LinkedList<InputSplit>();

    long next = 1;
    for (int i = 0; i < files; i++) {
        splits.add(new GeneratorSplit(next, next + records));
        next += records;//w  w  w. j a  va2 s .c  om
    }

    return splits;
}

From source file:net.mooncloud.mapreduce.lib.db.DBInputFormat.java

License:Apache License

/** {@inheritDoc} */
public List<InputSplit> getSplits(JobContext job) throws IOException {

    ResultSet results = null;/*from w  ww . j  a v a 2  s. c o  m*/
    Statement statement = null;
    try {
        statement = connection.createStatement();

        results = statement.executeQuery(getCountQuery());
        results.next();

        long count = results.getLong(1);
        int chunks = job.getConfiguration().getInt("mapred.map.tasks", 1);
        long chunkSize = (count / chunks);

        results.close();
        statement.close();

        List<InputSplit> splits = new ArrayList<InputSplit>();

        // Split the rows into n-number of chunks and adjust the last chunk
        // accordingly
        for (int i = 0; i < chunks; i++) {
            DBInputSplit split;

            if ((i + 1) == chunks)
                split = new DBInputSplit(i * chunkSize, count);
            else
                split = new DBInputSplit(i * chunkSize, (i * chunkSize) + chunkSize);

            splits.add(split);
        }

        connection.commit();
        return splits;
    } catch (SQLException e) {
        throw new IOException("Got SQLException", e);
    } finally {
        try {
            if (results != null) {
                results.close();
            }
        } catch (SQLException e1) {
        }
        try {
            if (statement != null) {
                statement.close();
            }
        } catch (SQLException e1) {
        }

        closeConnection();
    }
}