List of usage examples for org.apache.hadoop.mapreduce JobContext getCredentials
public Credentials getCredentials();
From source file:co.cask.cdap.internal.app.runtime.batch.dataset.partitioned.DynamicPartitioningOutputFormat.java
License:Apache License
@Override public void checkOutputSpecs(JobContext job) throws IOException { // Ensure that the output directory is set and not already there Path outDir = getOutputPath(job); if (outDir == null) { throw new InvalidJobConfException("Output directory not set."); }//from w w w .ja va 2s. c o m // get delegation token for outDir's file system TokenCache.obtainTokensForNamenodes(job.getCredentials(), new Path[] { outDir }, job.getConfiguration()); // we permit multiple jobs writing to the same output directory. We handle this by each one writing to distinct // paths within that directory. See createJobSpecificPath method and usages of it. // additionally check that output dataset and dynamic partitioner class name has been set in conf if (job.getConfiguration().get(Constants.Dataset.Partitioned.HCONF_ATTR_OUTPUT_DATASET) == null) { throw new InvalidJobConfException("The job configuration does not contain required property: " + Constants.Dataset.Partitioned.HCONF_ATTR_OUTPUT_DATASET); } Class<? extends DynamicPartitioner> partitionerClass = job.getConfiguration().getClass( PartitionedFileSetArguments.DYNAMIC_PARTITIONER_CLASS_NAME, null, DynamicPartitioner.class); if (partitionerClass == null) { throw new InvalidJobConfException("The job configuration does not contain required property: " + PartitionedFileSetArguments.DYNAMIC_PARTITIONER_CLASS_NAME); } Class<? extends FileOutputFormat> delegateOutputFormatClass = job.getConfiguration().getClass( Constants.Dataset.Partitioned.HCONF_ATTR_OUTPUT_FORMAT_CLASS_NAME, null, FileOutputFormat.class); if (delegateOutputFormatClass == null) { throw new InvalidJobConfException("The job configuration does not contain required property: " + Constants.Dataset.Partitioned.HCONF_ATTR_OUTPUT_FORMAT_CLASS_NAME); } }
From source file:com.asakusafw.runtime.stage.output.TemporaryOutputFormat.java
License:Apache License
@Override public void checkOutputSpecs(JobContext context) throws IOException, InterruptedException { if (context == null) { throw new IllegalArgumentException("context must not be null"); //$NON-NLS-1$ }/* ww w. java 2 s. c o m*/ Path path = getOutputPath(context); if (TemporaryOutputFormat.getOutputPath(context) == null) { throw new IOException("Temporary output path is not set"); } TokenCache.obtainTokensForNamenodes(context.getCredentials(), new Path[] { path }, context.getConfiguration()); if (path.getFileSystem(context.getConfiguration()).exists(path)) { throw new IOException(MessageFormat.format("Output directory {0} already exists", path)); } }
From source file:com.bonc.mr_roamRecognition_hjpt.comm.NewFileOutputFormat.java
License:Apache License
public void checkOutputSpecs(JobContext job) throws FileAlreadyExistsException, IOException { // Ensure that the output directory is set and not already there Path outDir = getOutputPath(job); if (outDir == null) { throw new InvalidJobConfException("Output directory not set."); }//from w w w. j av a 2 s . c o m // get delegation token for outDir's file system TokenCache.obtainTokensForNamenodes(job.getCredentials(), new Path[] { outDir }, job.getConfiguration()); // if (outDir.getFileSystem(job.getConfiguration()).exists(outDir)) { // throw new FileAlreadyExistsException("Output directory " + outDir + " already exists"); // } }
From source file:com.cloudera.recordservice.examples.terasort.RecordServiceTeraInputFormat.java
License:Apache License
@Override public List<InputSplit> getSplits(JobContext context) throws IOException, InterruptedException { PlanUtil.SplitsInfo info = PlanUtil.getSplits(context.getConfiguration(), context.getCredentials()); if (info.schema.getNumColumns() != 1 || info.schema.getColumnInfo(0).type.typeId != Schema.Type.STRING) { throw new IOException("Invalid data. Expecting schema to be a single STRING."); }//from www.j ava 2 s . c om return info.splits; }
From source file:com.cloudera.recordservice.examples.terasort.TeraOutputFormat.java
License:Apache License
@Override public void checkOutputSpecs(JobContext job) throws InvalidJobConfException, IOException { // Ensure that the output directory is set Path outDir = getOutputPath(job); if (outDir == null) { throw new InvalidJobConfException("Output directory not set in JobConf."); }/*from w ww . j av a2s .c o m*/ // get delegation token for outDir's file system TokenCache.obtainTokensForNamenodes(job.getCredentials(), new Path[] { outDir }, job.getConfiguration()); }
From source file:com.cloudera.recordservice.hcatalog.mapreduce.HCatRSBaseInputFormat.java
License:Apache License
/** * Logically split the set of input files for the job. Returns the * underlying InputFormat's splits//from ww w .j a va 2s.co m * @param jobContext the job context object * @return the splits, a RecordServiceSplit wrapper over the storage * handler InputSplits * @throws IOException or InterruptedException */ @Override public List<InputSplit> getSplits(JobContext jobContext) throws IOException, InterruptedException { Configuration conf = jobContext.getConfiguration(); // Get the job info from the configuration, // throws exception if not initialized InputJobInfo inputJobInfo; try { inputJobInfo = getJobInfo(conf); } catch (Exception e) { throw new IOException(e); } List<InputSplit> splits = new ArrayList<InputSplit>(); List<PartInfo> partitionInfoList = inputJobInfo.getPartitions(); if (partitionInfoList == null) { // No partitions match the specified partition filter return splits; } JobConf jobConf = HCatUtil.getJobConfFromContext(jobContext); Credentials credentials = jobContext.getCredentials(); PlanUtil.SplitsInfo splitsInfo = PlanUtil.getSplits(jobConf, credentials); return splitsInfo.splits; }
From source file:com.cloudera.recordservice.mapreduce.RecordServiceInputFormatBase.java
License:Apache License
@Override public List<InputSplit> getSplits(JobContext context) throws IOException, InterruptedException { return PlanUtil.getSplits(context.getConfiguration(), context.getCredentials()).splits; }
From source file:com.cloudera.recordservice.mapreduce.TextInputFormat.java
License:Apache License
@Override public List<InputSplit> getSplits(JobContext context) throws IOException, InterruptedException { PlanUtil.SplitsInfo splits = PlanUtil.getSplits(context.getConfiguration(), context.getCredentials()); verifyTextSchema(splits.schema);//from w ww . java 2s .c om return splits.splits; }
From source file:com.couchbase.sqoop.mapreduce.db.CouchbaseInputFormat.java
License:Apache License
@Override public List<InputSplit> getSplits(final JobContext job) throws IOException, InterruptedException { final List<URI> baseUris = new LinkedList<URI>(); baseUris.add(URI.create(dbConf.getUrlProperty())); // Tell things using Spy's logging to use log4j compat, hadoop uses log4j final Properties spyLogProperties = System.getProperties(); spyLogProperties.put("net.spy.log.LoggerImpl", "net.spy.memcached.compat.log.Log4JLogger"); System.setProperties(spyLogProperties); final String passwordSecurely = new String( job.getCredentials().getSecretKey(CouchbaseInputFormat.PASSWORD_SECRET_KEY), Charset.forName("UTF-8")); dbConf.setPassword(passwordSecurely); System.out.printf("User name is %s and Password from secure store is %s", dbConf.getUsername(), passwordSecurely);/* w ww. java 2 s . c o m*/ // final CouchbaseConnectionFactoryBuilder cfb = new CouchbaseConnectionFactoryBuilder(); // cfb.setOpTimeout(10000); // wait up to 10 seconds for an op to succeed // cfb.setOpQueueMaxBlockTime(60000); // wait up to 1 minute to enqueue // // final CouchbaseClient client = new CouchbaseClient(cfb.buildCouchbaseConnection(baseUris, // dbConf.getUsername(), // passwordSecurely)); final AuthDescriptor authDescriptor = new AuthDescriptor(new String[] { "PLAIN" }, new PlainCallbackHandler(dbConf.getUsername(), passwordSecurely)); CouchbaseInputFormat.CONN_BUILDER_FACTORY = new CouchbaseConnectionFactoryBuilder(); CouchbaseInputFormat.CONN_BUILDER_FACTORY.setOpTimeout(20000); CouchbaseInputFormat.CONN_BUILDER_FACTORY.setShouldOptimize(true); CouchbaseInputFormat.CONN_BUILDER_FACTORY.setAuthDescriptor(authDescriptor); CouchbaseInputFormat.CONN_BUILDER_FACTORY.setProtocol(ConnectionFactoryBuilder.Protocol.BINARY); CouchbaseInputFormat.CONN_BUILDER_FACTORY.setMaxReconnectDelay(10000); CouchbaseInputFormat.CONN_BUILDER_FACTORY.setOpQueueMaxBlockTime(10000); CouchbaseInputFormat.CONN_FACTORY = CouchbaseInputFormat.CONN_BUILDER_FACTORY .buildCouchbaseConnection(baseUris, dbConf.getUsername(), "", passwordSecurely); final CouchbaseClient client = new CouchbaseClient(CouchbaseInputFormat.CONN_FACTORY); final int numVBuckets = client.getNumVBuckets(); System.out.printf("No of v Buckets is %d", numVBuckets); client.shutdown(); final int chunks = ConfigurationHelper.getJobNumMaps(job); final int itemsPerChunk = numVBuckets / chunks; int extraItems = numVBuckets % chunks; final List<InputSplit> splits = new ArrayList<InputSplit>(); int splitIndex = 0; short[] curSplit = nextEmptySplit(itemsPerChunk, extraItems); extraItems--; for (short i = 0; i < numVBuckets + 1; i++) { if (splitIndex == curSplit.length) { final CouchbaseInputSplit split = new CouchbaseInputSplit(curSplit, job); splits.add(split); curSplit = nextEmptySplit(itemsPerChunk, extraItems); extraItems--; splitIndex = 0; } curSplit[splitIndex] = i; splitIndex++; } return splits; }
From source file:com.david.mos.out.FileOutputFormat.java
License:Apache License
public void checkOutputSpecs(JobContext job) throws FileAlreadyExistsException, IOException { // Ensure that the output directory is set and not already there Path outDir = getOutputPath(job); if (outDir == null) { throw new InvalidJobConfException("Output directory not set."); }/* w w w.j a va2 s. co m*/ // get delegation token for outDir's file system TokenCache.obtainTokensForNamenodes(job.getCredentials(), new Path[] { outDir }, job.getConfiguration()); if (outDir.getFileSystem(job.getConfiguration()).exists(outDir)) { throw new FileAlreadyExistsException("Output directory " + outDir + " already exists"); } }