List of usage examples for org.apache.hadoop.mapreduce Job isSuccessful
public boolean isSuccessful() throws IOException
From source file:org.apache.accumulo.test.continuous.ContinuousMoru.java
License:Apache License
@Override public int run(String[] args) throws IOException, InterruptedException, ClassNotFoundException, AccumuloSecurityException { Opts opts = new Opts(); BatchWriterOpts bwOpts = new BatchWriterOpts(); MapReduceClientOnDefaultTable clientOpts = new MapReduceClientOnDefaultTable("ci"); clientOpts.parseArgs(ContinuousMoru.class.getName(), args, bwOpts, opts); Job job = Job.getInstance(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis()); job.setJarByClass(this.getClass()); job.setInputFormatClass(AccumuloInputFormat.class); clientOpts.setAccumuloConfigs(job);/*from w w w .j a va 2 s . c o m*/ // set up ranges try { Set<Range> ranges = clientOpts.getConnector().tableOperations() .splitRangeByTablets(clientOpts.getTableName(), new Range(), opts.maxMaps); AccumuloInputFormat.setRanges(job, ranges); AccumuloInputFormat.setAutoAdjustRanges(job, false); } catch (Exception e) { throw new IOException(e); } job.setMapperClass(CMapper.class); job.setNumReduceTasks(0); job.setOutputFormatClass(AccumuloOutputFormat.class); AccumuloOutputFormat.setBatchWriterOptions(job, bwOpts.getBatchWriterConfig()); Configuration conf = job.getConfiguration(); conf.setLong(MIN, opts.min); conf.setLong(MAX, opts.max); conf.setInt(MAX_CF, opts.maxColF); conf.setInt(MAX_CQ, opts.maxColQ); conf.set(CI_ID, UUID.randomUUID().toString()); job.waitForCompletion(true); clientOpts.stopTracing(); return job.isSuccessful() ? 0 : 1; }
From source file:org.apache.accumulo.test.continuous.ContinuousVerify.java
License:Apache License
@Override public int run(String[] args) throws Exception { Opts opts = new Opts(); opts.parseArgs(this.getClass().getName(), args); Job job = Job.getInstance(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis()); job.setJarByClass(this.getClass()); job.setInputFormatClass(AccumuloInputFormat.class); opts.setAccumuloConfigs(job);/*from w w w .j a va2 s . co m*/ Set<Range> ranges = null; String clone = opts.getTableName(); Connector conn = null; if (opts.scanOffline) { Random random = new Random(); clone = opts.getTableName() + "_" + String.format("%016x", (random.nextLong() & 0x7fffffffffffffffl)); conn = opts.getConnector(); conn.tableOperations().clone(opts.getTableName(), clone, true, new HashMap<String, String>(), new HashSet<String>()); ranges = conn.tableOperations().splitRangeByTablets(opts.getTableName(), new Range(), opts.maxMaps); conn.tableOperations().offline(clone); AccumuloInputFormat.setInputTableName(job, clone); AccumuloInputFormat.setOfflineTableScan(job, true); } else { ranges = opts.getConnector().tableOperations().splitRangeByTablets(opts.getTableName(), new Range(), opts.maxMaps); } AccumuloInputFormat.setRanges(job, ranges); AccumuloInputFormat.setAutoAdjustRanges(job, false); job.setMapperClass(CMapper.class); job.setMapOutputKeyClass(LongWritable.class); job.setMapOutputValueClass(VLongWritable.class); job.setReducerClass(CReducer.class); job.setNumReduceTasks(opts.reducers); job.setOutputFormatClass(TextOutputFormat.class); job.getConfiguration().setBoolean("mapred.map.tasks.speculative.execution", opts.scanOffline); TextOutputFormat.setOutputPath(job, new Path(opts.outputDir)); job.waitForCompletion(true); if (opts.scanOffline) { conn.tableOperations().delete(clone); } opts.stopTracing(); return job.isSuccessful() ? 0 : 1; }
From source file:org.apache.accumulo.test.mapreduce.TeraSortIngest.java
License:Apache License
@Override public int run(String[] args) throws Exception { Job job = Job.getInstance(getConf()); job.setJobName("TeraSortCloud"); job.setJarByClass(this.getClass()); Opts opts = new Opts(); opts.parseArgs(TeraSortIngest.class.getName(), args); job.setInputFormatClass(RangeInputFormat.class); job.setMapperClass(SortGenMapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Mutation.class); job.setNumReduceTasks(0);/*from w w w . ja v a2 s .co m*/ job.setOutputFormatClass(AccumuloOutputFormat.class); opts.setAccumuloConfigs(job); BatchWriterConfig bwConfig = new BatchWriterConfig().setMaxMemory(10L * 1000 * 1000); AccumuloOutputFormat.setBatchWriterOptions(job, bwConfig); Configuration conf = job.getConfiguration(); conf.setLong(NUMROWS, opts.numRows); conf.setInt("cloudgen.minkeylength", opts.minKeyLength); conf.setInt("cloudgen.maxkeylength", opts.maxKeyLength); conf.setInt("cloudgen.minvaluelength", opts.minValueLength); conf.setInt("cloudgen.maxvaluelength", opts.maxValueLength); conf.set("cloudgen.tablename", opts.getTableName()); if (args.length > 10) conf.setInt(NUMSPLITS, opts.splits); job.waitForCompletion(true); return job.isSuccessful() ? 0 : 1; }
From source file:org.apache.accumulo.test.randomwalk.multitable.CopyTool.java
License:Apache License
@Override public int run(String[] args) throws Exception { Job job = Job.getInstance(getConf(), this.getClass().getSimpleName()); job.setJarByClass(this.getClass()); if (job.getJar() == null) { log.error("M/R requires a jar file! Run mvn package."); return 1; }/*from ww w . j a va 2s . co m*/ ClientConfiguration clientConf = new ClientConfiguration().withInstance(args[3]).withZkHosts(args[4]); job.setInputFormatClass(AccumuloInputFormat.class); AccumuloInputFormat.setInputTableName(job, args[2]); AccumuloInputFormat.setScanAuthorizations(job, Authorizations.EMPTY); AccumuloInputFormat.setZooKeeperInstance(job, clientConf); final String principal; final AuthenticationToken token; if (clientConf.getBoolean(ClientProperty.INSTANCE_RPC_SASL_ENABLED.getKey(), false)) { // Use the Kerberos creds to request a DelegationToken for MapReduce to use // We could use the specified keytab (args[1]), but we're already logged in and don't need to, so we can just use the current user KerberosToken kt = new KerberosToken(); try { UserGroupInformation user = UserGroupInformation.getCurrentUser(); if (!user.hasKerberosCredentials()) { throw new IllegalStateException("Expected current user to have Kerberos credentials"); } // Get the principal via UGI principal = user.getUserName(); // Connector w/ the Kerberos creds ZooKeeperInstance inst = new ZooKeeperInstance(clientConf); Connector conn = inst.getConnector(principal, kt); // Do the explicit check to see if the user has the permission to get a delegation token if (!conn.securityOperations().hasSystemPermission(conn.whoami(), SystemPermission.OBTAIN_DELEGATION_TOKEN)) { log.error(principal + " doesn't have the " + SystemPermission.OBTAIN_DELEGATION_TOKEN.name() + " SystemPermission neccesary to obtain a delegation token. MapReduce tasks cannot automatically use the client's" + " credentials on remote servers. Delegation tokens provide a means to run MapReduce without distributing the user's credentials."); throw new IllegalStateException( conn.whoami() + " does not have permission to obtain a delegation token"); } // Fetch a delegation token from Accumulo token = conn.securityOperations().getDelegationToken(new DelegationTokenConfig()); } catch (Exception e) { final String msg = "Failed to acquire DelegationToken for use with MapReduce"; log.error(msg, e); throw new RuntimeException(msg, e); } } else { // Simple principal + password principal = args[0]; token = new PasswordToken(args[1]); } AccumuloInputFormat.setConnectorInfo(job, principal, token); AccumuloOutputFormat.setConnectorInfo(job, principal, token); job.setMapperClass(SeqMapClass.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Mutation.class); job.setNumReduceTasks(0); job.setOutputFormatClass(AccumuloOutputFormat.class); AccumuloOutputFormat.setCreateTables(job, true); AccumuloOutputFormat.setDefaultTableName(job, args[5]); AccumuloOutputFormat.setZooKeeperInstance(job, clientConf); job.waitForCompletion(true); return job.isSuccessful() ? 0 : 1; }
From source file:org.apache.accumulo.test.randomwalk.sequential.MapRedVerifyTool.java
License:Apache License
@Override public int run(String[] args) throws Exception { Job job = Job.getInstance(getConf(), this.getClass().getSimpleName()); job.setJarByClass(this.getClass()); if (job.getJar() == null) { log.error("M/R requires a jar file! Run mvn package."); return 1; }//w w w. ja v a 2 s . c o m ClientConfiguration clientConf = ClientConfiguration.loadDefault().withInstance(args[3]) .withZkHosts(args[4]); AccumuloInputFormat.setInputTableName(job, args[2]); AccumuloInputFormat.setZooKeeperInstance(job, clientConf); AccumuloOutputFormat.setDefaultTableName(job, args[5]); AccumuloOutputFormat.setZooKeeperInstance(job, clientConf); job.setInputFormatClass(AccumuloInputFormat.class); if (clientConf.getBoolean(ClientProperty.INSTANCE_RPC_SASL_ENABLED.getKey(), false)) { // Better be logged in KerberosToken token = new KerberosToken(); try { UserGroupInformation user = UserGroupInformation.getCurrentUser(); if (!user.hasKerberosCredentials()) { throw new IllegalStateException("Expected current user to have Kerberos credentials"); } String newPrincipal = user.getUserName(); ZooKeeperInstance inst = new ZooKeeperInstance(clientConf); Connector conn = inst.getConnector(newPrincipal, token); // Do the explicit check to see if the user has the permission to get a delegation token if (!conn.securityOperations().hasSystemPermission(conn.whoami(), SystemPermission.OBTAIN_DELEGATION_TOKEN)) { log.error(newPrincipal + " doesn't have the " + SystemPermission.OBTAIN_DELEGATION_TOKEN.name() + " SystemPermission neccesary to obtain a delegation token. MapReduce tasks cannot automatically use the client's" + " credentials on remote servers. Delegation tokens provide a means to run MapReduce without distributing the user's credentials."); throw new IllegalStateException( conn.whoami() + " does not have permission to obtain a delegation token"); } // Fetch a delegation token from Accumulo AuthenticationToken dt = conn.securityOperations().getDelegationToken(new DelegationTokenConfig()); // Set the delegation token instead of the kerberos token AccumuloInputFormat.setConnectorInfo(job, newPrincipal, dt); AccumuloOutputFormat.setConnectorInfo(job, newPrincipal, dt); } catch (Exception e) { final String msg = "Failed to acquire DelegationToken for use with MapReduce"; log.error(msg, e); throw new RuntimeException(msg, e); } } else { AccumuloInputFormat.setConnectorInfo(job, args[0], new PasswordToken(args[1])); AccumuloOutputFormat.setConnectorInfo(job, args[0], new PasswordToken(args[1])); } job.setMapperClass(SeqMapClass.class); job.setMapOutputKeyClass(NullWritable.class); job.setMapOutputValueClass(IntWritable.class); job.setReducerClass(SeqReduceClass.class); job.setNumReduceTasks(1); job.setOutputFormatClass(AccumuloOutputFormat.class); AccumuloOutputFormat.setCreateTables(job, true); job.waitForCompletion(true); return job.isSuccessful() ? 0 : 1; }
From source file:org.apache.accumulo.test.randomwalk.shard.SortTool.java
License:Apache License
@Override public int run(String[] args) throws Exception { Job job = Job.getInstance(getConf(), this.getClass().getSimpleName()); job.setJarByClass(this.getClass()); if (job.getJar() == null) { log.error("M/R requires a jar file! Run mvn package."); return 1; }/*ww w . jav a 2s.co m*/ job.setInputFormatClass(SequenceFileInputFormat.class); SequenceFileInputFormat.setInputPaths(job, seqFile); job.setPartitionerClass(KeyRangePartitioner.class); KeyRangePartitioner.setSplitFile(job, splitFile); job.setMapOutputKeyClass(Key.class); job.setMapOutputValueClass(Value.class); job.setNumReduceTasks(splits.size() + 1); job.setOutputFormatClass(AccumuloFileOutputFormat.class); AccumuloFileOutputFormat.setOutputPath(job, new Path(outputDir)); job.waitForCompletion(true); return job.isSuccessful() ? 0 : 1; }
From source file:org.apache.accumulo.testing.core.continuous.ContinuousMoru.java
License:Apache License
@Override public int run(String[] args) throws IOException, InterruptedException, ClassNotFoundException, AccumuloSecurityException { Properties props = TestProps.loadFromFile(args[0]); ContinuousEnv env = new ContinuousEnv(props); Job job = Job.getInstance(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis()); job.setJarByClass(this.getClass()); job.setInputFormatClass(AccumuloInputFormat.class); AccumuloInputFormat.setConnectorInfo(job, env.getAccumuloUserName(), env.getToken()); AccumuloInputFormat.setInputTableName(job, env.getAccumuloTableName()); AccumuloInputFormat.setZooKeeperInstance(job, env.getClientConfiguration()); int maxMaps = Integer.parseInt(props.getProperty(TestProps.CI_VERIFY_MAX_MAPS)); // set up ranges try {//from w w w .j av a 2s. c o m Set<Range> ranges = env.getAccumuloConnector().tableOperations() .splitRangeByTablets(env.getAccumuloTableName(), new Range(), maxMaps); AccumuloInputFormat.setRanges(job, ranges); AccumuloInputFormat.setAutoAdjustRanges(job, false); } catch (Exception e) { throw new IOException(e); } job.setMapperClass(CMapper.class); job.setNumReduceTasks(0); job.setOutputFormatClass(AccumuloOutputFormat.class); AccumuloOutputFormat.setBatchWriterOptions(job, env.getBatchWriterConfig()); AccumuloOutputFormat.setConnectorInfo(job, env.getAccumuloUserName(), env.getToken()); AccumuloOutputFormat.setCreateTables(job, true); AccumuloOutputFormat.setDefaultTableName(job, env.getAccumuloTableName()); AccumuloOutputFormat.setZooKeeperInstance(job, env.getClientConfiguration()); Configuration conf = job.getConfiguration(); conf.setLong(MIN, env.getRowMin()); conf.setLong(MAX, env.getRowMax()); conf.setInt(MAX_CF, env.getMaxColF()); conf.setInt(MAX_CQ, env.getMaxColQ()); conf.set(CI_ID, UUID.randomUUID().toString()); job.waitForCompletion(true); return job.isSuccessful() ? 0 : 1; }
From source file:org.apache.accumulo.testing.core.continuous.ContinuousVerify.java
License:Apache License
@Override public int run(String[] args) throws Exception { Properties props = TestProps.loadFromFile(args[0]); ContinuousEnv env = new ContinuousEnv(props); Job job = Job.getInstance(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis()); job.setJarByClass(this.getClass()); job.setInputFormatClass(AccumuloInputFormat.class); boolean scanOffline = Boolean.parseBoolean(props.getProperty(TestProps.CI_VERIFY_SCAN_OFFLINE)); String tableName = env.getAccumuloTableName(); int maxMaps = Integer.parseInt(props.getProperty(TestProps.CI_VERIFY_MAX_MAPS)); int reducers = Integer.parseInt(props.getProperty(TestProps.CI_VERIFY_REDUCERS)); String outputDir = props.getProperty(TestProps.CI_VERIFY_OUTPUT_DIR); Set<Range> ranges;/*from ww w. j a v a2s .com*/ String clone = ""; Connector conn = env.getAccumuloConnector(); if (scanOffline) { Random random = new Random(); clone = tableName + "_" + String.format("%016x", (random.nextLong() & 0x7fffffffffffffffL)); conn.tableOperations().clone(tableName, clone, true, new HashMap<>(), new HashSet<>()); ranges = conn.tableOperations().splitRangeByTablets(tableName, new Range(), maxMaps); conn.tableOperations().offline(clone); AccumuloInputFormat.setInputTableName(job, clone); AccumuloInputFormat.setOfflineTableScan(job, true); } else { ranges = conn.tableOperations().splitRangeByTablets(tableName, new Range(), maxMaps); AccumuloInputFormat.setInputTableName(job, tableName); } AccumuloInputFormat.setRanges(job, ranges); AccumuloInputFormat.setAutoAdjustRanges(job, false); AccumuloInputFormat.setConnectorInfo(job, env.getAccumuloUserName(), env.getToken()); AccumuloInputFormat.setZooKeeperInstance(job, env.getClientConfiguration()); job.setMapperClass(CMapper.class); job.setMapOutputKeyClass(LongWritable.class); job.setMapOutputValueClass(VLongWritable.class); job.setReducerClass(CReducer.class); job.setNumReduceTasks(reducers); job.setOutputFormatClass(TextOutputFormat.class); job.getConfiguration().setBoolean("mapred.map.tasks.speculative.execution", scanOffline); TextOutputFormat.setOutputPath(job, new Path(outputDir)); job.waitForCompletion(true); if (scanOffline) { conn.tableOperations().delete(clone); } return job.isSuccessful() ? 0 : 1; }
From source file:org.apache.accumulo.testing.core.randomwalk.multitable.CopyTool.java
License:Apache License
@Override public int run(String[] args) throws Exception { Job job = Job.getInstance(getConf(), this.getClass().getSimpleName()); job.setJarByClass(this.getClass()); if (job.getJar() == null) { log.error("M/R requires a jar file! Run mvn package."); return 1; }/*from w w w .j av a 2s .c o m*/ ClientConfiguration clientConf = new ClientConfiguration().withInstance(args[3]).withZkHosts(args[4]); job.setInputFormatClass(AccumuloInputFormat.class); AccumuloInputFormat.setInputTableName(job, args[2]); AccumuloInputFormat.setScanAuthorizations(job, Authorizations.EMPTY); AccumuloInputFormat.setZooKeeperInstance(job, clientConf); final String principal; final AuthenticationToken token; if (clientConf.getBoolean(ClientProperty.INSTANCE_RPC_SASL_ENABLED.getKey(), false)) { // Use the Kerberos creds to request a DelegationToken for MapReduce // to use // We could use the specified keytab (args[1]), but we're already // logged in and don't need to, so we can just use the current user KerberosToken kt = new KerberosToken(); try { UserGroupInformation user = UserGroupInformation.getCurrentUser(); if (!user.hasKerberosCredentials()) { throw new IllegalStateException("Expected current user to have Kerberos credentials"); } // Get the principal via UGI principal = user.getUserName(); // Connector w/ the Kerberos creds ZooKeeperInstance inst = new ZooKeeperInstance(clientConf); Connector conn = inst.getConnector(principal, kt); // Do the explicit check to see if the user has the permission // to get a delegation token if (!conn.securityOperations().hasSystemPermission(conn.whoami(), SystemPermission.OBTAIN_DELEGATION_TOKEN)) { log.error(principal + " doesn't have the " + SystemPermission.OBTAIN_DELEGATION_TOKEN.name() + " SystemPermission neccesary to obtain a delegation token. MapReduce tasks cannot automatically use the client's" + " credentials on remote servers. Delegation tokens provide a means to run MapReduce without distributing the user's credentials."); throw new IllegalStateException( conn.whoami() + " does not have permission to obtain a delegation token"); } // Fetch a delegation token from Accumulo token = conn.securityOperations().getDelegationToken(new DelegationTokenConfig()); } catch (Exception e) { final String msg = "Failed to acquire DelegationToken for use with MapReduce"; log.error(msg, e); throw new RuntimeException(msg, e); } } else { // Simple principal + password principal = args[0]; token = new PasswordToken(args[1]); } AccumuloInputFormat.setConnectorInfo(job, principal, token); AccumuloOutputFormat.setConnectorInfo(job, principal, token); job.setMapperClass(SeqMapClass.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Mutation.class); job.setNumReduceTasks(0); job.setOutputFormatClass(AccumuloOutputFormat.class); AccumuloOutputFormat.setCreateTables(job, true); AccumuloOutputFormat.setDefaultTableName(job, args[5]); AccumuloOutputFormat.setZooKeeperInstance(job, clientConf); job.waitForCompletion(true); return job.isSuccessful() ? 0 : 1; }
From source file:org.apache.accumulo.testing.core.randomwalk.sequential.MapRedVerifyTool.java
License:Apache License
@Override public int run(String[] args) throws Exception { Job job = Job.getInstance(getConf(), this.getClass().getSimpleName()); job.setJarByClass(this.getClass()); if (job.getJar() == null) { log.error("M/R requires a jar file! Run mvn package."); return 1; }/*from w ww. java2s . c o m*/ ClientConfiguration clientConf = ClientConfiguration.loadDefault().withInstance(args[3]) .withZkHosts(args[4]); AccumuloInputFormat.setInputTableName(job, args[2]); AccumuloInputFormat.setZooKeeperInstance(job, clientConf); AccumuloOutputFormat.setDefaultTableName(job, args[5]); AccumuloOutputFormat.setZooKeeperInstance(job, clientConf); job.setInputFormatClass(AccumuloInputFormat.class); if (clientConf.getBoolean(ClientProperty.INSTANCE_RPC_SASL_ENABLED.getKey(), false)) { // Better be logged in KerberosToken token = new KerberosToken(); try { UserGroupInformation user = UserGroupInformation.getCurrentUser(); if (!user.hasKerberosCredentials()) { throw new IllegalStateException("Expected current user to have Kerberos credentials"); } String newPrincipal = user.getUserName(); ZooKeeperInstance inst = new ZooKeeperInstance(clientConf); Connector conn = inst.getConnector(newPrincipal, token); // Do the explicit check to see if the user has the permission // to get a delegation token if (!conn.securityOperations().hasSystemPermission(conn.whoami(), SystemPermission.OBTAIN_DELEGATION_TOKEN)) { log.error(newPrincipal + " doesn't have the " + SystemPermission.OBTAIN_DELEGATION_TOKEN.name() + " SystemPermission neccesary to obtain a delegation token. MapReduce tasks cannot automatically use the client's" + " credentials on remote servers. Delegation tokens provide a means to run MapReduce without distributing the user's credentials."); throw new IllegalStateException( conn.whoami() + " does not have permission to obtain a delegation token"); } // Fetch a delegation token from Accumulo AuthenticationToken dt = conn.securityOperations().getDelegationToken(new DelegationTokenConfig()); // Set the delegation token instead of the kerberos token AccumuloInputFormat.setConnectorInfo(job, newPrincipal, dt); AccumuloOutputFormat.setConnectorInfo(job, newPrincipal, dt); } catch (Exception e) { final String msg = "Failed to acquire DelegationToken for use with MapReduce"; log.error(msg, e); throw new RuntimeException(msg, e); } } else { AccumuloInputFormat.setConnectorInfo(job, args[0], new PasswordToken(args[1])); AccumuloOutputFormat.setConnectorInfo(job, args[0], new PasswordToken(args[1])); } job.setMapperClass(SeqMapClass.class); job.setMapOutputKeyClass(NullWritable.class); job.setMapOutputValueClass(IntWritable.class); job.setReducerClass(SeqReduceClass.class); job.setNumReduceTasks(1); job.setOutputFormatClass(AccumuloOutputFormat.class); AccumuloOutputFormat.setCreateTables(job, true); job.waitForCompletion(true); return job.isSuccessful() ? 0 : 1; }