List of usage examples for org.apache.hadoop.mapred JobClient JobClient
public JobClient(Configuration conf) throws IOException
From source file:DataJoinJob.java
License:Apache License
/** * Submit/run a map/reduce job.//from w w w .ja v a 2s .c om * * @param job * @return true for success * @throws IOException */ public static boolean runJob(JobConf job) throws IOException { JobClient jc = new JobClient(job); boolean sucess = true; RunningJob running = null; try { running = jc.submitJob(job); JobID jobId = running.getID(); System.out.println("Job " + jobId + " is submitted"); while (!running.isComplete()) { System.out.println("Job " + jobId + " is still running."); try { Thread.sleep(60000); } catch (InterruptedException e) { } running = jc.getJob(jobId); } sucess = running.isSuccessful(); } finally { if (!sucess && (running != null)) { running.killJob(); } jc.close(); } return sucess; }
From source file:Text2FormatStorageMR.java
License:Open Source License
@SuppressWarnings("deprecation") public static void main(String[] args) throws Exception { if (args.length != 2) { System.out.println("FormatFileMR <input> <output>"); System.exit(-1);//from ww w.j a v a2 s.c o m } JobConf conf = new JobConf(FormatStorageMR.class); conf.setJobName("Text2FormatMR"); conf.setNumMapTasks(1); conf.setNumReduceTasks(4); conf.setOutputKeyClass(LongWritable.class); conf.setOutputValueClass(Unit.Record.class); conf.setMapperClass(TextFileTestMapper.class); conf.setReducerClass(FormatFileTestReducer.class); conf.setInputFormat(TextInputFormat.class); conf.setOutputFormat(FormatStorageOutputFormat.class); conf.set("mapred.output.compress", "flase"); Head head = new Head(); initHead(head); head.toJobConf(conf); FileInputFormat.setInputPaths(conf, args[0]); Path outputPath = new Path(args[1]); FileOutputFormat.setOutputPath(conf, outputPath); FileSystem fs = outputPath.getFileSystem(conf); fs.delete(outputPath, true); JobClient jc = new JobClient(conf); RunningJob rj = null; rj = jc.submitJob(conf); String lastReport = ""; SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss,SSS"); long reportTime = System.currentTimeMillis(); long maxReportInterval = 3 * 1000; while (!rj.isComplete()) { try { Thread.sleep(1000); } catch (InterruptedException e) { } int mapProgress = Math.round(rj.mapProgress() * 100); int reduceProgress = Math.round(rj.reduceProgress() * 100); String report = " map = " + mapProgress + "%, reduce = " + reduceProgress + "%"; if (!report.equals(lastReport) || System.currentTimeMillis() >= reportTime + maxReportInterval) { String output = dateFormat.format(Calendar.getInstance().getTime()) + report; System.out.println(output); lastReport = report; reportTime = System.currentTimeMillis(); } } System.exit(0); }
From source file:Text2ColumntStorageMR.java
License:Open Source License
@SuppressWarnings("deprecation") public static void main(String[] args) throws Exception { if (args.length != 3) { System.out.println("Text2ColumnStorageMR <input> <output> <columnStorageMode>"); System.exit(-1);//from ww w. ja v a 2 s . c o m } JobConf conf = new JobConf(Text2ColumntStorageMR.class); conf.setJobName("Text2ColumnStorageMR"); conf.setNumMapTasks(1); conf.setNumReduceTasks(4); conf.setOutputKeyClass(LongWritable.class); conf.setOutputValueClass(Unit.Record.class); conf.setMapperClass(TextFileMapper.class); conf.setReducerClass(ColumnStorageReducer.class); conf.setInputFormat(TextInputFormat.class); conf.setOutputFormat((Class<? extends OutputFormat>) ColumnStorageHiveOutputFormat.class); conf.set("mapred.output.compress", "flase"); Head head = new Head(); initHead(head); head.toJobConf(conf); int bt = Integer.valueOf(args[2]); FileInputFormat.setInputPaths(conf, args[0]); Path outputPath = new Path(args[1]); FileOutputFormat.setOutputPath(conf, outputPath); FileSystem fs = outputPath.getFileSystem(conf); fs.delete(outputPath, true); JobClient jc = new JobClient(conf); RunningJob rj = null; rj = jc.submitJob(conf); String lastReport = ""; SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss,SSS"); long reportTime = System.currentTimeMillis(); long maxReportInterval = 3 * 1000; while (!rj.isComplete()) { try { Thread.sleep(1000); } catch (InterruptedException e) { } int mapProgress = Math.round(rj.mapProgress() * 100); int reduceProgress = Math.round(rj.reduceProgress() * 100); String report = " map = " + mapProgress + "%, reduce = " + reduceProgress + "%"; if (!report.equals(lastReport) || System.currentTimeMillis() >= reportTime + maxReportInterval) { String output = dateFormat.format(Calendar.getInstance().getTime()) + report; System.out.println(output); lastReport = report; reportTime = System.currentTimeMillis(); } } System.exit(0); }
From source file:FormatStorage2ColumnStorageMR.java
License:Open Source License
@SuppressWarnings("deprecation") public static void main(String[] args) throws Exception { if (args.length != 2) { System.out.println("FormatStorage2ColumnStorageMR <input> <output>"); System.exit(-1);//from w ww . j a v a 2 s .c om } JobConf conf = new JobConf(FormatStorageMR.class); conf.setJobName("FormatStorage2ColumnStorageMR"); conf.setNumMapTasks(1); conf.setNumReduceTasks(4); conf.setOutputKeyClass(LongWritable.class); conf.setOutputValueClass(Unit.Record.class); conf.setMapperClass(FormatStorageMapper.class); conf.setReducerClass(ColumnStorageReducer.class); conf.setInputFormat(FormatStorageInputFormat.class); conf.set("mapred.output.compress", "flase"); Head head = new Head(); initHead(head); head.toJobConf(conf); FileInputFormat.setInputPaths(conf, args[0]); Path outputPath = new Path(args[1]); FileOutputFormat.setOutputPath(conf, outputPath); FileSystem fs = outputPath.getFileSystem(conf); fs.delete(outputPath, true); JobClient jc = new JobClient(conf); RunningJob rj = null; rj = jc.submitJob(conf); String lastReport = ""; SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss,SSS"); long reportTime = System.currentTimeMillis(); long maxReportInterval = 3 * 1000; while (!rj.isComplete()) { try { Thread.sleep(1000); } catch (InterruptedException e) { } int mapProgress = Math.round(rj.mapProgress() * 100); int reduceProgress = Math.round(rj.reduceProgress() * 100); String report = " map = " + mapProgress + "%, reduce = " + reduceProgress + "%"; if (!report.equals(lastReport) || System.currentTimeMillis() >= reportTime + maxReportInterval) { String output = dateFormat.format(Calendar.getInstance().getTime()) + report; System.out.println(output); lastReport = report; reportTime = System.currentTimeMillis(); } } System.exit(0); }
From source file:azkaban.jobtype.javautils.AbstractHadoopJob.java
License:Apache License
public void run() throws Exception { JobConf conf = getJobConf();/* www . j a va 2 s .com*/ if (System.getenv(HADOOP_TOKEN_FILE_LOCATION) != null) { conf.set(MAPREDUCE_JOB_CREDENTIALS_BINARY, System.getenv(HADOOP_TOKEN_FILE_LOCATION)); } jobClient = new JobClient(conf); runningJob = jobClient.submitJob(conf); logger.info("See " + runningJob.getTrackingURL() + " for details."); jobClient.monitorAndPrintJob(conf, runningJob); if (!runningJob.isSuccessful()) { throw new Exception("Hadoop job:" + getJobName() + " failed!"); } // dump all counters Counters counters = runningJob.getCounters(); for (String groupName : counters.getGroupNames()) { Counters.Group group = counters.getGroup(groupName); logger.info("Group: " + group.getDisplayName()); for (Counter counter : group) logger.info(counter.getDisplayName() + ":\t" + counter.getValue()); } updateMapReduceJobState(conf); }
From source file:azkaban.jobtype.SecurePigWrapper.java
License:Apache License
public static void main(final String[] args) throws IOException, InterruptedException { final Logger logger = Logger.getRootLogger(); final Properties p = System.getProperties(); final Configuration conf = new Configuration(); SecurityUtils.getProxiedUser(p, logger, conf).doAs(new PrivilegedExceptionAction<Void>() { @Override/*ww w. j av a 2 s . co m*/ public Void run() throws Exception { prefetchToken(); org.apache.pig.Main.main(args); return null; } // For Pig jobs that need to do extra communication with the // JobTracker, it's necessary to pre-fetch a token and include it in // the credentials cache private void prefetchToken() throws InterruptedException, IOException { String shouldPrefetch = p.getProperty(OBTAIN_BINARY_TOKEN); if (shouldPrefetch != null && shouldPrefetch.equals("true")) { logger.info("Pre-fetching token"); Job job = new Job(conf, "totally phony, extremely fake, not real job"); JobConf jc = new JobConf(conf); JobClient jobClient = new JobClient(jc); logger.info("Pre-fetching: Got new JobClient: " + jc); Token<DelegationTokenIdentifier> mrdt = jobClient.getDelegationToken(new Text("hi")); job.getCredentials().addToken(new Text("howdy"), mrdt); File temp = File.createTempFile("mr-azkaban", ".token"); temp.deleteOnExit(); FileOutputStream fos = null; DataOutputStream dos = null; try { fos = new FileOutputStream(temp); dos = new DataOutputStream(fos); job.getCredentials().writeTokenStorageToStream(dos); } finally { if (dos != null) { dos.close(); } if (fos != null) { fos.close(); } } logger.info("Setting " + MAPREDUCE_JOB_CREDENTIALS_BINARY + " to " + temp.getAbsolutePath()); System.setProperty(MAPREDUCE_JOB_CREDENTIALS_BINARY, temp.getAbsolutePath()); } else { logger.info("Not pre-fetching token"); } } }); }
From source file:azkaban.security.commons.SecurityUtils.java
License:Apache License
public static synchronized void prefetchToken(final File tokenFile, final Props p, final Logger logger) throws InterruptedException, IOException { final Configuration conf = new Configuration(); logger.info("Getting proxy user for " + p.getString(TO_PROXY)); logger.info("Getting proxy user for " + p.toString()); getProxiedUser(p.toProperties(), logger, conf).doAs(new PrivilegedExceptionAction<Void>() { @Override// w w w .jav a 2s. c o m public Void run() throws Exception { getToken(p); return null; } private void getToken(Props p) throws InterruptedException, IOException { String shouldPrefetch = p.getString(OBTAIN_BINARY_TOKEN); if (shouldPrefetch != null && shouldPrefetch.equals("true")) { logger.info("Pre-fetching token"); logger.info("Pre-fetching fs token"); FileSystem fs = FileSystem.get(conf); Token<?> fsToken = fs.getDelegationToken(p.getString("user.to.proxy")); logger.info("Created token: " + fsToken.toString()); Job job = new Job(conf, "totally phony, extremely fake, not real job"); JobConf jc = new JobConf(conf); JobClient jobClient = new JobClient(jc); logger.info("Pre-fetching job token: Got new JobClient: " + jc); Token<DelegationTokenIdentifier> mrdt = jobClient.getDelegationToken(new Text("hi")); logger.info("Created token: " + mrdt.toString()); job.getCredentials().addToken(new Text("howdy"), mrdt); job.getCredentials().addToken(fsToken.getService(), fsToken); FileOutputStream fos = null; DataOutputStream dos = null; try { fos = new FileOutputStream(tokenFile); dos = new DataOutputStream(fos); job.getCredentials().writeTokenStorageToStream(dos); } finally { if (dos != null) { dos.close(); } if (fos != null) { fos.close(); } } logger.info("Loading hadoop tokens into " + tokenFile.getAbsolutePath()); p.put("HadoopTokenFileLoc", tokenFile.getAbsolutePath()); } else { logger.info("Not pre-fetching token"); } } }); }
From source file:azkaban.security.HadoopSecurityManager_H_1_0.java
License:Apache License
@Override public synchronized void prefetchToken(final File tokenFile, final String userToProxy, final Logger logger) throws HadoopSecurityManagerException { logger.info("Getting hadoop tokens for " + userToProxy); try {//from ww w . j av a 2 s .c o m getProxiedUser(userToProxy).doAs(new PrivilegedExceptionAction<Void>() { @Override public Void run() throws Exception { getToken(userToProxy); return null; } private void getToken(String userToProxy) throws InterruptedException, IOException, HadoopSecurityManagerException { FileSystem fs = FileSystem.get(conf); // check if we get the correct FS, and most importantly, the conf logger.info("Getting DFS token from " + fs.getCanonicalServiceName() + fs.getUri()); Token<?> fsToken = fs.getDelegationToken(userToProxy); if (fsToken == null) { logger.error("Failed to fetch DFS token for "); throw new HadoopSecurityManagerException("Failed to fetch DFS token for " + userToProxy); } logger.info("Created DFS token: " + fsToken.toString()); logger.info("Token kind: " + fsToken.getKind()); logger.info("Token id: " + fsToken.getIdentifier()); logger.info("Token service: " + fsToken.getService()); JobConf jc = new JobConf(conf); JobClient jobClient = new JobClient(jc); logger.info("Pre-fetching JT token: Got new JobClient: " + jc); Token<DelegationTokenIdentifier> mrdt = jobClient.getDelegationToken(new Text("mr token")); if (mrdt == null) { logger.error("Failed to fetch JT token for "); throw new HadoopSecurityManagerException("Failed to fetch JT token for " + userToProxy); } logger.info("Created JT token: " + mrdt.toString()); logger.info("Token kind: " + mrdt.getKind()); logger.info("Token id: " + mrdt.getIdentifier()); logger.info("Token service: " + mrdt.getService()); jc.getCredentials().addToken(mrdt.getService(), mrdt); jc.getCredentials().addToken(fsToken.getService(), fsToken); FileOutputStream fos = null; DataOutputStream dos = null; try { fos = new FileOutputStream(tokenFile); dos = new DataOutputStream(fos); jc.getCredentials().writeTokenStorageToStream(dos); } finally { if (dos != null) { dos.close(); } if (fos != null) { fos.close(); } } // stash them to cancel after use. logger.info("Tokens loaded in " + tokenFile.getAbsolutePath()); } }); } catch (Exception e) { e.printStackTrace(); throw new HadoopSecurityManagerException( "Failed to get hadoop tokens! " + e.getMessage() + e.getCause()); } }
From source file:azkaban.security.HadoopSecurityManager_H_1_0.java
License:Apache License
private void cancelMRJobTrackerToken(final Token<? extends TokenIdentifier> t, String userToProxy) throws HadoopSecurityManagerException { try {// w ww .j a va 2 s . c om getProxiedUser(userToProxy).doAs(new PrivilegedExceptionAction<Void>() { @SuppressWarnings("unchecked") @Override public Void run() throws Exception { cancelToken((Token<DelegationTokenIdentifier>) t); return null; } private void cancelToken(Token<DelegationTokenIdentifier> jt) throws IOException, InterruptedException { JobConf jc = new JobConf(conf); JobClient jobClient = new JobClient(jc); jobClient.cancelDelegationToken(jt); } }); } catch (Exception e) { e.printStackTrace(); throw new HadoopSecurityManagerException("Failed to cancel Token. " + e.getMessage() + e.getCause()); } }
From source file:azkaban.security.HadoopSecurityManager_H_1_0.java
License:Apache License
@Override public synchronized void prefetchToken(final File tokenFile, final Props props, final Logger logger) throws HadoopSecurityManagerException { final String userToProxy = props.getString(USER_TO_PROXY); logger.info("Getting hadoop tokens for " + userToProxy); final Credentials cred = new Credentials(); if (props.getBoolean(OBTAIN_HCAT_TOKEN, false)) { try {//from w w w. j ava2 s. c o m logger.info("Pre-fetching Hive MetaStore token from hive"); HiveConf hiveConf = new HiveConf(); logger.info("HiveConf.ConfVars.METASTOREURIS.varname " + hiveConf.get(HiveConf.ConfVars.METASTOREURIS.varname)); logger.info("HIVE_METASTORE_SASL_ENABLED " + hiveConf.get(HIVE_METASTORE_SASL_ENABLED)); logger.info("HIVE_METASTORE_KERBEROS_PRINCIPAL " + hiveConf.get(HIVE_METASTORE_KERBEROS_PRINCIPAL)); logger.info("HIVE_METASTORE_LOCAL " + hiveConf.get(HIVE_METASTORE_LOCAL)); HiveMetaStoreClient hiveClient = new HiveMetaStoreClient(hiveConf); String hcatTokenStr = hiveClient.getDelegationToken(userToProxy, UserGroupInformation.getLoginUser().getShortUserName()); Token<DelegationTokenIdentifier> hcatToken = new Token<DelegationTokenIdentifier>(); hcatToken.decodeFromUrlString(hcatTokenStr); logger.info("Created hive metastore token: " + hcatTokenStr); logger.info("Token kind: " + hcatToken.getKind()); logger.info("Token id: " + hcatToken.getIdentifier()); logger.info("Token service: " + hcatToken.getService()); cred.addToken(hcatToken.getService(), hcatToken); } catch (Exception e) { e.printStackTrace(); logger.error("Failed to get hive metastore token." + e.getMessage() + e.getCause()); } catch (Throwable t) { t.printStackTrace(); logger.error("Failed to get hive metastore token." + t.getMessage() + t.getCause()); } } try { getProxiedUser(userToProxy).doAs(new PrivilegedExceptionAction<Void>() { @Override public Void run() throws Exception { getToken(userToProxy); return null; } private void getToken(String userToProxy) throws InterruptedException, IOException, HadoopSecurityManagerException { logger.info("Here is the props for " + OBTAIN_NAMENODE_TOKEN + ": " + props.getBoolean(OBTAIN_NAMENODE_TOKEN)); if (props.getBoolean(OBTAIN_NAMENODE_TOKEN, false)) { FileSystem fs = FileSystem.get(conf); // check if we get the correct FS, and most importantly, the // conf logger.info("Getting DFS token from " + fs.getUri()); Token<?> fsToken = fs.getDelegationToken(userToProxy); if (fsToken == null) { logger.error("Failed to fetch DFS token for "); throw new HadoopSecurityManagerException( "Failed to fetch DFS token for " + userToProxy); } logger.info("Created DFS token: " + fsToken.toString()); logger.info("Token kind: " + fsToken.getKind()); logger.info("Token id: " + fsToken.getIdentifier()); logger.info("Token service: " + fsToken.getService()); cred.addToken(fsToken.getService(), fsToken); } if (props.getBoolean(OBTAIN_JOBTRACKER_TOKEN, false)) { JobClient jobClient = new JobClient(new JobConf()); logger.info("Pre-fetching JT token from JobTracker"); Token<DelegationTokenIdentifier> mrdt = jobClient.getDelegationToken(new Text("mr token")); if (mrdt == null) { logger.error("Failed to fetch JT token"); throw new HadoopSecurityManagerException("Failed to fetch JT token for " + userToProxy); } logger.info("Created JT token: " + mrdt.toString()); logger.info("Token kind: " + mrdt.getKind()); logger.info("Token id: " + mrdt.getIdentifier()); logger.info("Token service: " + mrdt.getService()); cred.addToken(mrdt.getService(), mrdt); } } }); FileOutputStream fos = null; DataOutputStream dos = null; try { fos = new FileOutputStream(tokenFile); dos = new DataOutputStream(fos); cred.writeTokenStorageToStream(dos); } finally { if (dos != null) { dos.close(); } if (fos != null) { fos.close(); } } // stash them to cancel after use. logger.info("Tokens loaded in " + tokenFile.getAbsolutePath()); } catch (Exception e) { e.printStackTrace(); throw new HadoopSecurityManagerException( "Failed to get hadoop tokens! " + e.getMessage() + e.getCause()); } catch (Throwable t) { t.printStackTrace(); throw new HadoopSecurityManagerException( "Failed to get hadoop tokens! " + t.getMessage() + t.getCause()); } }