List of usage examples for org.apache.hadoop.fs Path toString
@Override
public String toString()
From source file:com.chinamobile.bcbsp.bspcontroller.JobInProgress.java
License:Apache License
/** * jobInProgress construct method./*from ww w . ja v a 2s . c o m*/ * @param jobId * BSP job id. * @param jobFile * BSP job file. * @param controller * BSP controller * @param conf * BSP System configuration. * @throws IOException * exceptions during handle BSP job file. */ public JobInProgress(BSPJobID jobId, Path jobFile, BSPController controller, Configuration conf) throws IOException { this.jobId = jobId; // this.localFs = FileSystem.getLocal(conf); this.BSPlocalFs = new BSPLocalFileSystemImpl(conf); this.jobFile = jobFile; this.controller = controller; // this.status = new JobStatus(jobId, null, 0L, 0L, // JobStatus.State.PREP.value()); this.startTime = System.currentTimeMillis(); //ljn SGA_Graph this.superStepCounter = -2; // this.superStepCounter = 0; this.maxAttemptRecoveryCounter = conf.getInt(Constants.BC_BSP_JOB_RECOVERY_ATTEMPT_MAX, 0); this.maxStaffAttemptRecoveryCounter = conf.getInt(Constants.BC_BSP_STAFF_RECOVERY_ATTEMPT_MAX, 0); this.localJobFile = controller .getLocalPath(Constants.BC_BSP_LOCAL_SUBDIR_CONTROLLER + "/" + jobId + ".xml"); this.localJarFile = controller .getLocalPath(Constants.BC_BSP_LOCAL_SUBDIR_CONTROLLER + "/" + jobId + ".jar"); Path jobDir = controller.getSystemDirectoryForJob(jobId); // FileSystem fs = jobDir.getFileSystem(conf); BSPFileSystem bspfs = new BSPFileSystemImpl(controller, jobId, conf); bspfs.copyToLocalFile(jobFile, localJobFile); job = new BSPJob(jobId, localJobFile.toString()); this.conf = job.getConf(); this.numBSPStaffs = job.getNumBspStaff(); this.profile = new JobProfile(job.getUser(), jobId, jobFile.toString(), job.getJobName()); String jarFile = job.getJar(); if (jarFile != null) { bspfs.copyToLocalFile(new BSPHdfsImpl().newPath(jarFile), localJarFile); } // chang by cui this.priority = job.getPriority(); this.superStepNum = job.getNumSuperStep(); this.status = new JobStatus(jobId, null, 0L, 0L, 0L, JobStatus.State.PREP.value(), this.superStepNum); // new JobStatus(jobId, null, 0L, 0L, JobStatus.State.PREP.value()); this.status.setUsername(job.getUser()); this.status.setStartTime(startTime); // added by feng this.edgeNum = job.getOutgoingEdgeNum(); setCheckPointFrequency(); // For aggregation. /** Add the user program jar to the system's classpath. */ ClassLoaderUtil.addClassPath(localJarFile.toString()); loadAggregators(); this.gssc = new GeneralSSController(jobId); }
From source file:com.chinamobile.bcbsp.client.BSPJobClient.java
License:Apache License
/** * Submit a new job to run./* ww w . java2s . c om*/ * @param job BSPJob * @return Review comments: (1)The content of submitJobDir is decided by the * client. I think it is dangerous because two different clients maybe * generate the same submitJobDir. Review time: 2011-11-30; Reviewer: * Hongxu Zhang. Fix log: (1)In order to avoid the conflict, I use the * jobId to generate the submitJobDir. Because the jobId is unique so * this problem can be solved. Fix time: 2011-12-04; Programmer: * Zhigang Wang. Review comments: (2)There, the client must submit * relative information about the job. There maybe some exceptions * during this process. When exceptions occur, this job should not be * executed and the relative submitJobDir must be cleanup. Review * time: 2011-12-04; Reviewer: Hongxu Zhang. Fix log: (2)The process * of submiting files has been surrounded by try-catch. The * submitJobDir will be cleanup in the catch process. Fix time: * 2011-12-04; Programmer: Zhigang Wang. */ public RunningJob submitJobInternal(BSPJob job) { BSPJobID jobId = null; Path submitJobDir = null; try { jobId = jobSubmitClient.getNewJobId(); submitJobDir = new Path(getSystemDir(), "submit_" + jobId.toString()); Path submitJarFile = null; LOG.info("debug: job type is " + job.getJobType()); if (Constants.USER_BC_BSP_JOB_TYPE_C.equals(job.getJobType())) { submitJarFile = new Path(submitJobDir, "jobC"); LOG.info("debug:" + submitJarFile.toString()); } else { LOG.info("debug: before submitJarFile = new " + "Path(submitJobDir,job.jar);"); submitJarFile = new Path(submitJobDir, "job.jar"); LOG.info("debug:" + submitJarFile.toString()); } Path submitJobFile = new Path(submitJobDir, "job.xml"); Path submitSplitFile = new Path(submitJobDir, "job.split"); // set this user's id in job configuration, so later job files can // be accessed using this user's id UnixUserGroupInformation ugi = getUGI(job.getConf()); // Create a number of filenames in the BSPController's fs namespace FileSystem files = getFs(); files.delete(submitJobDir, true); submitJobDir = files.makeQualified(submitJobDir); submitJobDir = new Path(submitJobDir.toUri().getPath()); BSPFsPermission bspSysPerms = new BSPFspermissionImpl(2); FileSystem.mkdirs(files, submitJobDir, bspSysPerms.getFp()); files.mkdirs(submitJobDir); short replication = (short) job.getInt("bsp.submit.replication", 10); String originalJarPath = null; LOG.info("debug: job type is " + job.getJobType()); if (Constants.USER_BC_BSP_JOB_TYPE_C.equals(job.getJobType())) { LOG.info("debug: originalJarPath = job.getJobExe();" + job.getJobExe()); originalJarPath = job.getJobExe(); LOG.info("debug:" + submitJarFile.toString()); job.setJobExe(submitJarFile.toString()); } else { LOG.info("debug: jar"); originalJarPath = job.getJar(); job.setJar(submitJarFile.toString()); } if (originalJarPath != null) { // copy jar to BSPController's fs // use jar name if job is not named. if ("".equals(job.getJobName())) { job.setJobName(new Path(originalJarPath).getName()); } // job.setJar(submitJarFile.toString()); fs.copyFromLocalFile(new Path(originalJarPath), submitJarFile); fs.setReplication(submitJarFile, replication); fs.setPermission(submitJarFile, new BSPFspermissionImpl(0).getFp()); } else { LOG.warn("No job jar file set. User classes may not be found. " + "See BSPJob#setJar(String) or check Your jar file."); } // Set the user's name and working directory job.setUser(ugi.getUserName()); if (ugi.getGroupNames().length > 0) { job.set("group.name", ugi.getGroupNames()[0]); } if (new BSPHdfsImpl().getWorkingDirectory() == null) { job.setWorkingDirectory(fs.getWorkingDirectory()); } int maxClusterStaffs = jobSubmitClient.getClusterStatus(false).getMaxClusterStaffs(); if (job.getNumPartition() == 0) { job.setNumPartition(maxClusterStaffs); } if (job.getNumPartition() > maxClusterStaffs) { job.setNumPartition(maxClusterStaffs); } job.setNumBspStaff(job.getNumPartition()); int splitNum = 0; splitNum = writeSplits(job, submitSplitFile); if (splitNum > job.getNumPartition() && splitNum <= maxClusterStaffs) { job.setNumPartition(splitNum); job.setNumBspStaff(job.getNumPartition()); } if (splitNum > maxClusterStaffs) { LOG.error("Sorry, the number of files is more than maxClusterStaffs:" + maxClusterStaffs); throw new IOException("Could not launch job"); } job.set(Constants.USER_BC_BSP_JOB_SPLIT_FILE, submitSplitFile.toString()); LOG.info("[Max Staff Number] " + maxClusterStaffs); LOG.info("The number of splits for the job is: " + splitNum); LOG.info("The number of staffs for the job is: " + job.getNumBspStaff()); BSPFSDataOutputStream bspout = new BSPFSDataOutputStreamImpl(fs, submitJobFile, new BSPFspermissionImpl(0).getFp()); try { job.writeXml(bspout.getOut()); } finally { bspout.close(); } // Now, actually submit the job (using the submit name) JobStatus status = jobSubmitClient.submitJob(jobId, submitJobFile.toString()); if (status != null) { return new NetworkedJob(status); } else { throw new IOException("Could not launch job"); } } catch (FileNotFoundException fnfE) { LOG.error("Exception has been catched in BSPJobClient--submitJobInternal !", fnfE); Fault f = new Fault(Fault.Type.SYSTEMSERVICE, Fault.Level.INDETERMINATE, "null", fnfE.toString()); jobSubmitClient.recordFault(f); jobSubmitClient.recovery(jobId); try { FileSystem files = getFs(); files.delete(submitJobDir, true); } catch (IOException e) { //LOG.error("Failed to cleanup the submitJobDir:" + submitJobDir); throw new RuntimeException("Failed to cleanup the submitJobDir", e); } return null; } catch (ClassNotFoundException cnfE) { LOG.error("Exception has been catched in BSPJobClient--submitJobInternal !", cnfE); Fault f = new Fault(Fault.Type.SYSTEMSERVICE, Fault.Level.WARNING, "null", cnfE.toString()); jobSubmitClient.recordFault(f); jobSubmitClient.recovery(jobId); try { FileSystem files = getFs(); files.delete(submitJobDir, true); } catch (IOException e) { //LOG.error("Failed to cleanup the submitJobDir:" + submitJobDir); throw new RuntimeException("Failed to cleanup the submitJobDir", e); } return null; } catch (InterruptedException iE) { LOG.error("Exception has been catched in BSPJobClient--submitJobInternal !", iE); Fault f = new Fault(Fault.Type.SYSTEMSERVICE, Fault.Level.CRITICAL, "null", iE.toString()); jobSubmitClient.recordFault(f); jobSubmitClient.recovery(jobId); try { FileSystem files = getFs(); files.delete(submitJobDir, true); } catch (IOException e) { //LOG.error("Failed to cleanup the submitJobDir:" + submitJobDir); throw new RuntimeException("Failed to cleanup the submitJobDir", e); } return null; } catch (Exception ioE) { LOG.error("Exception has been catched in BSPJobClient--submitJobInternal !", ioE); Fault f = new Fault(Fault.Type.DISK, Fault.Level.CRITICAL, "null", ioE.toString()); jobSubmitClient.recordFault(f); jobSubmitClient.recovery(jobId); try { FileSystem files = getFs(); files.delete(submitJobDir, true); } catch (IOException e) { //LOG.error("Failed to cleanup the submitJobDir:" + submitJobDir); throw new RuntimeException("Failed to cleanup the submitJobDir", e); } return null; } }
From source file:com.chinamobile.bcbsp.fault.storage.AggValueCheckpoint.java
License:Apache License
/** * write the aggVlaues of staffs during local compute. * @param aggValues//from ww w . j a va 2 s .c o m * aggregate values * @param writeAggPath * aggregate values write path. * @param job * job to write * @param staff * staff aggregate values to write * @return write result. * @throws IOException exceptions during write into hdfs. */ public boolean writeAggCheckPoint(String aggValues, Path writeAggPath, BSPJob job, Staff staff) throws IOException { LOG.info("The aggValue Checkpoint write path is : " + writeAggPath.toString()); try { OutputFormat outputformat = (OutputFormat) ReflectionUtils.newInstance( job.getConf().getClass(Constants.USER_BC_BSP_JOB_OUTPUT_FORMAT_CLASS, OutputFormat.class), job.getConf()); outputformat.initialize(job.getConf()); RecordWriter output = outputformat.getRecordWriter(job, staff.getStaffAttemptId(), writeAggPath); output.write(new Text(Constants.SSC_SPLIT_FLAG), new Text(aggValues)); output.close(job); } catch (Exception e) { LOG.error("Exception has happened and been catched!", e); return false; } return true; }
From source file:com.chinamobile.bcbsp.fault.storage.Checkpoint.java
License:Apache License
/** * Write Check point/*w w w .j av a2 s . c o m*/ * @param graphData * graphdata to checkpoint * @param writePath * checkpoint write path * @param job * job to checkpoint * @param staff * staff to checkpoint * @return boolean Note it should be modified , some detailed opts to be * sealed into graph data interface and should not be exposed outside. */ @SuppressWarnings("unchecked") public boolean writeCheckPoint(GraphDataInterface graphData, Path writePath, BSPJob job, Staff staff) throws IOException { LOG.info("The init write path is : " + writePath.toString()); try { OutputFormat outputformat = null; if (job.getCheckpointType().equals("HBase")) { createTable(job, staff.getStaffAttemptId()); outputformat = ReflectionUtils.newInstance(TableOutputFormat.class, job.getConf()); job.getConf().set(TableOutputFormat.OUTPUT_TABLE, staff.getStaffAttemptId().toString()); } else if (job.getCheckpointType().equals("HDFS")) { outputformat = ReflectionUtils.newInstance(TextBSPFileOutputFormat.class, job.getConf()); } outputformat.initialize(job.getConf()); RecordWriter output = outputformat.getRecordWriter(job, staff.getStaffAttemptId(), writePath); graphData.saveAllVertices(output); output.close(job); } catch (Exception e) { LOG.error("Exception has happened and been catched!", e); return false; } return true; }
From source file:com.chinamobile.bcbsp.fault.storage.Checkpoint.java
License:Apache License
/** * Read Check point/*from w w w .j ava 2 s. c om*/ * @param readPath * checkpoint readpath * @param job * job to read the checkpoint * @param staff * staff to get the graphdata * @return checkpoint graphdata */ @SuppressWarnings("unchecked") public GraphDataInterface readCheckPoint(Path readPath, BSPJob job, Staff staff) { GraphDataInterface graphData = null; GraphDataFactory graphDataFactory = staff.getGraphDataFactory(); int version = job.getGraphDataVersion(); graphData = graphDataFactory.createGraphData(version, staff); Vertex vertexTmp = null; String s = null; if (job.getCheckpointType().equals("HBase")) { HTablePool pool = new HTablePool(job.getConf(), 1000); HTable table = (HTable) pool.getTable(staff.getStaffAttemptId().toString()); try { ResultScanner rs = table.getScanner(new Scan()); for (Result r : rs) { KeyValue[] keyValue = r.raw(); s = new String(r.getRow()) + Constants.KV_SPLIT_FLAG + new String(keyValue[0].getValue()); if (s != null) { try { vertexTmp = this.vertexClass.newInstance(); vertexTmp.fromString(s); } catch (Exception e) { throw new RuntimeException("[Checkpoint] caught: ", e); } if (vertexTmp != null) { LOG.info("vertexTmp = " + vertexTmp); graphData.addForAll(vertexTmp); } } } } catch (IOException e) { e.printStackTrace(); } } else if (job.getCheckpointType().equals("HDFS")) { LOG.info("The init read path is : " + readPath.toString()); String uri = readPath.toString() + "/" + staff.getStaffAttemptId().toString() + "/checkpoint.cp"; LOG.info("uri: " + uri); // Configuration conf = new Configuration(); // alter by gtt BSPHdfs HdfsCheckpoint = new BSPHdfsImpl(); InputStream in = null; BufferedReader bis = null; try { // FileSystem fs = FileSystem.get(URI.create(uri), conf); // in = fs.open(new Path(uri)); in = HdfsCheckpoint.hdfsCheckpoint(uri, HdfsCheckpoint.getConf()); bis = new BufferedReader(new InputStreamReader(new BufferedInputStream(in))); s = bis.readLine(); while (s != null) { try { vertexTmp = this.vertexClass.newInstance(); vertexTmp.fromString(s); } catch (Exception e) { // LOG.error("[Checkpoint] caught: ", e); throw new RuntimeException("[Checkpoint] caught: ", e); } if (vertexTmp != null) { graphData.addForAll(vertexTmp); } s = bis.readLine(); } graphData.finishAdd(); bis.close(); } catch (IOException e) { // LOG.error("Exception has happened and been catched!", e); throw new RuntimeException("Exception has happened and been catched!", e); } } return graphData; }
From source file:com.chinamobile.bcbsp.fault.storage.Checkpoint.java
License:Apache License
/** * write the message and graphdata information for * migrate slow staff./*from w w w .j av a2 s . com*/ * @param communicator * message information need to backup * @param graphData * graphdata to backup * @param writePath * backup write path * @param job * job to backup * @param staff * staff to backup * @return write result if success and nothing to write true * not false * @throws IOException * exceptions when write messages * @author liuzhicheng */ public boolean writeMessages(CommunicatorInterface communicator, GraphDataInterface graphData, Path writePath, BSPJob job, GraphStaffHandler graphStaffHandler, Staff staff, ConcurrentLinkedQueue<String> messagesQueue) throws IOException, InterruptedException { // if (communicator.getIncomedQueuesSize() == 0) { // LOG.info("Feng test! writeMessages "+communicator.getIncomedQueuesSize()); // return true; // } // LOG.info("The init write path is : " + writePath.toString()); // BSPHdfs HDFSCheckpoint = new BSPHdfsImpl(); // BSPoutHdfs OUT = new BSPoutHdfsImpl(); // OUT.fsDataOutputStream(writePath, HDFSCheckpoint.getConf()); // StringBuffer sb = new StringBuffer(); //boolean writeResult = false; try { //Staff staff = graphStaffHandler; // OutputFormat outputformat = (OutputFormat) ReflectionUtils.newInstance( // job.getConf().getClass(Constants.USER_BC_BSP_JOB_OUTPUT_FORMAT_CLASS, // OutputFormat.class), job.getConf()); // outputformat.initialize(job.getConf()); // RecordWriter output = outputformat.getRecordWriter(job, // staff.getStaffAttemptId(), writePath); LOG.info("The init write path is : " + writePath.toString()); BSPHdfs HDFSCheckpoint = new BSPHdfsImpl(); BSPoutHdfs OUT = new BSPoutHdfsImpl(); OUT.fsDataOutputStream(writePath, HDFSCheckpoint.getConf()); StringBuffer sb = new StringBuffer(); ConcurrentLinkedQueue<String> messages = messagesQueue; Iterator<String> it = messages.iterator(); // if(messages.size()!=0){ // LOG.info("Feng test writeMessages messages "+it.next()); // //continue; // } while (it.hasNext()) { String message = it.next(); LOG.info("Write messages " + message); sb.append(message); sb.append("\n"); } // Iterator<IMessage> messagesIter = messages.iterator(); // StringBuffer sb = new StringBuffer(); // //sb.append(vertexID + Constants.MESSAGE_SPLIT); // while (messagesIter.hasNext()) { // IMessage msg = messagesIter.next(); // String info = msg.intoString(); // if (info != null) { // sb.append(info + Constants.SPACE_SPLIT_FLAG); // } // } // if (sb.length() > 0) { // int k = sb.length(); // sb.delete(k - 1, k - 1); // } // //sb.append("\n"); // output.write(new Text(v.getVertexID() + Constants.MESSAGE_SPLIT), new Text(sb.toString())); // LOG.info("Feng test! message record"+sb.toString()); // // graphData.getAllVertex(graphStaffHandler,communicator,output); OUT.writeBytes(sb.toString()); OUT.flush(); OUT.close(); // return true; } catch (IOException e) { LOG.error("Exception has happened and been catched!", e); return false; } return true; // //Vertex<?, ?, Edge> vertex = graphData.getForAll(i); // String vertexID = vertex.getVertexID().toString(); // Iterator<IMessage> it = communicator.getMessageIterator(vertexID); // sb.append(vertexID + Constants.MESSAGE_SPLIT); // while (it.hasNext()) { // IMessage msg = it.next(); // String info = msg.intoString(); // if (info != null) { // sb.append(info + Constants.SPACE_SPLIT_FLAG); // } // } // sb.append("\n"); // } // OUT.writeBytes(sb.toString()); // OUT.flush(); // OUT.close() }
From source file:com.chinamobile.bcbsp.fault.storage.Checkpoint.java
License:Apache License
/** * read message information//from ww w. j a v a 2 s. c o m * @param readPath * message backup to read * @param job * need to read the message backup * @param staff * read the message backup * @return read successfully true * not false. */ public Map<String, LinkedList<IMessage>> readMessages(Path readPath, BSPJob job, Staff staff) { LOG.info("The init read path is : " + readPath.toString()); String uri = readPath.toString(); InputStream in = null; //boolean exist = false; BufferedReader bis = null; Map<String, LinkedList<IMessage>> incomedMessages = new HashMap<String, LinkedList<IMessage>>(); CommunicationFactory.setMessageClass(job.getMessageClass()); try { BSPHdfs BSPRead = new BSPHdfsImpl(); //exist = BSPRead.exists(BSPRead.newPath(uri)); in = BSPRead.hdfsCheckpoint(uri, BSPRead.getConf()); bis = new BufferedReader(new InputStreamReader(new BufferedInputStream(in))); String s = bis.readLine(); if (s == null) { LOG.info("message file is empty! "); } while (s != null) { try { String[] msgInfos = s.split(Constants.MESSAGE_SPLIT); if (msgInfos.length > 0) { LOG.info("message is not empty!" + msgInfos.length); } String vertexID = msgInfos[0]; String[] msgs = msgInfos[1].split(Constants.SPACE_SPLIT_FLAG); LinkedList<IMessage> list = new LinkedList<IMessage>(); for (int i = 0; i < msgs.length; i++) { String[] msgss = msgs[i].split(Constants.SPLIT_FLAG); IMessage msg = CommunicationFactory.createPagerankMessage(); if (msgss.length > 0) { msg.setMessageId(Integer.parseInt(msgss[0])); msg.setContent(Float.parseFloat(msgss[1])); // LOG.info("Read every message in checkpoint! vertexID "+ // msgss[0]+"vertexValue"+msgss[1]); } list.add(msg); } incomedMessages.put(vertexID, list); } catch (Exception e) { //LOG.error("[Checkpoint] caught: ", e); throw new RuntimeException("[Checkpoint] caught: ", e); } s = bis.readLine(); } bis.close(); } catch (IOException e) { //LOG.error("Exception has happened and been catched!", e); throw new RuntimeException("Exception has happened and been catched!", e); } return incomedMessages; }
From source file:com.chinamobile.bcbsp.io.BSPFileInputFormat.java
License:Apache License
/** * Add a {@link Path} to the list of inputs for the BC_BSP job. * * @param job// www. ja v a2s.c o m * the current job BSPJob. * @param path * {@link Path} to be added to the list of inputs for the BC_BSP job. */ public static void addInputPath(BSPJob job, Path path) throws IOException { Configuration conf = job.getConf(); FileSystem fs = FileSystem.get(conf); path = path.makeQualified(fs); String dirStr = StringUtils.escapeString(path.toString()); String dirs = conf.get(Constants.USER_BC_BSP_JOB_INPUT_DIR); conf.set(Constants.USER_BC_BSP_JOB_INPUT_DIR, dirs == null ? dirStr : dirs + "," + dirStr); }
From source file:com.chinamobile.bcbsp.io.BSPFileOutputFormat.java
License:Apache License
/** * Set the {@link Path} of the output directory for the BC-BSP job. * * @param job//from ww w . j av a 2s. co m * the current BSPJob job. * @param outputDir * the {@link Path} of the output directory for the BC-BSP job. */ public static void setOutputPath(BSPJob job, Path outputDir) { Configuration conf = job.getConf(); checkOutputSpecs(job, outputDir); conf.set(Constants.USER_BC_BSP_JOB_OUTPUT_DIR, outputDir.toString()); }
From source file:com.chinamobile.bcbsp.io.BSPFileOutputFormat.java
License:Apache License
/** * Get the {@link Path} to the output directory for the BC-BSP job. * * @param staffId//from w ww.j a v a 2s.c o m * id of this staff attempt * @param writePath * the {@link Path} of the checkpoint directory for the BC-BSP job. * @return the checkpoint directory name for the BC-BSP job. */ public static Path getOutputPath(StaffAttemptID staffId, Path writePath) { String name = writePath.toString() + "/" + staffId.toString() + "/checkpoint.cp"; return name == null ? null : new Path(name); }