List of usage examples for org.apache.hadoop.mapred InvalidJobConfException InvalidJobConfException
public InvalidJobConfException(Throwable t)
From source file:org.apache.hadoop.examples.terasort.TeraOutputFormat.java
License:Apache License
@Override public void checkOutputSpecs(JobContext job) throws InvalidJobConfException, IOException { // Ensure that the output directory is set Path outDir = getOutputPath(job); if (outDir == null) { throw new InvalidJobConfException("Output directory not set in JobConf."); }//from w w w .jav a2 s . com final Configuration jobConf = job.getConfiguration(); // get delegation token for outDir's file system TokenCache.obtainTokensForNamenodes(job.getCredentials(), new Path[] { outDir }, jobConf); final FileSystem fs = outDir.getFileSystem(jobConf); try { // existing output dir is considered empty iff its only content is the // partition file. // final FileStatus[] outDirKids = fs.listStatus(outDir); boolean empty = false; if (outDirKids != null && outDirKids.length == 1) { final FileStatus st = outDirKids[0]; final String fname = st.getPath().getName(); empty = !st.isDirectory() && TeraInputFormat.PARTITION_FILENAME.equals(fname); } if (TeraSort.getUseSimplePartitioner(job) || !empty) { throw new FileAlreadyExistsException("Output directory " + outDir + " already exists"); } } catch (FileNotFoundException ignored) { } }
From source file:org.apache.hama.bsp.FileOutputFormat.java
License:Apache License
@Override public void checkOutputSpecs(FileSystem ignored, BSPJob job) throws FileAlreadyExistsException, InvalidJobConfException, IOException { // Ensure that the output directory is set and not already there Path outDir = getOutputPath(job); if (outDir == null && job.getNumBspTask() != 0) { throw new InvalidJobConfException("Output directory not set in JobConf."); }//from ww w.j a v a 2s.c o m if (outDir != null) { FileSystem fs = outDir.getFileSystem(job.getConfiguration()); // normalize the output directory outDir = fs.makeQualified(outDir); setOutputPath(job, outDir); // check its existence if (fs.exists(outDir)) { throw new FileAlreadyExistsException("Output directory " + outDir + " already exists"); } } }
From source file:org.apache.nutch.fetcher.FetcherOutputFormat.java
License:Apache License
public void checkOutputSpecs(FileSystem fs, JobConf job) throws IOException { Path out = FileOutputFormat.getOutputPath(job); if ((out == null) && (job.getNumReduceTasks() != 0)) { throw new InvalidJobConfException("Output directory not set in JobConf."); }/* w w w. j a va2 s . c o m*/ if (fs == null) { fs = out.getFileSystem(job); } if (fs.exists(new Path(out, CrawlDatum.FETCH_DIR_NAME))) throw new IOException("Segment already fetched!"); }
From source file:org.apache.nutch.parse.ParseOutputFormat.java
License:Apache License
public void checkOutputSpecs(FileSystem fs, JobConf job) throws IOException { Path out = FileOutputFormat.getOutputPath(job); if ((out == null) && (job.getNumReduceTasks() != 0)) { throw new InvalidJobConfException("Output directory not set in JobConf."); }/*from www . j a va 2s . c o m*/ if (fs == null) { fs = out.getFileSystem(job); } if (fs.exists(new Path(out, CrawlDatum.PARSE_DIR_NAME))) throw new IOException("Segment already parsed!"); }
From source file:org.goldenorb.util.ResourceAllocator.java
License:Apache License
/** * Assigns resources.//from w w w. j av a 2 s . c o m * * @returns Map<M,Integer[]> */ public Map<M, Integer[]> assignResources(OrbConfiguration jobConf) throws InvalidJobConfException { logger.info("ResourceAllocator: assignResources()"); int requestedPartitions = jobConf.getOrbRequestedPartitions(); // if no partitions are requested, the job is invalid if (requestedPartitions <= 0) { logger.error("missing number of requested partitions for job"); throw new InvalidJobConfException("missing number of requested partitions for job"); } int reservedPartitions = jobConf.getOrbReservedPartitions(); // if this is zero, it screws up the resource allocation so we set it to Int max to move through // the allocation process int partitionsPerMachine = (conf.getNumberOfPartitionsPerMachine() == 0 ? Integer.MAX_VALUE : conf.getNumberOfPartitionsPerMachine()); logger.debug("requestedPartitions: {}", requestedPartitions); logger.debug("reservedPartitions: {}", reservedPartitions); logger.debug("partitionsPerMachine: {}", partitionsPerMachine); // some setup, organize the trackers and capture availability/reserved info sortTrackers(); updateOrbTrackerAvailability(); // if you request more than what's available, you get nothing (caller should wait and try again) if (requestedPartitions > totalAvailable || reservedPartitions > totalReserved) { return null; } // requestedPartitions functions as the countdown mechanism while (requestedPartitions > 0) { // until all partitions are assigned, iterate through trackers for (Iterator<M> iter = trackersByAvailable.iterator(); iter.hasNext() && requestedPartitions > 0;) { M tracker = iter.next(); // tracker has > partitionsPerMachine available if (partitionsPerMachine <= orbTrackerAvailability.get(tracker)[TRACKER_AVAILABLE] && requestedPartitions >= partitionsPerMachine) { assignAvailable(tracker, partitionsPerMachine, TRACKER_AVAILABLE); requestedPartitions -= partitionsPerMachine; decrementAvailable(tracker, partitionsPerMachine, TRACKER_AVAILABLE); } // tracker has at least one available else if (orbTrackerAvailability.get(tracker)[TRACKER_AVAILABLE] > 0) { assignAvailable(tracker, 1, TRACKER_AVAILABLE); requestedPartitions -= 1; decrementAvailable(tracker, 1, TRACKER_AVAILABLE); } // tracker is full, skip it else { continue; } } } // requestedPartitions functions as the countdown mechanism while (reservedPartitions > 0) { // until all partitions are assigned, iterate through trackers for (Iterator<M> iter = trackersByReserved.iterator(); iter.hasNext() && reservedPartitions > 0;) { M tracker = iter.next(); // tracker has at least one available if (orbTrackerAvailability.get(tracker)[TRACKER_RESERVED] > 0) { assignAvailable(tracker, 1, TRACKER_RESERVED); reservedPartitions -= 1; decrementAvailable(tracker, 1, TRACKER_RESERVED); } // tracker is full, skip it else { continue; } } } return orbTrackerAssignments; }
From source file:terasort.io.TeraOutputFormat.java
License:Apache License
@Override public void checkOutputSpecs(JobContext job) throws IOException { // Ensure that the output directory is set Path outDir = getOutputPath(job); if (outDir == null) { throw new InvalidJobConfException("Output directory not set in JobConf."); }/*w ww. ja v a2 s.com*/ final Configuration jobConf = job.getConfiguration(); // get delegation token for outDir's file system TokenCache.obtainTokensForNamenodes(job.getCredentials(), new Path[] { outDir }, jobConf); final FileSystem fs = outDir.getFileSystem(jobConf); if (fs.exists(outDir)) { // existing output dir is considered empty iff its only content is the // partition file. // final FileStatus[] outDirKids = fs.listStatus(outDir); boolean empty = false; if (outDirKids != null && outDirKids.length == 1) { final FileStatus st = outDirKids[0]; final String fname = st.getPath().getName(); empty = !st.isDirectory(); } if (!empty) { throw new FileAlreadyExistsException("Output directory " + outDir + " already exists"); } } }