List of usage examples for org.apache.hadoop.fs FileStatus getModificationTime
public long getModificationTime()
From source file:edu.uci.ics.asterix.aoya.AsterixYARNClient.java
License:Apache License
/** * Finds the minimal classes and JARs needed to start the AM only. * @return Resources the AM needs to start on the initial container. * @throws IllegalStateException/*from ww w . jav a 2 s.c om*/ * @throws IOException */ private List<DFSResourceCoordinate> installAmLibs() throws IllegalStateException, IOException { List<DFSResourceCoordinate> resources = new ArrayList<DFSResourceCoordinate>(2); FileSystem fs = FileSystem.get(conf); String fullLibPath = CONF_DIR_REL + instanceFolder + "am_jars" + Path.SEPARATOR; String[] cp = System.getProperty("java.class.path").split(System.getProperty("path.separator")); String asterixJarPattern = "^(asterix).*(jar)$"; //starts with asterix,ends with jar String commonsJarPattern = "^(commons).*(jar)$"; String surefireJarPattern = "^(surefire).*(jar)$"; //for maven tests String jUnitTestPattern = "^(asterix-yarn" + File.separator + "target)$"; LOG.info(File.separator); for (String j : cp) { String[] pathComponents = j.split(Pattern.quote(File.separator)); LOG.info(j); LOG.info(pathComponents[pathComponents.length - 1]); if (pathComponents[pathComponents.length - 1].matches(asterixJarPattern) || pathComponents[pathComponents.length - 1].matches(commonsJarPattern) || pathComponents[pathComponents.length - 1].matches(surefireJarPattern) || pathComponents[pathComponents.length - 1].matches(jUnitTestPattern)) { LOG.info("Loading JAR/classpath: " + j); File f = new File(j); Path dst = new Path(fs.getHomeDirectory(), fullLibPath + f.getName()); if (!fs.exists(dst) || refresh) { fs.copyFromLocalFile(false, true, new Path(f.getAbsolutePath()), dst); } FileStatus dstSt = fs.getFileStatus(dst); LocalResource amLib = Records.newRecord(LocalResource.class); amLib.setType(LocalResourceType.FILE); amLib.setVisibility(LocalResourceVisibility.PRIVATE); amLib.setResource(ConverterUtils.getYarnUrlFromPath(dst)); amLib.setTimestamp(dstSt.getModificationTime()); amLib.setSize(dstSt.getLen()); DFSResourceCoordinate amLibCoord = new DFSResourceCoordinate(); amLibCoord.res = amLib; amLibCoord.name = f.getName(); if (f.getName().contains("asterix-yarn") || f.getName().contains("surefire")) { amLibCoord.envs.put(dst.toUri().toString(), AConstants.APPLICATIONMASTERJARLOCATION); amLibCoord.envs.put(Long.toString(dstSt.getLen()), AConstants.APPLICATIONMASTERJARLEN); amLibCoord.envs.put(Long.toString(dstSt.getModificationTime()), AConstants.APPLICATIONMASTERJARTIMESTAMP); } resources.add(amLibCoord); } } if (resources.size() == 0) { throw new IOException("Required JARs are missing. Please check your directory structure"); } return resources; }
From source file:edu.uci.ics.asterix.aoya.AsterixYARNClient.java
License:Apache License
/** * Uploads binary resources to HDFS for use by the AM * @return//ww w . ja v a 2 s . co m * @throws IOException * @throws YarnException */ public List<DFSResourceCoordinate> distributeBinaries() throws IOException, YarnException { List<DFSResourceCoordinate> resources = new ArrayList<DFSResourceCoordinate>(2); // Copy the application master jar to the filesystem // Create a local resource to point to the destination jar path FileSystem fs = FileSystem.get(conf); Path src, dst; FileStatus destStatus; String pathSuffix; // adding info so we can add the jar to the App master container path // Add the asterix tarfile to HDFS for easy distribution // Keep it all archived for now so add it as a file... pathSuffix = CONF_DIR_REL + instanceFolder + "asterix-server.zip"; dst = new Path(fs.getHomeDirectory(), pathSuffix); if (refresh) { if (fs.exists(dst)) { fs.delete(dst, false); } } if (!fs.exists(dst)) { src = new Path(asterixZip); LOG.info("Copying Asterix distributable to DFS"); fs.copyFromLocalFile(false, true, src, dst); } destStatus = fs.getFileStatus(dst); LocalResource asterixTarLoc = Records.newRecord(LocalResource.class); asterixTarLoc.setType(LocalResourceType.ARCHIVE); asterixTarLoc.setVisibility(LocalResourceVisibility.PRIVATE); asterixTarLoc.setResource(ConverterUtils.getYarnUrlFromPath(dst)); asterixTarLoc.setTimestamp(destStatus.getModificationTime()); // adding info so we can add the tarball to the App master container path DFSResourceCoordinate tar = new DFSResourceCoordinate(); tar.envs.put(dst.toUri().toString(), AConstants.TARLOCATION); tar.envs.put(Long.toString(asterixTarLoc.getSize()), AConstants.TARLEN); tar.envs.put(Long.toString(asterixTarLoc.getTimestamp()), AConstants.TARTIMESTAMP); tar.res = asterixTarLoc; tar.name = "asterix-server.zip"; resources.add(tar); // Set the log4j properties if needed if (!log4jPropFile.isEmpty()) { Path log4jSrc = new Path(log4jPropFile); Path log4jDst = new Path(fs.getHomeDirectory(), "log4j.props"); fs.copyFromLocalFile(false, true, log4jSrc, log4jDst); FileStatus log4jFileStatus = fs.getFileStatus(log4jDst); LocalResource log4jRsrc = Records.newRecord(LocalResource.class); log4jRsrc.setType(LocalResourceType.FILE); log4jRsrc.setVisibility(LocalResourceVisibility.PRIVATE); log4jRsrc.setResource(ConverterUtils.getYarnUrlFromURI(log4jDst.toUri())); log4jRsrc.setTimestamp(log4jFileStatus.getModificationTime()); log4jRsrc.setSize(log4jFileStatus.getLen()); DFSResourceCoordinate l4j = new DFSResourceCoordinate(); tar.res = log4jRsrc; tar.name = "log4j.properties"; resources.add(l4j); } resources.addAll(installAmLibs()); return resources; }
From source file:edu.uci.ics.asterix.external.adapter.factory.HDFSAdapterFactory.java
License:Apache License
/** * Instead of creating the split using the input format, we do it manually * This function returns fileSplits (1 per hdfs file block) irrespective of the number of partitions * and the produced splits only cover intersection between current files in hdfs and files stored internally * in AsterixDB/*from ww w. j av a2 s. c o m*/ * 1. NoOp means appended file * 2. AddOp means new file * 3. UpdateOp means the delta of a file * * @return * @throws IOException */ protected InputSplit[] getSplits(JobConf conf) throws IOException { // Create file system object FileSystem fs = FileSystem.get(conf); ArrayList<FileSplit> fileSplits = new ArrayList<FileSplit>(); ArrayList<ExternalFile> orderedExternalFiles = new ArrayList<ExternalFile>(); // Create files splits for (ExternalFile file : files) { Path filePath = new Path(file.getFileName()); FileStatus fileStatus; try { fileStatus = fs.getFileStatus(filePath); } catch (FileNotFoundException e) { // file was deleted at some point, skip to next file continue; } if (file.getPendingOp() == ExternalFilePendingOp.PENDING_ADD_OP && fileStatus.getModificationTime() == file.getLastModefiedTime().getTime()) { // Get its information from HDFS name node BlockLocation[] fileBlocks = fs.getFileBlockLocations(fileStatus, 0, file.getSize()); // Create a split per block for (BlockLocation block : fileBlocks) { if (block.getOffset() < file.getSize()) { fileSplits.add(new FileSplit(filePath, block.getOffset(), (block.getLength() + block.getOffset()) < file.getSize() ? block.getLength() : (file.getSize() - block.getOffset()), block.getHosts())); orderedExternalFiles.add(file); } } } else if (file.getPendingOp() == ExternalFilePendingOp.PENDING_NO_OP && fileStatus.getModificationTime() == file.getLastModefiedTime().getTime()) { long oldSize = 0L; long newSize = file.getSize(); for (int i = 0; i < files.size(); i++) { if (files.get(i).getFileName() == file.getFileName() && files.get(i).getSize() != file.getSize()) { newSize = files.get(i).getSize(); oldSize = file.getSize(); break; } } // Get its information from HDFS name node BlockLocation[] fileBlocks = fs.getFileBlockLocations(fileStatus, 0, newSize); // Create a split per block for (BlockLocation block : fileBlocks) { if (block.getOffset() + block.getLength() > oldSize) { if (block.getOffset() < newSize) { // Block interact with delta -> Create a split long startCut = (block.getOffset() > oldSize) ? 0L : oldSize - block.getOffset(); long endCut = (block.getOffset() + block.getLength() < newSize) ? 0L : block.getOffset() + block.getLength() - newSize; long splitLength = block.getLength() - startCut - endCut; fileSplits.add(new FileSplit(filePath, block.getOffset() + startCut, splitLength, block.getHosts())); orderedExternalFiles.add(file); } } } } } fs.close(); files = orderedExternalFiles; return fileSplits.toArray(new FileSplit[fileSplits.size()]); }
From source file:edu.uci.ics.asterix.external.indexing.input.AbstractHDFSLookupInputStream.java
License:Apache License
public boolean fetchRecord(int fileNumber, long recordOffset) throws Exception { if (fileNumber != this.fileNumber) { // New file number this.fileNumber = fileNumber; filesIndexAccessor.searchForFile(fileNumber, file); try {//from ww w .ja v a 2s . co m FileStatus fileStatus = fs.getFileStatus(new Path(file.getFileName())); if (fileStatus.getModificationTime() != file.getLastModefiedTime().getTime()) { this.fileNumber = fileNumber; skipFile = true; return false; } else { this.fileNumber = fileNumber; skipFile = false; openFile(file.getFileName()); } } catch (FileNotFoundException e) { // We ignore File not found exceptions <- it means file was deleted and so we don't care about it anymore -> this.fileNumber = fileNumber; skipFile = true; return false; } } else if (skipFile) { return false; } return read(recordOffset); }
From source file:edu.uci.ics.asterix.external.indexing.input.GenericFileAwareRecordReader.java
License:Apache License
private boolean moveToNext() throws IOException { for (; currentSplitIndex < inputSplits.length; currentSplitIndex++) { /**/*from ww w. j av a 2 s .c om*/ * read all the partitions scheduled to the current node */ if (readSchedule[currentSplitIndex].equals(nodeName)) { /** * pick an unread split to read synchronize among * simultaneous partitions in the same machine */ synchronized (executed) { if (executed[currentSplitIndex] == false) { executed[currentSplitIndex] = true; } else { continue; } } /** * read the split */ try { String fileName = ((FileSplit) (inputSplits[currentSplitIndex])).getPath().toUri().getPath(); FileStatus fileStatus = hadoopFS.getFileStatus(new Path(fileName)); //skip if not the same file stored in the files snapshot if (fileStatus.getModificationTime() != files.get(currentSplitIndex).getLastModefiedTime() .getTime()) continue; reader = getRecordReader(currentSplitIndex); } catch (Exception e) { continue; } key = reader.createKey(); value = reader.createValue(); return true; } } return false; }
From source file:edu.uci.ics.asterix.external.indexing.input.RCFileDataReader.java
License:Apache License
private boolean moveToNext() throws IOException { for (; currentSplitIndex < inputSplits.length; currentSplitIndex++) { /**/*ww w . j ava 2 s . c o m*/ * read all the partitions scheduled to the current node */ if (readSchedule[currentSplitIndex].equals(nodeName)) { /** * pick an unread split to read synchronize among * simultaneous partitions in the same machine */ synchronized (executed) { if (executed[currentSplitIndex] == false) { executed[currentSplitIndex] = true; } else { continue; } } /** * read the split */ try { if (files != null) { fileName = ((FileSplit) (inputSplits[currentSplitIndex])).getPath().toUri().getPath(); FileStatus fileStatus = hadoopFS.getFileStatus(new Path(fileName)); //skip if not the same file stored in the files snapshot if (fileStatus.getModificationTime() != files.get(currentSplitIndex).getLastModefiedTime() .getTime()) continue; } reader = getRecordReader(currentSplitIndex); recordGroupOffset = -1; nextRecordGroupOffset = reader.getPos(); } catch (Exception e) { continue; } key = reader.createKey(); value = reader.createValue(); return true; } } return false; }
From source file:edu.uci.ics.asterix.external.indexing.input.RCFileLookupReader.java
License:Apache License
public Writable read(int fileNumber, long recordGroupOffset, int rowNumber) throws Exception { if (fileNumber != this.fileNumber) { filesIndexAccessor.searchForFile(fileNumber, currentFile); try {/*from ww w . j ava 2s . c o m*/ FileStatus fileStatus = fs.getFileStatus(new Path(currentFile.getFileName())); if (fileStatus.getModificationTime() != currentFile.getLastModefiedTime().getTime()) { this.fileNumber = fileNumber; skipFile = true; return null; } else { this.fileNumber = fileNumber; skipFile = false; } } catch (FileNotFoundException e) { // Couldn't find file, skip it this.fileNumber = fileNumber; skipFile = true; return null; } // Close old file and open new one if (reader != null) reader.close(); reader = new Reader(fs, new Path(currentFile.getFileName()), conf); this.recordGroupOffset = -1; this.rowNumber = -1; } else if (skipFile) { return null; } // Seek to the record group if needed if (recordGroupOffset != this.recordGroupOffset) { this.recordGroupOffset = recordGroupOffset; if (reader.getPosition() != recordGroupOffset) reader.seek(recordGroupOffset); reader.resetBuffer(); this.rowNumber = -1; } // skip rows to the record row while (this.rowNumber < rowNumber) { reader.next(rcKey); reader.getCurrentRow(rcValue); this.rowNumber++; } return rcValue; }
From source file:edu.uci.ics.asterix.external.indexing.input.SequenceFileLookupReader.java
License:Apache License
@Override public Writable read(int fileNumber, long recordOffset) throws Exception { if (fileNumber != this.fileNumber) { //get file name this.fileNumber = fileNumber; filesIndexAccessor.searchForFile(fileNumber, file); try {//w w w .j ava 2s. c om FileStatus fileStatus = fs.getFileStatus(new Path(file.getFileName())); if (fileStatus.getModificationTime() != file.getLastModefiedTime().getTime()) { this.fileNumber = fileNumber; skipFile = true; return null; } else { this.fileNumber = fileNumber; skipFile = false; openFile(file.getFileName()); } } catch (FileNotFoundException e) { // file was not found, do nothing and skip its tuples this.fileNumber = fileNumber; skipFile = true; return null; } } else if (skipFile) { return null; } reader.seek(recordOffset); reader.next(key, value); return value; }
From source file:edu.uci.ics.asterix.external.indexing.input.TextFileLookupReader.java
License:Apache License
@SuppressWarnings("deprecation") @Override//from w w w . java 2 s .c o m public String read(int fileNumber, long recordOffset) throws Exception { if (fileNumber != this.fileNumber) { this.fileNumber = fileNumber; filesIndexAccessor.searchForFile(fileNumber, file); try { FileStatus fileStatus = fs.getFileStatus(new Path(file.getFileName())); if (fileStatus.getModificationTime() != file.getLastModefiedTime().getTime()) { this.fileNumber = fileNumber; skipFile = true; return null; } else { this.fileNumber = fileNumber; skipFile = false; openFile(file.getFileName()); } } catch (FileNotFoundException e) { // File is not there, skip it and do nothing this.fileNumber = fileNumber; skipFile = true; return null; } } else if (skipFile) { return null; } reader.seek(recordOffset); return reader.readLine(); }
From source file:edu.uci.ics.asterix.external.indexing.input.TextualDataReader.java
License:Apache License
@SuppressWarnings("unchecked") private boolean moveToNext() throws IOException { for (; currentSplitIndex < inputSplits.length; currentSplitIndex++) { /**/*from w ww. j a va 2 s. c o m*/ * read all the partitions scheduled to the current node */ if (readSchedule[currentSplitIndex].equals(nodeName)) { /** * pick an unread split to read synchronize among * simultaneous partitions in the same machine */ synchronized (executed) { if (executed[currentSplitIndex] == false) { executed[currentSplitIndex] = true; } else { continue; } } /** * read the split */ try { if (files != null) { fileName = ((FileSplit) (inputSplits[currentSplitIndex])).getPath().toUri().getPath(); FileStatus fileStatus = hadoopFS.getFileStatus(new Path(fileName)); // Skip if not the same file stored in the files snapshot if (fileStatus.getModificationTime() != files.get(currentSplitIndex).getLastModefiedTime() .getTime()) continue; } // It is the same file reader = getRecordReader(currentSplitIndex); } catch (Exception e) { // ignore exceptions <-- This might change later --> continue; } key = reader.createKey(); value = (Text) reader.createValue(); return true; } } return false; }