List of usage examples for org.apache.hadoop.fs FileSystem rename
public abstract boolean rename(Path src, Path dst) throws IOException;
From source file:org.apache.sysml.runtime.controlprogram.parfor.ResultMergeLocalFile.java
License:Apache License
private static void copyAllFiles(String fnameNew, ArrayList<MatrixObject> inMO) throws CacheException, IOException { JobConf job = new JobConf(ConfigurationManager.getCachedJobConf()); Path path = new Path(fnameNew); FileSystem fs = IOUtilFunctions.getFileSystem(path, job); //create output dir fs.mkdirs(path);/*w ww.j ava 2s .c o m*/ //merge in all input matrix objects IDSequence seq = new IDSequence(); for (MatrixObject in : inMO) { if (LOG.isTraceEnabled()) LOG.trace("ResultMerge (local, file): Merge input " + in.hashCode() + " (fname=" + in.getFileName() + ") via file rename."); //copy over files (just rename file or entire dir) Path tmpPath = new Path(in.getFileName()); String lname = tmpPath.getName(); fs.rename(tmpPath, new Path(fnameNew + "/" + lname + seq.getNextID())); } }
From source file:org.apache.sysml.runtime.io.WriterTextCSV.java
License:Apache License
@SuppressWarnings("unchecked") public final void addHeaderToCSV(String srcFileName, String destFileName, long rlen, long clen) throws IOException { Configuration conf = new Configuration(ConfigurationManager.getCachedJobConf()); Path srcFilePath = new Path(srcFileName); Path destFilePath = new Path(destFileName); FileSystem fs = IOUtilFunctions.getFileSystem(srcFilePath, conf); if (!_props.hasHeader()) { // simply move srcFile to destFile /*/*from ww w . j av a 2 s . com*/ * TODO: Remove this roundabout way! * For example: destFilePath = /user/biadmin/csv/temp/out/file.csv * & the only path that exists already on HDFS is /user/biadmin/csv/. * In this case: the directory structure /user/biadmin/csv/temp/out must be created. * Simple hdfs.rename() does not seem to create this directory structure. */ // delete the destination file, if exists already fs.delete(destFilePath, true); // Create /user/biadmin/csv/temp/out/file.csv so that ..../temp/out/ is created. fs.createNewFile(destFilePath); // delete the file "file.csv" but preserve the directory structure /user/biadmin/csv/temp/out/ fs.delete(destFilePath, true); // finally, move the data to destFilePath = /user/biadmin/csv/temp/out/file.csv fs.rename(srcFilePath, destFilePath); return; } // construct the header line StringBuilder sb = new StringBuilder(); for (int i = 0; i < clen; i++) { sb.append("C" + (i + 1)); if (i < clen - 1) sb.append(_props.getDelim()); } sb.append('\n'); if (fs.isDirectory(srcFilePath)) { // compute sorted order among part files ArrayList<Path> files = new ArrayList<>(); for (FileStatus stat : fs.listStatus(srcFilePath, CSVReblockMR.hiddenFileFilter)) files.add(stat.getPath()); Collections.sort(files); // first part file path Path firstpart = files.get(0); // create a temp file, and add header and contents of first part Path tmp = new Path(firstpart.toString() + ".tmp"); OutputStream out = fs.create(tmp, true); out.write(sb.toString().getBytes()); sb.setLength(0); // copy rest of the data from firstpart InputStream in = null; try { in = fs.open(firstpart); IOUtils.copyBytes(in, out, conf, true); } finally { IOUtilFunctions.closeSilently(in); IOUtilFunctions.closeSilently(out); } // rename tmp to firstpart fs.delete(firstpart, true); fs.rename(tmp, firstpart); // rename srcfile to destFile fs.delete(destFilePath, true); fs.createNewFile(destFilePath); // force the creation of directory structure fs.delete(destFilePath, true); // delete the file, but preserve the directory structure fs.rename(srcFilePath, destFilePath); // move the data } else if (fs.isFile(srcFilePath)) { // create destination file OutputStream out = fs.create(destFilePath, true); // write header out.write(sb.toString().getBytes()); sb.setLength(0); // copy the data from srcFile InputStream in = null; try { in = fs.open(srcFilePath); IOUtils.copyBytes(in, out, conf, true); } finally { IOUtilFunctions.closeSilently(in); IOUtilFunctions.closeSilently(out); } } else { throw new IOException(srcFilePath.toString() + ": No such file or directory"); } }
From source file:org.apache.sysml.runtime.matrix.data.MultipleOutputCommitter.java
License:Apache License
private void moveFileToDestination(TaskAttemptContext context, FileSystem fs, Path file) throws IOException { TaskAttemptID attemptId = context.getTaskAttemptID(); // get output index and final destination String name = file.getName(); //e.g., 0-r-00000 int index = Integer.parseInt(name.substring(0, name.indexOf("-"))); Path dest = new Path(outputs[index], name); //e.g., outX/0-r-00000 // move file from 'file' to 'finalPath' if (!fs.rename(file, dest)) { if (!fs.delete(dest, true)) throw new IOException("Failed to delete earlier output " + dest + " for rename of " + file + " in task " + attemptId); if (!fs.rename(file, dest)) throw new IOException( "Failed to save output " + dest + " for rename of " + file + " in task: " + attemptId); }//from w w w . j av a 2 s . c o m }
From source file:org.apache.sysml.runtime.util.MapReduceTool.java
License:Apache License
public static void renameFileOnHDFS(String originalDir, String newDir) throws IOException { Path pathOrig = new Path(originalDir); Path pathNew = new Path(newDir); if (!IOUtilFunctions.isSameFileScheme(pathOrig, pathNew)) throw new IOException("Cannot rename files to different target file system."); deleteFileIfExistOnHDFS(newDir);/*from w w w . ja v a 2 s . co m*/ FileSystem fs = IOUtilFunctions.getFileSystem(pathOrig); if (fs.exists(pathOrig)) fs.rename(pathOrig, pathNew); else throw new FileNotFoundException(originalDir); }
From source file:org.apache.tajo.master.exec.DDLExecutor.java
License:Apache License
/** * ALTER TABLE SET .../*w w w .j a va 2 s.co m*/ */ public void alterTable(TajoMaster.MasterContext context, final QueryContext queryContext, final AlterTableNode alterTable) throws IOException { final CatalogService catalog = context.getCatalog(); final String tableName = alterTable.getTableName(); String databaseName; String simpleTableName; if (CatalogUtil.isFQTableName(tableName)) { String[] split = CatalogUtil.splitFQTableName(tableName); databaseName = split[0]; simpleTableName = split[1]; } else { databaseName = queryContext.getCurrentDatabase(); simpleTableName = tableName; } final String qualifiedName = CatalogUtil.buildFQName(databaseName, simpleTableName); if (!catalog.existsTable(databaseName, simpleTableName)) { throw new NoSuchTableException(qualifiedName); } switch (alterTable.getAlterTableOpType()) { case RENAME_TABLE: if (!catalog.existsTable(databaseName, simpleTableName)) { throw new NoSuchTableException(alterTable.getTableName()); } if (catalog.existsTable(databaseName, alterTable.getNewTableName())) { throw new AlreadyExistsTableException(alterTable.getNewTableName()); } TableDesc desc = catalog.getTableDesc(databaseName, simpleTableName); if (!desc.isExternal()) { // if the table is the managed table Path oldPath = StorageUtil.concatPath(context.getConf().getVar(TajoConf.ConfVars.WAREHOUSE_DIR), databaseName, simpleTableName); Path newPath = StorageUtil.concatPath(context.getConf().getVar(TajoConf.ConfVars.WAREHOUSE_DIR), databaseName, alterTable.getNewTableName()); FileSystem fs = oldPath.getFileSystem(context.getConf()); if (!fs.exists(oldPath)) { throw new IOException("No such a table directory: " + oldPath); } if (fs.exists(newPath)) { throw new IOException("Already table directory exists: " + newPath); } fs.rename(oldPath, newPath); } catalog.alterTable(CatalogUtil.renameTable(qualifiedName, alterTable.getNewTableName(), AlterTableType.RENAME_TABLE)); break; case RENAME_COLUMN: if (existColumnName(qualifiedName, alterTable.getNewColumnName())) { throw new ColumnNameAlreadyExistException(alterTable.getNewColumnName()); } catalog.alterTable(CatalogUtil.renameColumn(qualifiedName, alterTable.getColumnName(), alterTable.getNewColumnName(), AlterTableType.RENAME_COLUMN)); break; case ADD_COLUMN: if (existColumnName(qualifiedName, alterTable.getAddNewColumn().getSimpleName())) { throw new ColumnNameAlreadyExistException(alterTable.getAddNewColumn().getSimpleName()); } catalog.alterTable(CatalogUtil.addNewColumn(qualifiedName, alterTable.getAddNewColumn(), AlterTableType.ADD_COLUMN)); break; default: //TODO } }
From source file:org.apache.tajo.master.exec.QueryExecutor.java
License:Apache License
private void insertNonFromQuery(QueryContext queryContext, InsertNode insertNode, SubmitQueryResponse.Builder responseBuilder) throws Exception { String nodeUniqName = insertNode.getTableName() == null ? insertNode.getPath().getName() : insertNode.getTableName(); String queryId = nodeUniqName + "_" + System.currentTimeMillis(); FileSystem fs = TajoConf.getWarehouseDir(context.getConf()).getFileSystem(context.getConf()); Path stagingDir = QueryMasterTask.initStagingDir(context.getConf(), queryId.toString(), queryContext); Path stagingResultDir = new Path(stagingDir, TajoConstants.RESULT_DIR_NAME); TableDesc tableDesc = null;/*from w ww . ja va 2 s. c o m*/ Path finalOutputDir = null; if (insertNode.getTableName() != null) { tableDesc = this.catalog.getTableDesc(insertNode.getTableName()); finalOutputDir = new Path(tableDesc.getPath()); } else { finalOutputDir = insertNode.getPath(); } TaskAttemptContext taskAttemptContext = new TaskAttemptContext(queryContext, null, null, null, stagingDir); taskAttemptContext.setOutputPath(new Path(stagingResultDir, "part-01-000000")); EvalExprExec evalExprExec = new EvalExprExec(taskAttemptContext, (EvalExprNode) insertNode.getChild()); StoreTableExec exec = new StoreTableExec(taskAttemptContext, insertNode, evalExprExec); try { exec.init(); exec.next(); } finally { exec.close(); } if (insertNode.isOverwrite()) { // INSERT OVERWRITE INTO // it moves the original table into the temporary location. // Then it moves the new result table into the original table location. // Upon failed, it recovers the original table if possible. boolean movedToOldTable = false; boolean committed = false; Path oldTableDir = new Path(stagingDir, TajoConstants.INSERT_OVERWIRTE_OLD_TABLE_NAME); try { if (fs.exists(finalOutputDir)) { fs.rename(finalOutputDir, oldTableDir); movedToOldTable = fs.exists(oldTableDir); } else { // if the parent does not exist, make its parent directory. fs.mkdirs(finalOutputDir.getParent()); } fs.rename(stagingResultDir, finalOutputDir); committed = fs.exists(finalOutputDir); } catch (IOException ioe) { // recover the old table if (movedToOldTable && !committed) { fs.rename(oldTableDir, finalOutputDir); } } } else { FileStatus[] files = fs.listStatus(stagingResultDir); for (FileStatus eachFile : files) { Path targetFilePath = new Path(finalOutputDir, eachFile.getPath().getName()); if (fs.exists(targetFilePath)) { targetFilePath = new Path(finalOutputDir, eachFile.getPath().getName() + "_" + System.currentTimeMillis()); } fs.rename(eachFile.getPath(), targetFilePath); } } if (insertNode.hasTargetTable()) { TableStats stats = tableDesc.getStats(); long volume = Query.getTableVolume(context.getConf(), finalOutputDir); stats.setNumBytes(volume); stats.setNumRows(1); CatalogProtos.UpdateTableStatsProto.Builder builder = CatalogProtos.UpdateTableStatsProto.newBuilder(); builder.setTableName(tableDesc.getName()); builder.setStats(stats.getProto()); catalog.updateTableStats(builder.build()); responseBuilder.setTableDesc(tableDesc.getProto()); } else { TableStats stats = new TableStats(); long volume = Query.getTableVolume(context.getConf(), finalOutputDir); stats.setNumBytes(volume); stats.setNumRows(1); // Empty TableDesc List<CatalogProtos.ColumnProto> columns = new ArrayList<CatalogProtos.ColumnProto>(); CatalogProtos.TableDescProto tableDescProto = CatalogProtos.TableDescProto.newBuilder() .setTableName(nodeUniqName) .setMeta( CatalogProtos.TableProto.newBuilder().setStoreType(CatalogProtos.StoreType.CSV).build()) .setSchema(CatalogProtos.SchemaProto.newBuilder().addAllFields(columns).build()) .setStats(stats.getProto()).build(); responseBuilder.setTableDesc(tableDescProto); } // If queryId == NULL_QUERY_ID and MaxRowNum == -1, TajoCli prints only number of inserted rows. responseBuilder.setMaxRowNum(-1); responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto()); responseBuilder.setResultCode(ClientProtos.ResultCode.OK); }
From source file:org.apache.tajo.storage.FileTablespace.java
License:Apache License
/** * Finalizes result data. Tajo stores result data in the staging directory. * If the query fails, clean up the staging directory. * Otherwise the query is successful, move to the final directory from the staging directory. * * @param queryContext The query property * @param changeFileSeq If true change result file name with max sequence. * @return Saved path/*from w w w . j a va2 s . co m*/ * @throws java.io.IOException */ protected Path commitOutputData(OverridableConf queryContext, boolean changeFileSeq) throws IOException { Path stagingDir = new Path(queryContext.get(QueryVars.STAGING_DIR)); Path stagingResultDir = new Path(stagingDir, TajoConstants.RESULT_DIR_NAME); Path finalOutputDir; if (!queryContext.get(QueryVars.OUTPUT_TABLE_URI, "").isEmpty()) { finalOutputDir = new Path(queryContext.get(QueryVars.OUTPUT_TABLE_URI)); try { FileSystem fs = stagingResultDir.getFileSystem(conf); if (queryContext.getBool(QueryVars.OUTPUT_OVERWRITE, false)) { // INSERT OVERWRITE INTO // It moves the original table into the temporary location. // Then it moves the new result table into the original table location. // Upon failed, it recovers the original table if possible. boolean movedToOldTable = false; boolean committed = false; Path oldTableDir = new Path(stagingDir, TajoConstants.INSERT_OVERWIRTE_OLD_TABLE_NAME); ContentSummary summary = fs.getContentSummary(stagingResultDir); // When inserting empty data into a partitioned table, check if keep existing data need to be remove or not. boolean overwriteEnabled = queryContext .getBool(SessionVars.PARTITION_NO_RESULT_OVERWRITE_ENABLED); // If existing data doesn't need to keep, check if there are some files. if ((!queryContext.get(QueryVars.OUTPUT_PARTITIONS, "").isEmpty()) && (!overwriteEnabled || (overwriteEnabled && summary.getFileCount() > 0L))) { // This is a map for existing non-leaf directory to rename. A key is current directory and a value is // renaming directory. Map<Path, Path> renameDirs = TUtil.newHashMap(); // This is a map for recovering existing partition directory. A key is current directory and a value is // temporary directory to back up. Map<Path, Path> recoveryDirs = TUtil.newHashMap(); try { if (!fs.exists(finalOutputDir)) { fs.mkdirs(finalOutputDir); } visitPartitionedDirectory(fs, stagingResultDir, finalOutputDir, stagingResultDir.toString(), renameDirs, oldTableDir); // Rename target partition directories for (Map.Entry<Path, Path> entry : renameDirs.entrySet()) { // Backup existing data files for recovering if (fs.exists(entry.getValue())) { String recoveryPathString = entry.getValue().toString() .replaceAll(finalOutputDir.toString(), oldTableDir.toString()); Path recoveryPath = new Path(recoveryPathString); fs.rename(entry.getValue(), recoveryPath); fs.exists(recoveryPath); recoveryDirs.put(entry.getValue(), recoveryPath); } // Delete existing directory fs.delete(entry.getValue(), true); // Rename staging directory to final output directory fs.rename(entry.getKey(), entry.getValue()); } } catch (IOException ioe) { // Remove created dirs for (Map.Entry<Path, Path> entry : renameDirs.entrySet()) { fs.delete(entry.getValue(), true); } // Recovery renamed dirs for (Map.Entry<Path, Path> entry : recoveryDirs.entrySet()) { fs.delete(entry.getValue(), true); fs.rename(entry.getValue(), entry.getKey()); } throw new IOException(ioe.getMessage()); } } else { // no partition try { // if the final output dir exists, move all contents to the temporary table dir. // Otherwise, just make the final output dir. As a result, the final output dir will be empty. if (fs.exists(finalOutputDir)) { fs.mkdirs(oldTableDir); for (FileStatus status : fs.listStatus(finalOutputDir, hiddenFileFilter)) { fs.rename(status.getPath(), oldTableDir); } movedToOldTable = fs.exists(oldTableDir); } else { // if the parent does not exist, make its parent directory. fs.mkdirs(finalOutputDir); } // Move the results to the final output dir. for (FileStatus status : fs.listStatus(stagingResultDir)) { fs.rename(status.getPath(), finalOutputDir); } // Check the final output dir committed = fs.exists(finalOutputDir); } catch (IOException ioe) { // recover the old table if (movedToOldTable && !committed) { // if commit is failed, recover the old data for (FileStatus status : fs.listStatus(finalOutputDir, hiddenFileFilter)) { fs.delete(status.getPath(), true); } for (FileStatus status : fs.listStatus(oldTableDir)) { fs.rename(status.getPath(), finalOutputDir); } } throw new IOException(ioe.getMessage()); } } } else { String queryType = queryContext.get(QueryVars.COMMAND_TYPE); if (queryType != null && queryType.equals(NodeType.INSERT.name())) { // INSERT INTO an existing table NumberFormat fmt = NumberFormat.getInstance(); fmt.setGroupingUsed(false); fmt.setMinimumIntegerDigits(3); if (!queryContext.get(QueryVars.OUTPUT_PARTITIONS, "").isEmpty()) { for (FileStatus eachFile : fs.listStatus(stagingResultDir)) { if (eachFile.isFile()) { LOG.warn("Partition table can't have file in a staging dir: " + eachFile.getPath()); continue; } moveResultFromStageToFinal(fs, stagingResultDir, eachFile, finalOutputDir, fmt, -1, changeFileSeq); } } else { int maxSeq = StorageUtil.getMaxFileSequence(fs, finalOutputDir, false) + 1; for (FileStatus eachFile : fs.listStatus(stagingResultDir)) { if (eachFile.getPath().getName().startsWith("_")) { continue; } moveResultFromStageToFinal(fs, stagingResultDir, eachFile, finalOutputDir, fmt, maxSeq++, changeFileSeq); } } // checking all file moved and remove empty dir verifyAllFileMoved(fs, stagingResultDir); FileStatus[] files = fs.listStatus(stagingResultDir); if (files != null && files.length != 0) { for (FileStatus eachFile : files) { LOG.error("There are some unmoved files in staging dir:" + eachFile.getPath()); } } } else { // CREATE TABLE AS SELECT (CTAS) if (fs.exists(finalOutputDir)) { for (FileStatus status : fs.listStatus(stagingResultDir)) { fs.rename(status.getPath(), finalOutputDir); } } else { fs.rename(stagingResultDir, finalOutputDir); } LOG.info("Moved from the staging dir to the output directory '" + finalOutputDir); } } // remove the staging directory if the final output dir is given. Path stagingDirRoot = stagingDir.getParent(); fs.delete(stagingDirRoot, true); } catch (Throwable t) { LOG.error(t); throw new IOException(t); } } else { finalOutputDir = new Path(stagingDir, TajoConstants.RESULT_DIR_NAME); } return finalOutputDir; }
From source file:org.apache.tajo.storage.FileTablespace.java
License:Apache License
/** * Attach the sequence number to the output file name and than move the file into the final result path. * * @param fs FileSystem/* w w w .j a va 2 s .c o m*/ * @param stagingResultDir The staging result dir * @param fileStatus The file status * @param finalOutputPath Final output path * @param nf Number format * @param fileSeq The sequence number * @throws java.io.IOException */ private void moveResultFromStageToFinal(FileSystem fs, Path stagingResultDir, FileStatus fileStatus, Path finalOutputPath, NumberFormat nf, int fileSeq, boolean changeFileSeq) throws IOException { if (fileStatus.isDirectory()) { String subPath = extractSubPath(stagingResultDir, fileStatus.getPath()); if (subPath != null) { Path finalSubPath = new Path(finalOutputPath, subPath); if (!fs.exists(finalSubPath)) { fs.mkdirs(finalSubPath); } int maxSeq = StorageUtil.getMaxFileSequence(fs, finalSubPath, false); for (FileStatus eachFile : fs.listStatus(fileStatus.getPath())) { if (eachFile.getPath().getName().startsWith("_")) { continue; } moveResultFromStageToFinal(fs, stagingResultDir, eachFile, finalOutputPath, nf, ++maxSeq, changeFileSeq); } } else { throw new IOException("Wrong staging dir:" + stagingResultDir + "," + fileStatus.getPath()); } } else { String subPath = extractSubPath(stagingResultDir, fileStatus.getPath()); if (subPath != null) { Path finalSubPath = new Path(finalOutputPath, subPath); if (changeFileSeq) { finalSubPath = new Path(finalSubPath.getParent(), replaceFileNameSeq(finalSubPath, fileSeq, nf)); } if (!fs.exists(finalSubPath.getParent())) { fs.mkdirs(finalSubPath.getParent()); } if (fs.exists(finalSubPath)) { throw new IOException("Already exists data file:" + finalSubPath); } boolean success = fs.rename(fileStatus.getPath(), finalSubPath); if (success) { LOG.info("Moving staging file[" + fileStatus.getPath() + "] + " + "to final output[" + finalSubPath + "]"); } else { LOG.error("Can't move staging file[" + fileStatus.getPath() + "] + " + "to final output[" + finalSubPath + "]"); } } } }
From source file:org.apache.tajo.storage.StorageManager.java
License:Apache License
/** * Finalizes result data. Tajo stores result data in the staging directory. * If the query fails, clean up the staging directory. * Otherwise the query is successful, move to the final directory from the staging directory. * * @param queryContext The query property * @param finalEbId The final execution block id * @param plan The query plan//from w w w .j a v a 2 s . c om * @param schema The final output schema * @param tableDesc The description of the target table * @param changeFileSeq If true change result file name with max sequence. * @return Saved path * @throws java.io.IOException */ protected Path commitOutputData(OverridableConf queryContext, ExecutionBlockId finalEbId, LogicalPlan plan, Schema schema, TableDesc tableDesc, boolean changeFileSeq) throws IOException { Path stagingDir = new Path(queryContext.get(QueryVars.STAGING_DIR)); Path stagingResultDir = new Path(stagingDir, TajoConstants.RESULT_DIR_NAME); Path finalOutputDir; if (!queryContext.get(QueryVars.OUTPUT_TABLE_PATH, "").isEmpty()) { finalOutputDir = new Path(queryContext.get(QueryVars.OUTPUT_TABLE_PATH)); try { FileSystem fs = stagingResultDir.getFileSystem(conf); if (queryContext.getBool(QueryVars.OUTPUT_OVERWRITE, false)) { // INSERT OVERWRITE INTO // It moves the original table into the temporary location. // Then it moves the new result table into the original table location. // Upon failed, it recovers the original table if possible. boolean movedToOldTable = false; boolean committed = false; Path oldTableDir = new Path(stagingDir, TajoConstants.INSERT_OVERWIRTE_OLD_TABLE_NAME); ContentSummary summary = fs.getContentSummary(stagingResultDir); if (!queryContext.get(QueryVars.OUTPUT_PARTITIONS, "").isEmpty() && summary.getFileCount() > 0L) { // This is a map for existing non-leaf directory to rename. A key is current directory and a value is // renaming directory. Map<Path, Path> renameDirs = TUtil.newHashMap(); // This is a map for recovering existing partition directory. A key is current directory and a value is // temporary directory to back up. Map<Path, Path> recoveryDirs = TUtil.newHashMap(); try { if (!fs.exists(finalOutputDir)) { fs.mkdirs(finalOutputDir); } visitPartitionedDirectory(fs, stagingResultDir, finalOutputDir, stagingResultDir.toString(), renameDirs, oldTableDir); // Rename target partition directories for (Map.Entry<Path, Path> entry : renameDirs.entrySet()) { // Backup existing data files for recovering if (fs.exists(entry.getValue())) { String recoveryPathString = entry.getValue().toString() .replaceAll(finalOutputDir.toString(), oldTableDir.toString()); Path recoveryPath = new Path(recoveryPathString); fs.rename(entry.getValue(), recoveryPath); fs.exists(recoveryPath); recoveryDirs.put(entry.getValue(), recoveryPath); } // Delete existing directory fs.delete(entry.getValue(), true); // Rename staging directory to final output directory fs.rename(entry.getKey(), entry.getValue()); } } catch (IOException ioe) { // Remove created dirs for (Map.Entry<Path, Path> entry : renameDirs.entrySet()) { fs.delete(entry.getValue(), true); } // Recovery renamed dirs for (Map.Entry<Path, Path> entry : recoveryDirs.entrySet()) { fs.delete(entry.getValue(), true); fs.rename(entry.getValue(), entry.getKey()); } throw new IOException(ioe.getMessage()); } } else { // no partition try { // if the final output dir exists, move all contents to the temporary table dir. // Otherwise, just make the final output dir. As a result, the final output dir will be empty. if (fs.exists(finalOutputDir)) { fs.mkdirs(oldTableDir); for (FileStatus status : fs.listStatus(finalOutputDir, StorageManager.hiddenFileFilter)) { fs.rename(status.getPath(), oldTableDir); } movedToOldTable = fs.exists(oldTableDir); } else { // if the parent does not exist, make its parent directory. fs.mkdirs(finalOutputDir); } // Move the results to the final output dir. for (FileStatus status : fs.listStatus(stagingResultDir)) { fs.rename(status.getPath(), finalOutputDir); } // Check the final output dir committed = fs.exists(finalOutputDir); } catch (IOException ioe) { // recover the old table if (movedToOldTable && !committed) { // if commit is failed, recover the old data for (FileStatus status : fs.listStatus(finalOutputDir, StorageManager.hiddenFileFilter)) { fs.delete(status.getPath(), true); } for (FileStatus status : fs.listStatus(oldTableDir)) { fs.rename(status.getPath(), finalOutputDir); } } throw new IOException(ioe.getMessage()); } } } else { String queryType = queryContext.get(QueryVars.COMMAND_TYPE); if (queryType != null && queryType.equals(NodeType.INSERT.name())) { // INSERT INTO an existing table NumberFormat fmt = NumberFormat.getInstance(); fmt.setGroupingUsed(false); fmt.setMinimumIntegerDigits(3); if (!queryContext.get(QueryVars.OUTPUT_PARTITIONS, "").isEmpty()) { for (FileStatus eachFile : fs.listStatus(stagingResultDir)) { if (eachFile.isFile()) { LOG.warn("Partition table can't have file in a staging dir: " + eachFile.getPath()); continue; } moveResultFromStageToFinal(fs, stagingResultDir, eachFile, finalOutputDir, fmt, -1, changeFileSeq); } } else { int maxSeq = StorageUtil.getMaxFileSequence(fs, finalOutputDir, false) + 1; for (FileStatus eachFile : fs.listStatus(stagingResultDir)) { if (eachFile.getPath().getName().startsWith("_")) { continue; } moveResultFromStageToFinal(fs, stagingResultDir, eachFile, finalOutputDir, fmt, maxSeq++, changeFileSeq); } } // checking all file moved and remove empty dir verifyAllFileMoved(fs, stagingResultDir); FileStatus[] files = fs.listStatus(stagingResultDir); if (files != null && files.length != 0) { for (FileStatus eachFile : files) { LOG.error("There are some unmoved files in staging dir:" + eachFile.getPath()); } } } else { // CREATE TABLE AS SELECT (CTAS) if (fs.exists(finalOutputDir)) { for (FileStatus status : fs.listStatus(stagingResultDir)) { fs.rename(status.getPath(), finalOutputDir); } } else { fs.rename(stagingResultDir, finalOutputDir); } LOG.info("Moved from the staging dir to the output directory '" + finalOutputDir); } } // remove the staging directory if the final output dir is given. Path stagingDirRoot = stagingDir.getParent(); fs.delete(stagingDirRoot, true); } catch (Throwable t) { LOG.error(t); throw new IOException(t); } } else { finalOutputDir = new Path(stagingDir, TajoConstants.RESULT_DIR_NAME); } return finalOutputDir; }
From source file:org.apache.tez.engine.lib.output.LocalOnFileSorterOutput.java
License:Apache License
@Override public void close() throws IOException, InterruptedException { LOG.debug("Closing LocalOnFileSorterOutput"); super.close(); TezTaskOutput mapOutputFile = sorter.getMapOutput(); FileSystem localFs = FileSystem.getLocal(mapOutputFile.getConf()); Path src = mapOutputFile.getOutputFile(); Path dst = mapOutputFile.getInputFileForWrite(sorter.getTaskAttemptId().getTaskID(), localFs.getFileStatus(src).getLen()); if (LOG.isDebugEnabled()) { LOG.debug("Renaming src = " + src + ", dst = " + dst); }/*from www . j av a2 s .c o m*/ localFs.rename(src, dst); }