Example usage for org.apache.hadoop.fs Path getParent

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path getParent.

Prototype

public Path getParent() 

Source Link

Document

Returns the parent of a path or null if at root.

Usage

From source file:org.apache.accumulo.tserver.log.LocalWALRecovery.java

License:Apache License

public void recoverLocalWriteAheadLogs(FileSystem fs) throws IOException {
    for (String directory : options.directories) {
        File localDirectory = new File(directory);
        if (!localDirectory.isAbsolute()) {
            localDirectory = new File(System.getenv("ACCUMULO_HOME"), directory);
        }/* w  w w. ja v a2 s.c om*/

        if (!localDirectory.isDirectory()) {
            log.warn("Local walog dir " + localDirectory.getAbsolutePath()
                    + " does not exist or is not a directory.");
            continue;
        }

        if (options.destination == null) {
            // Defer loading the default value until now because it might require talking to zookeeper.
            options.destination = ServerConstants.getWalDirs()[0];
        }
        log.info("Copying WALs to " + options.destination);

        for (File file : localDirectory.listFiles()) {
            String name = file.getName();
            try {
                UUID.fromString(name);
            } catch (IllegalArgumentException ex) {
                log.info("Ignoring non-log file " + file.getAbsolutePath());
                continue;
            }

            LogFileKey key = new LogFileKey();
            LogFileValue value = new LogFileValue();

            log.info("Openning local log " + file.getAbsolutePath());

            Path localWal = new Path(file.toURI());
            FileSystem localFs = FileSystem.getLocal(fs.getConf());

            Reader reader = new SequenceFile.Reader(localFs, localWal, localFs.getConf());
            // Reader reader = new SequenceFile.Reader(localFs.getConf(), SequenceFile.Reader.file(localWal));
            Path tmp = new Path(options.destination + "/" + name + ".copy");
            FSDataOutputStream writer = fs.create(tmp);
            while (reader.next(key, value)) {
                try {
                    key.write(writer);
                    value.write(writer);
                } catch (EOFException ex) {
                    break;
                }
            }
            writer.close();
            reader.close();
            fs.rename(tmp, new Path(tmp.getParent(), name));

            if (options.deleteLocal) {
                if (file.delete()) {
                    log.info("Copied and deleted: " + name);
                } else {
                    log.info("Failed to delete: " + name + " (but it is safe for you to delete it manually).");
                }
            } else {
                log.info("Safe to delete: " + name);
            }
        }
    }
}

---

From source file:org.apache.accumulo.tserver.log.SortedLogRecovery.java

License:Apache License

private String getPathSuffix(String pathString) {
    Path path = new Path(pathString);
    if (path.depth() < 2)
        throw new IllegalArgumentException("Bad path " + pathString);
    return path.getParent().getName() + "/" + path.getName();
}

---

From source file:org.apache.accumulo.tserver.TabletServer.java

License:Apache License

public void recover(VolumeManager fs, KeyExtent extent, TableConfiguration tconf, List<LogEntry> logEntries,
        Set<String> tabletFiles, MutationReceiver mutationReceiver) throws IOException {
    List<Path> recoveryLogs = new ArrayList<>();
    List<LogEntry> sorted = new ArrayList<>(logEntries);
    Collections.sort(sorted, new Comparator<LogEntry>() {
        @Override//from   w  w  w .  j a v  a2 s  .  c o m
        public int compare(LogEntry e1, LogEntry e2) {
            return (int) (e1.timestamp - e2.timestamp);
        }
    });
    for (LogEntry entry : sorted) {
        Path recovery = null;
        Path finished = RecoveryPath.getRecoveryPath(fs, fs.getFullPath(FileType.WAL, entry.filename));
        finished = SortedLogState.getFinishedMarkerPath(finished);
        TabletServer.log.info("Looking for " + finished);
        if (fs.exists(finished)) {
            recovery = finished.getParent();
        }
        if (recovery == null)
            throw new IOException("Unable to find recovery files for extent " + extent + " logEntry: " + entry);
        recoveryLogs.add(recovery);
    }
    logger.recover(fs, extent, tconf, recoveryLogs, tabletFiles, mutationReceiver);
}

---

From source file:org.apache.ambari.fast_hdfs_resource.Resource.java

License:Apache License

public void fillInParentDirectories(FileSystem dfs, String path, HashSet<String> resultSet) throws IOException {
    Path filePath = new Path(path);

    while (true) {
        filePath = filePath.getParent();

        // if(filePath.isRoot()) {
        if (filePath.getParent() == null) {
            break;
        }/*from ww  w.j  av a2 s  . c  om*/
        resultSet.add(filePath.toString());
    }
}

---

From source file:org.apache.apex.malhar.lib.fs.s3.S3Reconciler.java

License:Apache License

/**
 * Remove intermediate files//  w  ww .  j  av  a  2  s .  c  om
 */
protected void removeIntermediateFiles(FSRecordCompactionOperator.OutputMetaData metaData) {
    logger.debug("found metaData = {}", metaData);
    committedTuples.remove(metaData);
    try {
        Path dest = new Path(metaData.getPath());
        //Deleting the intermediate files and when writing to tmp files
        // there can be vagrant tmp files which we have to clean
        FileStatus[] statuses = fs.listStatus(dest.getParent());

        for (FileStatus status : statuses) {
            String statusName = status.getPath().getName();
            if (statusName.endsWith(TMP_EXTENSION) && statusName.startsWith(metaData.getFileName())) {
                //a tmp file has tmp extension always preceded by timestamp
                String actualFileName = statusName.substring(0,
                        statusName.lastIndexOf('.', statusName.lastIndexOf('.') - 1));
                logger.debug("actualFileName = {}", actualFileName);
                if (metaData.getFileName().equals(actualFileName)) {
                    logger.debug("deleting stray file {}", statusName);
                    fs.delete(status.getPath(), true);
                }
            } else if (statusName.equals(metaData.getFileName())) {
                logger.info("deleting s3-compaction file {}", statusName);
                fs.delete(status.getPath(), true);
            }
        }
    } catch (IOException e) {
        logger.error("Unable to Delete a file: {}", metaData.getFileName());
    }
}

---

From source file:org.apache.apex.malhar.lib.io.fs.AbstractFileOutputOperator.java

License:Apache License

/**
 * Finalizing a file means that the same file will never be open again.
 *
 * @param fileName name of the file to finalize
 *//*from w w w  .jav a  2s  .  c o m*/
protected void finalizeFile(String fileName) throws IOException {
    String tmpFileName = fileNameToTmpName.get(fileName);
    Path srcPath = new Path(filePath + Path.SEPARATOR + tmpFileName);
    Path destPath = new Path(filePath + Path.SEPARATOR + fileName);

    if (!fs.exists(destPath)) {
        LOG.debug("rename from tmp {} actual {} ", tmpFileName, fileName);
        rename(srcPath, destPath);
    } else if (fs.exists(srcPath)) {
        /*if the destination and src both exists that means there was a failure between file rename and clearing the
        endOffset so we just delete the tmp file*/
        LOG.debug("deleting tmp {}", tmpFileName);
        fs.delete(srcPath, true);
    }
    endOffsets.remove(fileName);
    fileNameToTmpName.remove(fileName);

    //when writing to tmp files there can be vagrant tmp files which we have to clean
    FileStatus[] statuses = fs.listStatus(destPath.getParent());
    for (FileStatus status : statuses) {
        String statusName = status.getPath().getName();
        if (statusName.endsWith(TMP_EXTENSION) && statusName.startsWith(destPath.getName())) {
            //a tmp file has tmp extension always preceded by timestamp
            String actualFileName = statusName.substring(0,
                    statusName.lastIndexOf('.', statusName.lastIndexOf('.') - 1));
            if (fileName.equals(actualFileName)) {
                LOG.debug("deleting stray file {}", statusName);
                fs.delete(status.getPath(), true);
            }
        }
    }
}

---

From source file:org.apache.beam.sdk.io.hdfs.HadoopFileSystem.java

License:Apache License

/** Ensures that the target directory exists for the given filePath. */
private void mkdirs(Path filePath) throws IOException {
    final org.apache.hadoop.fs.FileSystem fs = filePath.getFileSystem(configuration);
    final Path targetDirectory = filePath.getParent();
    if (!fs.exists(targetDirectory)) {
        LOG.debug(String.format(LOG_CREATE_DIRECTORY, Path.getPathWithoutSchemeAndAuthority(targetDirectory)));
        if (!fs.mkdirs(targetDirectory)) {
            throw new IOException(String.format(
                    "Unable to create target directory %s. No further information provided by underlying filesystem.",
                    targetDirectory));//from w  w  w.  j av  a 2s.c o  m
        }
    }
}

---

From source file:org.apache.bigtop.bigpetstore.etl.PigCSVCleaner.java

License:Apache License

public PigCSVCleaner(Path inputPath, Path outputPath, ExecType ex, File... scripts) throws Exception {
    FileSystem fs = FileSystem.get(inputPath.toUri(), new Configuration());

    if (!fs.exists(inputPath)) {
        throw new RuntimeException("INPUT path DOES NOT exist : " + inputPath);
    }/* w  w w.  j  a  v a 2  s  . co m*/

    if (fs.exists(outputPath)) {
        throw new RuntimeException("OUTPUT already exists : " + outputPath);
    }
    // run pig in local mode
    pigServer = new PigServer(ex);

    /**
     * First, split the tabs up.
     *
     * BigPetStore,storeCode_OK,2 1,yang,jay,3,flea collar,69.56,Mon Dec 15 23:33:49 EST 1969
     *
     * ("BigPetStore,storeCode_OK,2", "1,yang,jay,3,flea collar,69.56,Mon Dec 15 23:33:49 EST 1969")
     */
    pigServer.registerQuery("csvdata = LOAD '<i>' AS (ID,DETAILS);".replaceAll("<i>", inputPath.toString()));

    // currentCustomerId, firstName, lastName, product.id, product.name.toLowerCase, product.price, date
    /**
     * Now, we want to split the two tab delimited fields into uniform
     * fields of comma separated values. To do this, we 1) Internally split
     * the FIRST and SECOND fields by commas "a,b,c" --> (a,b,c) 2) FLATTEN
     * the FIRST and SECOND fields. (d,e) (a,b,c) -> d e a b c
     */
    pigServer.registerQuery("id_details = FOREACH csvdata GENERATE " + "FLATTEN(STRSPLIT(ID, ',', 3)) AS "
            + "(drop, code, transaction) ,"

            + "FLATTEN(STRSPLIT(DETAILS, ',', 7)) AS "
            + "(custId, fname, lname, productId, product:chararray, price, date);");
    pigServer.registerQuery("mahout_records = FOREACH id_details GENERATE custId, productId, 1;");
    pigServer.store("id_details", getCleanedTsvPath(outputPath).toString());
    pigServer.store("mahout_records", new Path(outputPath, OUTPUTS.MahoutPaths.Mahout.name()).toString());
    /**
     * Now we run scripts... this is where you can add some
     * arbitrary analytics.
     *
     * We add "input" and "output" parameters so that each
     * script can read them and use them if they want.
     *
     * Otherwise, just hardcode your inputs into your pig scripts.
     */
    int i = 0;
    for (File script : scripts) {
        Map<String, String> parameters = new HashMap<>();
        parameters.put("input", getCleanedTsvPath(outputPath).toString());

        Path dir = outputPath.getParent();
        Path adHocOut = new Path(dir, OUTPUTS.pig_ad_hoc_script.name() + (i++));
        System.out.println("Setting default output to " + adHocOut);
        parameters.put("output", adHocOut.toString());
        pigServer.registerScript(script.getAbsolutePath(), parameters);
    }
}

---

From source file:org.apache.blur.analysis.HdfsFieldManager.java

License:Apache License

@Override
protected boolean tryToStore(FieldTypeDefinition fieldTypeDefinition, String fieldName) throws IOException {
    Tracer trace = Trace.trace("filesystem - tryToStore fieldName", Trace.param("fieldName", fieldName),
            Trace.param("storagePath", _storagePath));
    try {//from  ww  w  .ja  va  2 s  .  co  m
        // Might want to make this a ZK lock
        _lock.lock();
        try {
            String fieldType = fieldTypeDefinition.getFieldType();
            boolean fieldLessIndexed = fieldTypeDefinition.isFieldLessIndexed();
            boolean sortEnable = fieldTypeDefinition.isSortEnable();
            boolean multiValueField = fieldTypeDefinition.isMultiValueField();
            LOG.info(
                    "Attempting to store new field [{0}] with fieldLessIndexing [{1}] with type [{2}] and properties [{3}]",
                    fieldName, fieldLessIndexed, fieldType, fieldTypeDefinition.getProperties());
            Properties properties = new Properties();
            setProperty(properties, FAMILY, fieldTypeDefinition.getFamily());
            setProperty(properties, FAMILY, fieldTypeDefinition.getFamily());
            setProperty(properties, COLUMN_NAME, fieldTypeDefinition.getColumnName());
            setProperty(properties, SUB_COLUMN_NAME, fieldTypeDefinition.getSubColumnName());
            setProperty(properties, FIELD_LESS_INDEXING, Boolean.toString(fieldLessIndexed));
            setProperty(properties, SORTENABLED, Boolean.toString(sortEnable));
            setProperty(properties, MULTI_VALUE_FIELD, Boolean.toString(multiValueField));

            setProperty(properties, FIELD_TYPE, fieldType);
            Map<String, String> props = fieldTypeDefinition.getProperties();
            if (props != null) {
                for (Entry<String, String> e : props.entrySet()) {
                    properties.setProperty(e.getKey(), e.getValue());
                }
            }
            Path path = getFieldPath(fieldName);
            if (_fileSystem.exists(path)) {
                LOG.info("Field [{0}] already exists.", fieldName);
                return false;
            }
            Path tmpPath = new Path(path.getParent(), UUID.randomUUID().toString() + ".tmp");
            FSDataOutputStream outputStream = _fileSystem.create(tmpPath, false);
            properties.store(outputStream, getComments());
            outputStream.close();
            if (_fileSystem.rename(tmpPath, path)) {
                // @TODO make this configurable
                _fileSystem.setReplication(path, (short) 10);
                return true;
            } else {
                _fileSystem.delete(tmpPath, false);
                LOG.info("Field [{0}] already exists.", fieldName, fieldLessIndexed, fieldType, props);
                return false;
            }
        } finally {
            _lock.unlock();
        }
    } finally {
        trace.done();
    }
}

---

From source file:org.apache.blur.manager.writer.IndexImporter.java

License:Apache License

@Override
public void run() {
    // Only allow one import to occur in the process at a time.
    _globalLock.lock();/*  w w w .  j a  v a2  s .c  o m*/
    try {
        if (_lastCleanup + _cleanupDelay < System.currentTimeMillis()) {
            try {
                cleanupOldDirs();
            } catch (IOException e) {
                LOG.error("Unknown error while trying to clean old directories on [{1}/{2}].", e, _shard,
                        _table);
            }
            _lastCleanup = System.currentTimeMillis();
        }
        Path path = _shardContext.getHdfsDirPath();
        Configuration configuration = _shardContext.getTableContext().getConfiguration();
        try {
            FileSystem fileSystem = path.getFileSystem(configuration);
            SortedSet<FileStatus> listStatus;
            while (true) {
                try {
                    listStatus = sort(fileSystem.listStatus(path, new PathFilter() {
                        @Override
                        public boolean accept(Path path) {
                            if (path != null && path.getName().endsWith(COMMIT)) {
                                return true;
                            }
                            return false;
                        }
                    }));
                    break;
                } catch (FileNotFoundException e) {
                    LOG.warn("File not found error, retrying.");
                }
                try {
                    Thread.sleep(100);
                } catch (InterruptedException e) {
                    return;
                }
            }
            for (FileStatus fileStatus : listStatus) {
                Path file = fileStatus.getPath();
                if (fileStatus.isDir() && file.getName().endsWith(COMMIT)) {
                    // rename to inuse, if good continue else rename to badindex
                    Path inuse = new Path(file.getParent(), rename(file.getName(), INUSE));
                    touch(fileSystem, new Path(file, INPROGRESS));
                    if (fileSystem.rename(file, inuse)) {
                        if (_testError != null) {
                            _testError.run();
                        }
                        HdfsDirectory hdfsDirectory = new HdfsDirectory(configuration, inuse);
                        try {
                            if (DirectoryReader.indexExists(hdfsDirectory)) {
                                IndexAction indexAction = getIndexAction(hdfsDirectory, fileSystem);
                                _blurIndex.process(indexAction);
                                return;
                            } else {
                                Path badindex = new Path(file.getParent(), rename(file.getName(), BADINDEX));
                                if (fileSystem.rename(inuse, badindex)) {
                                    LOG.error(
                                            "Directory found at [{0}] is not a vaild index, renaming to [{1}].",
                                            inuse, badindex);
                                } else {
                                    LOG.fatal(
                                            "Directory found at [{0}] is not a vaild index, could not rename to [{1}].",
                                            inuse, badindex);
                                }
                            }
                        } finally {
                            hdfsDirectory.close();
                        }
                    } else {
                        LOG.fatal("Could not rename [{0}] to inuse dir.", file);
                    }
                }
            }
        } catch (IOException e) {
            LOG.error("Unknown error while trying to refresh imports on [{1}/{2}].", e, _shard, _table);
        }
    } finally {
        _globalLock.unlock();
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »