Example usage for org.apache.hadoop.fs Path toString

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path toString.

Prototype

@Override
    public String toString() 

Source Link

Usage

From source file:com.ibm.stocator.fs.ObjectStoreFileSystem.java

License:Open Source License

@Override
public ContentSummary getContentSummary(Path f) throws IOException {
    LOG.debug("get content summary: {}", f.toString());
    return super.getContentSummary(f);
}

---

From source file:com.ibm.stocator.fs.ObjectStoreFileSystem.java

License:Open Source License

@Override
public long getDefaultBlockSize(Path f) {
    LOG.trace("Get default block size for: {}", f.toString());
    return super.getDefaultBlockSize(f);
}

---

From source file:com.ibm.stocator.fs.ObjectStoreFileSystem.java

License:Open Source License

/**
 * Extract object name from path. If addTaskIdCompositeName=true then
 * schema://tone1.lvm/aa/bb/cc/one3.txt/_temporary/0/_temporary/
 * attempt_201610052038_0001_m_000007_15/part-00007 will extract get
 * aa/bb/cc/201610052038_0001_m_000007_15-one3.txt
 * otherwise object name will be aa/bb/cc/one3.txt
 *
 * @param path path to extract from//from  ww w  .  j av  a2  s  . c o m
 * @param boundary boundary to search in a path
 * @param addTaskIdCompositeName if true will add task-id to the object name
 * @return new object name
 * @throws IOException if object name is missing
 */
private String getObjectName(Path fullPath, String boundary, boolean addTaskIdCompositeName)
        throws IOException {
    String path = fullPath.toString();
    String noPrefix = path.substring(hostNameScheme.length());
    int npIdx = noPrefix.indexOf(boundary);
    String objectName = "";
    if (npIdx >= 0) {
        if (npIdx == 0 || npIdx == 1 && noPrefix.startsWith("/")) {
            //no object name present
            //schema://tone1.lvm/_temporary/0/_temporary/attempt_201610038_0001_m_000007_15/part-0007
            //schema://tone1.lvm_temporary/0/_temporary/attempt_201610038_0001_m_000007_15/part-0007
            throw new IOException("Object name is missing");
        } else {
            //path matches pattern in javadoc
            objectName = noPrefix.substring(0, npIdx - 1);
            if (addTaskIdCompositeName) {
                String taskAttempt = Utils.extractTaskID(path);
                String objName = fullPath.getName();
                if (taskAttempt != null && !objName.startsWith(HADOOP_ATTEMPT)) {
                    objName = fullPath.getName() + "-" + taskAttempt;
                }
                objectName = objectName + "/" + objName;
            }
        }
        return objectName;
    }
    return noPrefix;
}

---

From source file:com.ibm.stocator.fs.ObjectStoreFileSystem.java

License:Open Source License

@Override
public FileStatus[] globStatus(Path pathPattern) throws IOException {
    LOG.debug("Glob status: {}", pathPattern.toString());
    return new ObjectStoreGlobber(this, pathPattern, DEFAULT_FILTER).glob();
}

---

From source file:com.ibm.stocator.fs.ObjectStoreFileSystem.java

License:Open Source License

@Override
public FileStatus[] globStatus(Path pathPattern, PathFilter filter) throws IOException {
    LOG.debug("Glob status {} with path filter {}", pathPattern.toString(), filter.toString());
    return new ObjectStoreGlobber(this, pathPattern, filter).glob();
}

---

From source file:com.ibm.stocator.fs.swift.pushdown.SwiftPushdownInputStream.java

License:Open Source License

public SwiftPushdownInputStream(SwiftAPIClient storeNative, String hostName, Path path) throws IOException {
    super(storeNative);
    LOG.debug("init: {}", path.toString());
    String str = " SwiftPushdownInputStream.constructor storeNative " + storeNative + " hostName = " + hostName
            + " path = " + path;
    // SwiftInputStream.printStackTrace(str);
    LOG.debug(str);/* ww  w .j a v a2s.co m*/
    Container theContainer = nativeStore.getAccount().getContainer(nativeStore.getDataRoot());
    blockSize = nativeStore.getBlockSize();
    storletVersion = nativeStore.getStorletVersion();
    csvRecordDelimiter = nativeStore.getCsvRecordDelimiter();
    delimiterLength = csvRecordDelimiter.length();
    maxRecordSize = nativeStore.getMaxRecordSize();
    dynamicStorletDebug = nativeStore.getDynamicStorletDebug();

    // handlePushdownHeaders does the following:
    // 1. put into pushdownHeaders all the necessary headers, except for the prefixLength
    //    which can not be determined till seek is invoked
    // 2. returns a path equals to input path but where the query string was truncated
    String noQueryPath = handlePushdownHeaders(path, theContainer, pushdownHeaders);
    String objectName = noQueryPath.substring(hostName.length());

    storedObject = theContainer.getObject(objectName);
    if (!storedObject.exists()) {
        throw new FileNotFoundException(objectName + " does not exists");
    } else {
        LOG.debug("SwiftPushdownInputStream.constructor blockSize = " + blockSize + " csvRecordDelimiter = "
                + csvRecordDelimiter + " delimiterLength = " + delimiterLength);
    }
}

---

From source file:com.ibm.stocator.fs.swift.pushdown.SwiftPushdownInputStream.java

License:Open Source License

/**
 * Adds within pushdownHeaders headers needed for pushdown storlet invocation
 * @return  a path equals to input path but where the query string was truncated
 *//*w w  w  .  ja va 2 s  .c  o m*/
private String handlePushdownHeaders(final Path path, final Container theContainer,
        final List<GeneralHeader> pushdownHeaders) {
    String fullPath = path.toString();

    int queryStartIndex = fullPath.indexOf(PushdownStorletConstants.SWIFT_STORLET_QUERY_START);
    if (queryStartIndex < 0) {
        LOG.debug("No pushdown since " + PushdownStorletConstants.SWIFT_STORLET_QUERY_START
                + " not to be found in full path " + fullPath);
        return fullPath;
    }

    String queryPath = fullPath.substring(queryStartIndex + 1);

    String[] theParams = queryPath.split(PushdownStorletConstants.SWIFT_PUSHDOWN_STORLET_QUERY_SEPARATOR); // ;
    int addedQueryParams = 0;

    for (String nextParam : theParams) {
        LOG.debug(" handlePushdownHeaders handling now " + nextParam);
        String[] paramParts = nextParam
                .split(PushdownStorletConstants.SWIFT_PUSHDOWN_STORLET_QUERY_PARAM_EQUAL);
        if (paramParts.length != 2) {
            LOG.warn("Skipping bad formed Storlet query parameter " + nextParam);
        } else {
            addStorletParameterHeader(pushdownHeaders, paramParts[0],
                    paramTransform(paramParts[0], paramParts[1]));
            addedQueryParams++;
        }
    }

    if (addedQueryParams > 0) {
        // add all the headers needed for the storlet invocation:

        // 1. this is the header that request the CSV storlet invocation:
        final String storletName = PushdownStorletConstants.SWIFT_PUSHDOWN_STORLET_NAME_PREFIX + storletVersion
                + PushdownStorletConstants.SWIFT_PUSHDOWN_STORLET_NAME_SUFIX;

        pushdownHeaders.add(
                new GeneralHeader(PushdownStorletConstants.SWIFT_PUSHDOWN_STORLET_HEADER_NAME, storletName));

        LOG.debug("Number of added ##Headers to container is " + pushdownHeaders.size());
    }

    return fullPath.substring(0, queryStartIndex);
}

---

From source file:com.ibm.stocator.fs.swift.SwiftAPIClient.java

License:Open Source License

@Override
public FileStatus getObjectMetadata(String hostName, Path path) throws IOException, FileNotFoundException {
    LOG.debug("Get object metadata: {}, hostname: {}", path, hostName);
    Container cont = mAccount.getContainer(container);
    /*//from w ww .  j a v  a  2 s  .  com
      The requested path is equal to hostName.
      HostName is equal to hostNameScheme, thus the container.
      Therefore we have no object to look for and
      we return the FileStatus as a directory.
      Containers have to lastModified.
     */
    if (path.toString().equals(hostName)) {
        LOG.debug("Object metadata requested on container!");
        return new FileStatus(0L, true, 1, blockSize, 0L, path);
    }
    /*
      The requested path is not equal to the container.
      We need to check if the object requested is a real object or a directory.
      This may be triggered when users want to access a directory rather than
      the entire container.
      A directory in Swift can have two implementations:
      1) a zero byte object with the name of the directory
      2) no zero byte object with the name of the directory
    */
    String objectName = path.toString().substring(hostName.length());
    if (objectName.endsWith("/")) {
        /*
          removing the trailing slash because it is not supported in Swift
          an request on an object (not a container) that has a trailing slash will lead
          to a 404 response message
        */
        objectName = objectName.substring(0, objectName.length() - 1);
    }
    StoredObject so = cont.getObject(objectName);
    boolean isDirectory = false;
    if (so.exists()) {
        // We need to check if the object size is equal to zero
        // If so, it might be a directory
        long contentLength = so.getContentLength();
        String lastModified = so.getLastModified();
        if (contentLength == 0) {
            Collection<DirectoryOrObject> directoryFiles = cont.listDirectory(objectName, '/', "", 10);
            if (directoryFiles != null && directoryFiles.size() != 0) {
                // The zero length object is a directory
                isDirectory = true;
            }
        }
        LOG.trace("Got object. isDirectory: {}  lastModified: {}", isDirectory, lastModified);
        return new FileStatus(contentLength, isDirectory, 1, blockSize, getLastModified(lastModified), path);
    }
    // We need to check if it may be a directory with no zero byte file associated
    Collection<DirectoryOrObject> directoryFiles = cont.listDirectory(objectName, '/', "", 10);
    if (directoryFiles != null && directoryFiles.size() != 0) {
        // In this case there is no lastModified
        LOG.debug("Got object. isDirectory: {}  lastModified: {}", isDirectory, null);
        return new FileStatus(0, isDirectory, 1, blockSize, 0L, path);
    }
    LOG.debug("Not found {}", path.toString());
    return null;
}

---

From source file:com.ibm.stocator.fs.swift.SwiftAPIClient.java

License:Open Source License

public boolean exists(String hostName, Path path) throws IOException, FileNotFoundException {
    LOG.trace("Object exists: {}", path);
    StoredObject so = mAccount.getContainer(container).getObject(path.toString().substring(hostName.length()));
    return so.exists();
}

---

From source file:com.ibm.stocator.fs.swift.SwiftAPIClient.java

License:Open Source License

/**
 * {@inheritDoc}/*  w  w w.j a v a 2s.c o m*/
 *
 * some examples of failed attempts:
 * a/b/c.data/part-00099-attempt_201603171503_0001_m_000099_119
 * a/b/c.data/part-00099-attempt_201603171503_0001_m_000099_120
 * a/b/c.data/part-00099-attempt_201603171503_0001_m_000099_121
 * a/b/c.data/part-00099-attempt_201603171503_0001_m_000099_122
 * or
 * a/b/c.data/part-r-00000-48ae3461-203f-4dd3-b141-a45426e2d26c
 * .csv-attempt_201603171328_0000_m_000000_1
 * a/b/c.data/part-r-00000-48ae3461-203f-4dd3-b141-a45426e2d26c
 * .csv-attempt_201603171328_0000_m_000000_0
 * in all the cases format is objectname-taskid where
 * taskid may vary, depends how many tasks were re-submitted
 * @param hostName
 * @param path
 * @param fullListing
 * @return Array of Hadoop FileStatus
 * @throws IOException
 */
public FileStatus[] list(String hostName, Path path, boolean fullListing) throws IOException {
    LOG.debug("List container: raw path parent", path.toString());
    Container cObj = mAccount.getContainer(container);
    String obj;
    if (path.toString().startsWith(container)) {
        obj = path.toString().substring(container.length() + 1);
    } else {
        obj = path.toString().substring(hostName.length());
    }
    LOG.debug("List container for {} container {}", obj, container);
    ArrayList<FileStatus> tmpResult = new ArrayList<FileStatus>();
    PaginationMap paginationMap = cObj.getPaginationMap(obj, pageListSize);
    FileStatus fs = null;
    StoredObject previousElement = null;
    for (Integer page = 0; page < paginationMap.getNumberOfPages(); page++) {
        Collection<StoredObject> res = cObj.list(paginationMap, page);
        if (page == 0 && (res == null || res.isEmpty())) {
            FileStatus[] emptyRes = {};
            LOG.debug("List {} in container {} is empty", obj, container);
            return emptyRes;
        }
        for (StoredObject tmp : res) {
            if (previousElement == null) {
                // first entry
                setCorrectSize(tmp, cObj);
                previousElement = tmp.getAsObject();
                continue;
            }
            String unifiedObjectName = extractUnifiedObjectName(tmp.getName());
            if (isSparkOrigin(unifiedObjectName) && !fullListing) {
                LOG.trace("{} created by Spark", unifiedObjectName);
                if (!isJobSuccessfull(unifiedObjectName)) {
                    LOG.trace("{} created by failed Spark job. Skipped", unifiedObjectName);
                    if (fModeAutomaticDelete) {
                        delete(hostName, new Path(tmp.getName()), true);
                    }
                    continue;
                } else {
                    // if we here - data created by spark and job completed successfully
                    // however there be might parts of failed tasks that were not aborted
                    // we need to make sure there are no failed attempts
                    if (nameWithoutTaskID(tmp.getName()).equals(nameWithoutTaskID(previousElement.getName()))) {
                        // found failed that was not aborted.
                        LOG.trace("Collisiion found between {} and {}", previousElement.getName(),
                                tmp.getName());
                        setCorrectSize(tmp, cObj);
                        if (previousElement.getContentLength() < tmp.getContentLength()) {
                            LOG.trace("New canditate is {}. Removed {}", tmp.getName(),
                                    previousElement.getName());
                            previousElement = tmp.getAsObject();
                        }
                        continue;
                    }
                }
            }
            fs = null;
            if (previousElement.getContentLength() > 0 || fullListing) {
                fs = getFileStatus(previousElement, cObj, hostName, path);
                tmpResult.add(fs);
            }
            previousElement = tmp.getAsObject();
        }
    }
    if (previousElement != null && (previousElement.getContentLength() > 0 || fullListing)) {
        fs = getFileStatus(previousElement, cObj, hostName, path);
        tmpResult.add(fs);
    }
    LOG.debug("Listing of {} completed with {} results", path.toString(), tmpResult.size());
    return tmpResult.toArray(new FileStatus[tmpResult.size()]);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »