nextflow.fs.dx.DxFileSystemProvider.java Source code

Java tutorial

Introduction

Here is the source code for nextflow.fs.dx.DxFileSystemProvider.java

Source

/*
 * Copyright (c) 2013, the authors.
 *
 *   This file is part of 'DXFS'.
 *
 *   DXFS is free software: you can redistribute it and/or modify
 *   it under the terms of the GNU General Public License as published by
 *   the Free Software Foundation, either version 3 of the License, or
 *   (at your option) any later version.
 *
 *   DXFS is distributed in the hope that it will be useful,
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *   GNU General Public License for more details.
 *
 *   You should have received a copy of the GNU General Public License
 *   along with DXFS.  If not, see <http://www.gnu.org/licenses/>.
 */

package nextflow.fs.dx;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;
import java.nio.channels.FileChannel;
import java.nio.channels.SeekableByteChannel;
import java.nio.charset.Charset;
import java.nio.file.AccessMode;
import java.nio.file.CopyOption;
import java.nio.file.DirectoryStream;
import java.nio.file.DirectoryStream.Filter;
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.FileStore;
import java.nio.file.FileSystem;
import java.nio.file.FileSystemAlreadyExistsException;
import java.nio.file.Files;
import java.nio.file.LinkOption;
import java.nio.file.OpenOption;
import java.nio.file.Path;
import java.nio.file.ProviderMismatchException;
import java.nio.file.StandardCopyOption;
import java.nio.file.StandardOpenOption;
import java.nio.file.attribute.BasicFileAttributes;
import java.nio.file.attribute.FileAttribute;
import java.nio.file.attribute.FileAttributeView;
import java.nio.file.spi.FileSystemProvider;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import com.fasterxml.jackson.databind.JsonNode;
import nextflow.fs.dx.api.DxApi;
import nextflow.fs.dx.api.DxHttpClient;
import nextflow.fs.dx.api.DxJson;
import org.apache.http.HttpVersion;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.params.ClientPNames;
import org.apache.http.client.params.CookiePolicy;
import org.apache.http.params.CoreProtocolPNames;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * NIO2 File system provider for DnaNexus cloud storage
 *
 * @author  Paolo Di Tommaso <paolo.ditommaso@gmail.com>
 * @author  Beatriz San Juan <bmsanjuan@gmail.com>
 *
 */

public class DxFileSystemProvider extends FileSystemProvider {

    private static final Logger log = LoggerFactory.getLogger(DxFileSystemProvider.class);

    /**
     * The *scheme* defined by this provider
     */
    public static String SCHEME = "dxfs";

    /**
     * Pattern matching a DnaNexus URI e.g. dxfs://container:some/path
     */
    public static Pattern URI_PATTERN = Pattern.compile("^" + SCHEME + "://(([^/]+):)?(.*)$");

    /**
     * Pattern matching a DnaNexus project or context ID
     */
    static Pattern CONTEXT_ID_PATTERN = Pattern.compile("^(project-|container-)[a-zA-Z0-9]{24}$");

    /**
     *  Pattern matching a DnaNexus file ID
     */
    public static Pattern FILE_ID_PATTERN = Pattern.compile("file-[a-zA-Z0-9]{24}");

    /**
     * Hold DnaNexus context ID, either a project id or a data container container
     * <p>
     * Read more
     * https://wiki.dnanexus.com/API-Specification-v1.0.0/Data-Containers#
     */
    final String defaultContextId;

    final ConcurrentHashMap<String, DxFileSystem> fileSystems = new ConcurrentHashMap<>();

    final DxApi api;

    public DxFileSystemProvider() {
        this.defaultContextId = getDefaultDxContextId();
        this.api = DxApi.getInstance();
    }

    protected DxFileSystemProvider(String containerId, DxApi api) {
        this.defaultContextId = containerId;
        this.api = api;
    }

    /**
     * Find out the default DnaNexus project id in the default user configuration file,
     * i.e. the file {@code $HOME/.dnanexus_config/environment.json}
     *
     * @return The string value
     */
    static String getDefaultDxContextId() {

        String result;

        @SuppressWarnings({ "unchecked", "rawtypes" })
        Map<String, String> props = new HashMap(System.getProperties());
        result = getContextIdByMap(props, null);

        if (result == null) {
            result = getContextIdByMap(System.getenv(), null);
        }

        if (result == null) {
            String home = System.getProperty("user.home");
            File config = new File(home, ".dnanexus_config/environment.json");
            if (!config.exists()) {
                return null;
            }
            result = getContextIdByConfig(config);
            log.debug("Using DX_PROJECT_CONTEXT_ID = {} in config file: {}", result, config);
        }

        return result;
    }

    static String getContextIdByMap(Map<String, ?> map, String defValue) {

        if (map.containsKey("DX_WORKSPACE_ID")) {
            String result = map.get("DX_WORKSPACE_ID").toString();
            log.debug("Using DX_WORKSPACE_ID = {}", result);
            return result;
        } else if (map.containsKey("DX_PROJECT_CONTEXT_ID")) {
            String result = map.get("DX_PROJECT_CONTEXT_ID").toString();
            log.debug("Using DX_PROJECT_CONTEXT_ID = {}", result);
            return result;
        }

        if (defValue != null) {
            log.debug("Using default context id = {}", defValue);
        }
        return defValue;
    }

    /**
     * Find out the default DnaNexus project id in the specified configuration file
     *
     * @return The string value
     */
    static String getContextIdByConfig(File config) {

        StringBuilder buffer = new StringBuilder();
        try {
            BufferedReader reader = Files.newBufferedReader(config.toPath(), Charset.defaultCharset());
            String line;
            while ((line = reader.readLine()) != null) {
                buffer.append(line).append('\n');
            }

            JsonNode object = DxJson.parseJson(buffer.toString());
            return object.get("DX_PROJECT_CONTEXT_ID").textValue();
        } catch (FileNotFoundException e) {
            throw new IllegalStateException(String.format(
                    "Unable to load DnaNexus configuration file: %s -- cannot configure file system", config), e);
        } catch (IOException e) {
            throw new IllegalStateException("Unable to configure DnaNexus file system", e);
        }
    }

    protected DxApi api() {
        return api;
    }

    @Override
    public String getScheme() {
        return SCHEME;
    }

    @Override
    public Path getPath(URI uri) {
        log.trace("Get path by URI: {}", uri);
        PathTokens tokens = resolveUri(uri, defaultContextId);

        DxFileSystem dxFileSystem = getOrCreateFileSystem(tokens.contextId, tokens.name);

        return new DxPath(dxFileSystem, tokens.filePath);
    }

    protected DxFileSystem newFileSystem() {
        return newFileSystem(defaultContextId, defaultContextId);
    }

    protected DxFileSystem newFileSystem(String contextId, String label) {
        if (contextId == null) {
            throw new IllegalStateException("Missing 'contextId' attribute");
        }
        return new DxFileSystem(this, contextId, label);
    }

    @Override
    public final FileSystem newFileSystem(URI uri, Map<String, ?> env) {

        final String defContextId = getContextIdByMap(env, defaultContextId);
        final PathTokens tokens = resolveUri(uri, defContextId);
        final String dxContextId = tokens.contextId;
        final DxFileSystem result = newFileSystem(dxContextId, tokens.name);

        if (fileSystems.putIfAbsent(dxContextId, result) != null) {
            throw new FileSystemAlreadyExistsException();
        }

        return result;
    }

    // -- package private
    final DxFileSystem getOrCreateFileSystem(String contextId, String name) {

        DxFileSystem dxFileSystem = fileSystems.get(contextId);
        if (dxFileSystem == null) {
            log.debug("Creating a new DxFileSystem object with context-id: {}", contextId);
            dxFileSystem = newFileSystem(contextId, name);
            DxFileSystem former = fileSystems.putIfAbsent(contextId, dxFileSystem);
            if (former != null) {
                log.trace(
                        "Look ma, got a concurrent creation of a DxFileSystem for context-id: {} -- using the previous instance",
                        contextId);
                return former;
            }
        }

        return dxFileSystem;
    }

    // -- package private
    final DxFileSystem getOrCreateFileSystem(String contextId) {
        return getOrCreateFileSystem(contextId, contextId);
    }

    static class PathTokens {

        /** Descriptive name of the context/project */
        String name;

        /** The real container/project id */
        String contextId;

        /** The fil path in the container/project */
        String filePath;

        @Override
        public boolean equals(Object o) {
            if (this == o)
                return true;
            if (o == null || getClass() != o.getClass())
                return false;

            PathTokens that = (PathTokens) o;

            if (contextId != null ? !contextId.equals(that.contextId) : that.contextId != null)
                return false;
            if (filePath != null ? !filePath.equals(that.filePath) : that.filePath != null)
                return false;
            if (name != null ? !name.equals(that.name) : that.name != null)
                return false;

            return true;
        }

        @Override
        public int hashCode() {
            int result = name != null ? name.hashCode() : 0;
            result = 31 * result + (contextId != null ? contextId.hashCode() : 0);
            result = 31 * result + (filePath != null ? filePath.hashCode() : 0);
            return result;
        }
    }

    protected PathTokens checkUri(URI uri) {

        Matcher matcher = URI_PATTERN.matcher(uri.toString());
        if (!matcher.matches()) {
            throw new IllegalArgumentException("URI does not match this provider: " + uri);
        }

        String contextId = matcher.group(2);
        String path = matcher.group(3);

        PathTokens tokens = new PathTokens();
        tokens.name = contextId;
        tokens.contextId = contextId;
        tokens.filePath = path;
        return tokens;
    }

    protected PathTokens resolveUri(URI uri, String defContextId) {

        PathTokens tokens = checkUri(uri);
        if (tokens.contextId == null) {
            tokens.contextId = defContextId;
            return tokens;
        }

        Matcher matcher = CONTEXT_ID_PATTERN.matcher(tokens.contextId);
        if (!matcher.matches()) {
            // look for this container name by invoking the remote API
            try {
                List<Map<String, Object>> found = api.projectFind(tokens.contextId);
                if (found == null || found.size() != 1) {
                    throw new IllegalStateException(
                            String.format("Unable to retrieve project-id by name: '%s' (1)", tokens.contextId));
                }
                tokens.contextId = found.get(0).get("id").toString();
            } catch (IOException e) {
                throw new IllegalStateException(
                        String.format("Unable to retrieve project-id by name: '%s' (2)", tokens.contextId), e);
            }
        }

        return tokens;
    }

    @Override
    public final FileSystem getFileSystem(URI uri) {
        log.trace("Parsing URI: {}", uri);
        PathTokens tokens = resolveUri(uri, defaultContextId);
        return fileSystems.get(tokens.contextId);
    }

    @Override
    public DirectoryStream<Path> newDirectoryStream(Path dir, Filter<? super Path> filter) throws IOException {
        return new DxDirectoryStream(toDxPath(dir), filter);
    }

    public InputStream newInputStream(Path file, OpenOption... options) throws IOException {
        if (options.length > 0) {
            for (OpenOption opt : options) {
                if (opt != StandardOpenOption.READ)
                    throw new UnsupportedOperationException("'" + opt + "' not allowed");
            }
        }

        final DxPath path = toDxPath(file);
        final String fileId = path.getFileId();
        final Map<String, Object> download = api.fileDownload(fileId);
        final String url = (String) download.get("url");
        final Map<String, String> headers = (Map<String, String>) download.get("headers");

        final HttpClient client = DxHttpClient.getInstance().http();
        client.getParams().setParameter(CoreProtocolPNames.PROTOCOL_VERSION, HttpVersion.HTTP_1_1);

        HttpGet get = new HttpGet(url);
        get.getParams().setParameter(ClientPNames.COOKIE_POLICY, CookiePolicy.IGNORE_COOKIES);
        for (Map.Entry<String, String> item : headers.entrySet()) {
            get.setHeader(item.getKey(), item.getValue());
        }

        return client.execute(get).getEntity().getContent();
    }

    private static void checkAllowedOptions(Set<? extends OpenOption> allowed, OpenOption... options) {
        if (options == null)
            return;
        for (OpenOption opt : options) {
            if (!allowed.contains(opt)) {
                throw new UnsupportedOperationException(opt.toString() + " options not allowed");
            }
        }
    }

    private static Set<? extends OpenOption> OUTPUT_STREAM_VALID_OPTIONS = new HashSet<>(
            Arrays.asList(StandardOpenOption.CREATE, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE,
                    StandardOpenOption.TRUNCATE_EXISTING));

    /**
     *
     * @param path
     * @param options
     * @return
     * @throws IOException
     */
    public OutputStream newOutputStream(Path path, OpenOption... options) throws IOException {

        checkAllowedOptions(OUTPUT_STREAM_VALID_OPTIONS, options);

        // create the file
        final DxPath thePath = toDxPath(path);
        final DxFileSystem theFileSystem = thePath.getFileSystem();
        final String fileId = theFileSystem.fileNew(thePath);

        // set the type accordingly
        thePath.fileId = fileId;
        thePath.type = DxPath.PathType.FILE;

        // create the output stream uploaded
        return new DxUploadOutputStream(fileId, api);

    }

    /**
     * Operation not supported
     *
     * @throws UnsupportedOperationException
     */
    @Override
    public FileChannel newFileChannel(Path path, Set<? extends OpenOption> options, FileAttribute<?>... attrs)
            throws IOException {
        throw new UnsupportedOperationException();
    }

    /**
     * Operation not supported
     *
     * @throws UnsupportedOperationException
     */
    @Override
    public SeekableByteChannel newByteChannel(Path path, Set<? extends OpenOption> options,
            FileAttribute<?>... attrs) throws IOException {

        throw new UnsupportedOperationException();
    }

    /**
     * Create a new *remote* folder
     * <p>
     *
     * See https://wiki.dnanexus.com/API-Specification-v1.0.0/Folders%20and%20Deletion#API-method:-/class-xxxx/newFolder
     *
     * @param path
     * @param attrs
     * @throws IOException
     */
    @Override
    public void createDirectory(Path path, FileAttribute<?>... attrs) throws IOException {

        if (attrs.length > 0) {
            throw new UnsupportedOperationException(
                    "Attributes on directories are not supported by DnaNexus file system");
        }

        DxPath dxPath = toDxPath(path);
        DxFileSystem theFileSystem = dxPath.getFileSystem();
        theFileSystem.createFolder(dxPath);
        dxPath.type = DxPath.PathType.DIRECTORY;
    }

    /**
     * Deletes a file. This method works in exactly the  manner specified by the
     * {@link Files#delete} method.
     *
     * @param   path
     *          the path to the file to delete
     *
     * @throws java.nio.file.NoSuchFileException
     *          if the file does not exist <i>(optional specific exception)</i>
     * @throws java.nio.file.DirectoryNotEmptyException
     *          if the file is a directory and could not otherwise be deleted
     *          because the directory is not empty <i>(optional specific
     *          exception)</i>
     * <p>
     * See http://wiki.dnanexus.com/API-Specification-v1.0.0/Folders-and-Deletion#API-method:-/class-xxxx/removeObjects
     *
     * @param path
     * @throws IOException
     */
    @Override
    public void delete(Path path) throws IOException {

        DxPath dxPath = toDxPath(path);
        DxFileSystem dxFileSystem = dxPath.getFileSystem();
        DxFileAttributes attr = dxPath.readAttributes();

        if (attr.isDirectory()) {
            dxFileSystem.deleteFolder(dxPath, false);
        } else {
            dxFileSystem.deleteFiles(dxPath);
        }
        // clear all the attributes on this file since does not exist any more
        dxPath.clearAttributes();
    }

    public void deleteDir(DxPath path, boolean recurse) throws IOException {
        path.getFileSystem().deleteFolder(path, recurse);
    }

    /**
     * Implements the *copy* operation using the DnaNexus API *clone*
     *
     *
     * <p>
     * See clone https://wiki.dnanexus.com/API-Specification-v1.0.0/Cloning#API-method%3A-%2Fclass-xxxx%2Fclone
     *
     * @param source
     * @param target
     * @param options
     * @throws IOException
     */

    @Override
    public void copy(Path source, Path target, CopyOption... options) throws IOException {

        List<CopyOption> opts = Arrays.asList(options);
        boolean targetExists = Files.exists(target);

        if (targetExists) {
            if (Files.isRegularFile(target)) {
                if (opts.contains(StandardCopyOption.REPLACE_EXISTING)) {
                    Files.delete(target);
                } else {
                    throw new FileAlreadyExistsException("Copy failed -- target file already exists: " + target);
                }

            } else if (Files.isDirectory(target)) {
                target = target.resolve(source.getFileName());
            } else {
                throw new UnsupportedOperationException();
            }
        }

        String name1 = source.getFileName().toString();
        String name2 = target.getFileName().toString();
        if (!name1.equals(name2)) {
            throw new UnsupportedOperationException(
                    "Copy to a file with a different name is not supported: " + source.toString());
        }

        final DxPath dxSource = toDxPath(source);
        final DxFileSystem dxFileSystem = dxSource.getFileSystem();
        dxFileSystem.fileCopy(dxSource, toDxPath(target));
    }

    // TODO move
    @Override
    public void move(Path source, Path target, CopyOption... options) throws IOException {
        // see:
        //    container move    https://wiki.dnanexus.com/API-Specification-v1.0.0/Folders-and-Deletion#API-method%3A-%2Fclass-xxxx%2Fmove
        //    container rename  https://wiki.dnanexus.com/API-Specification-v1.0.0/Folders-and-Deletion#API-method%3A-%2Fclass-xxxx%2FrenameFolder
        //    file rename       https://wiki.dnanexus.com/API-Specification-v1.0.0/Name#API-method%3A-%2Fclass-xxxx%2Frename
    }

    @Override
    public boolean isSameFile(Path path, Path path2) throws IOException {
        return path.normalize().compareTo(path2.normalize()) == 0;
    }

    @Override
    public boolean isHidden(Path path) throws IOException {
        return readAttributes(path, DxFileAttributes.class).isHidden();
    }

    //TODO getFileStore
    @Override
    public FileStore getFileStore(Path path) throws IOException {
        throw new UnsupportedOperationException();
    }

    /**
     *
     * TODO checkAccess
     * http://openjdk.java.net/projects/nio/javadoc/java/nio/file/spi/FileSystemProvider.html#checkAccess(java.nio.file.Path, java.nio.file.AccessMode...)
     *
     * @param path
     * @param modes
     * @throws IOException
     */
    @Override
    public void checkAccess(Path path, AccessMode... modes) throws IOException {

        toDxPath(path).readAttributes();

    }

    @Override
    @SuppressWarnings("unchecked")
    public <V extends FileAttributeView> V getFileAttributeView(Path path, Class<V> type, LinkOption... options) {

        if (type == DxFileAttributeView.class) {
            return (V) new DxFileAttributeView(toDxPath(path));
        }

        return null;
    }

    @Override
    @SuppressWarnings("unchecked")
    public <V extends BasicFileAttributes> V readAttributes(Path path, Class<V> type, LinkOption... options)
            throws IOException {

        if (type == BasicFileAttributes.class || type == DxFileAttributes.class) {
            DxFileAttributeView view = new DxFileAttributeView(toDxPath(path));
            return (V) view.readAttributes();
        }

        return null;
    }

    @Override
    public Map<String, Object> readAttributes(Path path, String attributes, LinkOption... options)
            throws IOException {

        int pos = attributes.indexOf(':');
        if (pos != -1) {
            String view = attributes.substring(0, pos++);
            if (!view.equals(DxFileAttributeView.NAME)) {
                throw new IllegalArgumentException(
                        String.format("Illegal view for DnaNexus file system: '%s'", view));
            }

            attributes = attributes.substring(pos);
        }

        DxFileAttributeView view = new DxFileAttributeView(toDxPath(path));
        return view.readAttributes(attributes);

    }

    @Override
    public void setAttribute(Path path, String attribute, Object value, LinkOption... options) throws IOException {

        final DxPath dxPath = toDxPath(path);
        final DxFileSystem dxFileSystem = dxPath.getFileSystem();
        if (attribute.equals("tags")) {
            dxFileSystem.fileAddTags(dxPath, (String[]) value);

        } else if (attribute.equals("types")) {
            dxFileSystem.fileAddTypes(dxPath, (String[]) value);
        } else {
            throw new UnsupportedOperationException(String.format("Attribute '%s' cannot be changed", attribute));
        }

    }

    // Checks that the given file is a UnixPath
    static final DxPath toDxPath(Path path) {
        if (path == null) {
            throw new NullPointerException();
        }

        if (!(path instanceof DxPath)) {
            throw new ProviderMismatchException();
        }

        return (DxPath) path;
    }

    /**
     * @return The current installed instance of the {@code DxFileSystemProvider} or {@code null} if
     * no DX provider is installed
     */

    static DxFileSystemProvider instance = null;

    static DxFileSystemProvider defaultInstance() {

        if (instance != null)
            return instance;

        for (FileSystemProvider provider : FileSystemProvider.installedProviders()) {
            if (provider instanceof DxFileSystemProvider) {
                return instance = (DxFileSystemProvider) provider;
            }
        }

        return null;
    }

}