Example usage for org.apache.hadoop.fs FileSystem getHomeDirectory

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem getHomeDirectory.

Prototype

public Path getHomeDirectory() 

Source Link

Document

Return the current user's home directory in this FileSystem.

Usage

From source file:edu.nyu.vida.data_polygamy.utils.FrameworkUtils.java

License:BSD License

public static String searchAggregatesHeader(final String name, Configuration conf, boolean s3)
        throws IOException {

    PathFilter filter = new PathFilter() {

        @Override//from   w w w . jav a 2  s . c  o m
        public boolean accept(Path arg0) {
            if (arg0.getName().contains(name + "-"))
                return true;
            return false;
        }
    };

    Path path = null;
    FileSystem fs = null;

    if (s3) {
        path = new Path(conf.get("bucket") + preProcessingDir);
        fs = FileSystem.get(path.toUri(), conf);
    } else {
        fs = FileSystem.get(new Configuration());
        path = new Path(fs.getHomeDirectory() + "/" + preProcessingDir);
    }

    FileStatus[] status = fs.listStatus(path, filter);

    if (s3)
        fs.close();

    String fileName = "";
    for (FileStatus fileStatus : status) {
        fileName = fileStatus.getPath().getName();
        if (fileName.endsWith(".aggregates"))
            return fileName;
    }

    return null;
}

---

From source file:edu.nyu.vida.data_polygamy.utils.FrameworkUtils.java

License:BSD License

public static String[] searchAggregates(final String name, Configuration conf, boolean s3) throws IOException {

    PathFilter filter = new PathFilter() {

        @Override//  w  w w  .j  a  v a2s.  c  om
        public boolean accept(Path arg0) {
            if (arg0.getName().contains("_SUCCESS"))
                return false;
            return true;
        }
    };

    Path path = null;
    FileSystem fs = null;

    if (s3) {
        path = new Path(conf.get("bucket") + aggregatesDir + "/" + name);
        fs = FileSystem.get(path.toUri(), conf);
    } else {
        fs = FileSystem.get(new Configuration());
        path = new Path(fs.getHomeDirectory() + "/" + aggregatesDir + "/" + name);
    }

    FileStatus[] status;

    try {
        status = fs.listStatus(path, filter);
    } catch (FileNotFoundException e) {
        return new String[0];
    }

    if (s3)
        fs.close();

    String[] names = new String[status.length];
    String fileName = "";
    for (int i = 0; i < status.length; i++) {
        fileName = status[i].getPath().getName();
        names[i] = fileName;
    }

    return names;
}

---

From source file:edu.nyu.vida.data_polygamy.utils.FrameworkUtils.java

License:BSD License

public static String[] searchIndex(final String name, Configuration conf, boolean s3) throws IOException {

    PathFilter filter = new PathFilter() {

        @Override//w ww.  j a  v a 2s  . c o  m
        public boolean accept(Path arg0) {
            if (arg0.getName().contains("_SUCCESS"))
                return false;
            return true;
        }
    };

    Path path = null;
    FileSystem fs = null;

    if (s3) {
        path = new Path(conf.get("bucket") + indexDir + "/" + name);
        fs = FileSystem.get(path.toUri(), conf);
    } else {
        fs = FileSystem.get(new Configuration());
        path = new Path(fs.getHomeDirectory() + "/" + indexDir + "/" + name);
    }

    FileStatus[] status;

    try {
        status = fs.listStatus(path, filter);
    } catch (FileNotFoundException e) {
        return new String[0];
    }

    if (s3)
        fs.close();

    String[] names = new String[status.length];
    String fileName = "";
    for (int i = 0; i < status.length; i++) {
        fileName = status[i].getPath().getName();
        names[i] = fileName;
    }

    return names;
}

---

From source file:edu.nyu.vida.data_polygamy.utils.FrameworkUtils.java

License:BSD License

public static String[] searchDataAttributes(final String name, Configuration conf, boolean s3)
        throws IOException {

    PathFilter filter = new PathFilter() {

        @Override//from  w w w. j  a v a2  s.  c o m
        public boolean accept(Path arg0) {
            if (arg0.getName().contains("_SUCCESS"))
                return false;
            return true;
        }
    };

    Path path = null;
    FileSystem fs = null;

    if (s3) {
        path = new Path(conf.get("bucket") + dataAttributesDir + "/" + name);
        fs = FileSystem.get(path.toUri(), conf);
    } else {
        fs = FileSystem.get(new Configuration());
        path = new Path(fs.getHomeDirectory() + "/" + dataAttributesDir + "/" + name);
    }

    FileStatus[] status;

    try {
        status = fs.listStatus(path, filter);
    } catch (FileNotFoundException e) {
        return new String[0];
    }

    if (s3)
        fs.close();

    String[] names = new String[status.length];
    String fileName = "";
    for (int i = 0; i < status.length; i++) {
        fileName = status[i].getPath().getName();
        names[i] = fileName;
    }

    return names;
}

---

From source file:edu.nyu.vida.data_polygamy.utils.GetMergeFiles.java

License:BSD License

public static void main(String[] args) throws IllegalArgumentException, IOException, URISyntaxException {
    String fromDirectory = args[0];
    String toEventsDirectory = args[1];
    String toOutliersDirectory = args[2];
    String metadataFile = args[3];

    // Detecting datasets.

    HashSet<String> datasets = new HashSet<String>();

    FileReader fileReader = new FileReader(metadataFile);
    BufferedReader bufferedReader = new BufferedReader(fileReader);

    String line;//from w  w  w  . j  av  a  2  s  . com
    while ((line = bufferedReader.readLine()) != null) {
        String[] parts = line.split(",");
        datasets.add(parts[0]);
    }
    bufferedReader.close();

    // Downloading relationships.

    String relationshipPatternStr = "([a-zA-Z0-9]{4}\\-[a-zA-Z0-9]{4})\\-([a-zA-Z0-9]{4}\\-[a-zA-Z0-9]{4})";
    Pattern relationshipPattern = Pattern.compile(relationshipPatternStr);

    Configuration conf = new Configuration();
    FileSystem fs = FileSystem.get(conf);
    FileSystem localFS = FileSystem.getLocal(conf);

    for (FileStatus status : fs.listStatus(new Path(fs.getHomeDirectory() + "/" + fromDirectory))) {
        if (!status.isDirectory()) {
            continue;
        }
        Path file = status.getPath();

        Matcher m = relationshipPattern.matcher(file.getName());
        if (!m.find())
            continue;

        String ds1 = m.group(1);
        String ds2 = m.group(2);

        if (!datasets.contains(ds1))
            continue;
        if (!datasets.contains(ds2))
            continue;

        for (FileStatus statusDir : fs.listStatus(file)) {
            if (!statusDir.isDirectory()) {
                continue;
            }

            Path fromPath = statusDir.getPath();
            String toPathStr;
            if (fromPath.getName().contains("events")) {
                toPathStr = toEventsDirectory + "/" + fromPath.getParent().getName() + "-" + fromPath.getName();
            } else {
                toPathStr = toOutliersDirectory + "/" + fromPath.getParent().getName() + "-"
                        + fromPath.getName();
            }
            Path toPath = new Path(toPathStr);

            System.out.println("Copying:");
            System.out.println("  From: " + fromPath.toString());
            System.out.println("  To: " + toPath.toString());

            FileUtil.copyMerge(fs, // HDFS File System
                    fromPath, // HDFS path
                    localFS, // Local File System
                    toPath, // Local Path
                    false, // Do not delete HDFS path
                    conf, // Configuration
                    null);
        }
    }
}

---

From source file:edu.uci.ics.asterix.aoya.AsterixYARNClient.java

License:Apache License

/**
 * Initialize and register the application attempt with the YARN ResourceManager.
 * //from w ww. j a v a2  s  . c o m
 * @return
 * @throws IOException
 * @throws YarnException
 */
public YarnClientApplication makeApplicationContext() throws IOException, YarnException {

    //first check to see if an instance already exists.
    FileSystem fs = FileSystem.get(conf);
    Path lock = new Path(fs.getHomeDirectory(), CONF_DIR_REL + instanceFolder + instanceLock);
    LOG.info("Running Deployment");
    yarnClient.start();
    if (fs.exists(lock)) {
        ApplicationId lockAppId = getLockFile();
        try {
            ApplicationReport previousAppReport = yarnClient.getApplicationReport(lockAppId);
            YarnApplicationState prevStatus = previousAppReport.getYarnApplicationState();
            if (!(prevStatus == YarnApplicationState.FAILED || prevStatus == YarnApplicationState.KILLED
                    || prevStatus == YarnApplicationState.FINISHED) && mode != Mode.DESTROY
                    && mode != Mode.BACKUP && mode != Mode.RESTORE) {
                throw new IllegalStateException("Instance is already running in: " + lockAppId);
            } else if (mode != Mode.DESTROY && mode != Mode.BACKUP && mode != Mode.RESTORE) {
                //stale lock file
                LOG.warn("Stale lockfile detected. Instance attempt " + lockAppId
                        + " may have exited abnormally");
                deleteLockFile();
            }
        } catch (YarnException e) {
            LOG.warn(
                    "Stale lockfile detected, but the RM has no record of this application's last run. This is normal if the cluster was restarted.");
            deleteLockFile();
        }
    }

    // Get a new application id
    YarnClientApplication app = yarnClient.createApplication();
    GetNewApplicationResponse appResponse = app.getNewApplicationResponse();
    int maxMem = appResponse.getMaximumResourceCapability().getMemory();
    LOG.info("Max mem capabililty of resources in this cluster " + maxMem);

    // A resource ask cannot exceed the max.
    if (amMemory > maxMem) {
        LOG.info("AM memory specified above max threshold of cluster. Using max value." + ", specified="
                + amMemory + ", max=" + maxMem);
        amMemory = maxMem;
    }

    // set the application name
    ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();
    appContext.setApplicationName(appName);

    return app;
}

---

From source file:edu.uci.ics.asterix.aoya.AsterixYARNClient.java

License:Apache License

/**
 * Upload the Asterix cluster description on to the DFS. This will persist the state of the instance.
 * /* w  w  w  .  j  a  v  a  2 s .c  o  m*/
 * @return
 * @throws YarnException
 * @throws IOException
 */
private List<DFSResourceCoordinate> deployConfig() throws YarnException, IOException {

    FileSystem fs = FileSystem.get(conf);
    List<DFSResourceCoordinate> resources = new ArrayList<DFSResourceCoordinate>(2);

    String pathSuffix = CONF_DIR_REL + instanceFolder + CONFIG_DEFAULT_NAME;
    Path dstConf = new Path(fs.getHomeDirectory(), pathSuffix);
    FileStatus destStatus;
    try {
        destStatus = fs.getFileStatus(dstConf);
    } catch (IOException e) {
        throw new YarnException("Asterix instance by that name does not appear to exist in DFS");
    }
    LocalResource asterixConfLoc = Records.newRecord(LocalResource.class);
    asterixConfLoc.setType(LocalResourceType.FILE);
    asterixConfLoc.setVisibility(LocalResourceVisibility.PRIVATE);
    asterixConfLoc.setResource(ConverterUtils.getYarnUrlFromPath(dstConf));
    asterixConfLoc.setTimestamp(destStatus.getModificationTime());

    DFSResourceCoordinate conf = new DFSResourceCoordinate();
    conf.envs.put(dstConf.toUri().toString(), AConstants.CONFLOCATION);
    conf.envs.put(Long.toString(asterixConfLoc.getSize()), AConstants.CONFLEN);
    conf.envs.put(Long.toString(asterixConfLoc.getTimestamp()), AConstants.CONFTIMESTAMP);
    conf.name = CONFIG_DEFAULT_NAME;
    conf.res = asterixConfLoc;
    resources.add(conf);

    return resources;

}

---

From source file:edu.uci.ics.asterix.aoya.AsterixYARNClient.java

License:Apache License

/**
 * Install the current Asterix parameters to the DFS. This can be modified via alter.
 * /*from   ww  w. j  a v a 2 s .  c  o  m*/
 * @throws YarnException
 * @throws IOException
 */
private void installConfig() throws YarnException, IOException {
    FileSystem fs = FileSystem.get(conf);
    String pathSuffix = CONF_DIR_REL + instanceFolder + CONFIG_DEFAULT_NAME;
    Path dstConf = new Path(fs.getHomeDirectory(), pathSuffix);
    try {
        fs.getFileStatus(dstConf);
        if (mode == Mode.INSTALL) {
            throw new IllegalStateException("Instance with this name already exists.");
        }
    } catch (FileNotFoundException e) {
        if (mode == Mode.START) {
            throw new IllegalStateException("Instance does not exist for this user", e);
        }
    }
    if (mode == Mode.INSTALL) {
        Path src = new Path(asterixConf);
        fs.copyFromLocalFile(false, true, src, dstConf);
    }

}

---

From source file:edu.uci.ics.asterix.aoya.AsterixYARNClient.java

License:Apache License

/**
 * Upload External libraries and functions to HDFS for an instance to use when started
 * @throws IllegalStateException/*from  w ww  .  j ava2  s .c  om*/
 * @throws IOException
 */

private void installExtLibs() throws IllegalStateException, IOException {
    FileSystem fs = FileSystem.get(conf);
    if (!instanceExists()) {
        throw new IllegalStateException("No instance by name " + instanceName + " found.");
    }
    if (isRunning()) {
        throw new IllegalStateException(
                "Instance " + instanceName + " is running. Please stop it before installing any libraries.");
    }
    String libPathSuffix = CONF_DIR_REL + instanceFolder + "library" + Path.SEPARATOR + libDataverse
            + Path.SEPARATOR;
    Path src = new Path(extLibs);
    String fullLibPath = libPathSuffix + src.getName();
    Path libFilePath = new Path(fs.getHomeDirectory(), fullLibPath);
    LOG.info("Copying Asterix external library to DFS");
    fs.copyFromLocalFile(false, true, src, libFilePath);
}

---

From source file:edu.uci.ics.asterix.aoya.AsterixYARNClient.java

License:Apache License

/**
 * Finds the minimal classes and JARs needed to start the AM only.
 * @return Resources the AM needs to start on the initial container.
 * @throws IllegalStateException/*from   w w w . j a  v  a 2s. c  om*/
 * @throws IOException
 */
private List<DFSResourceCoordinate> installAmLibs() throws IllegalStateException, IOException {
    List<DFSResourceCoordinate> resources = new ArrayList<DFSResourceCoordinate>(2);
    FileSystem fs = FileSystem.get(conf);
    String fullLibPath = CONF_DIR_REL + instanceFolder + "am_jars" + Path.SEPARATOR;
    String[] cp = System.getProperty("java.class.path").split(System.getProperty("path.separator"));
    String asterixJarPattern = "^(asterix).*(jar)$"; //starts with asterix,ends with jar
    String commonsJarPattern = "^(commons).*(jar)$";
    String surefireJarPattern = "^(surefire).*(jar)$"; //for maven tests
    String jUnitTestPattern = "^(asterix-yarn" + File.separator + "target)$";

    LOG.info(File.separator);
    for (String j : cp) {
        String[] pathComponents = j.split(Pattern.quote(File.separator));
        LOG.info(j);
        LOG.info(pathComponents[pathComponents.length - 1]);
        if (pathComponents[pathComponents.length - 1].matches(asterixJarPattern)
                || pathComponents[pathComponents.length - 1].matches(commonsJarPattern)
                || pathComponents[pathComponents.length - 1].matches(surefireJarPattern)
                || pathComponents[pathComponents.length - 1].matches(jUnitTestPattern)) {
            LOG.info("Loading JAR/classpath: " + j);
            File f = new File(j);
            Path dst = new Path(fs.getHomeDirectory(), fullLibPath + f.getName());
            if (!fs.exists(dst) || refresh) {
                fs.copyFromLocalFile(false, true, new Path(f.getAbsolutePath()), dst);
            }
            FileStatus dstSt = fs.getFileStatus(dst);
            LocalResource amLib = Records.newRecord(LocalResource.class);
            amLib.setType(LocalResourceType.FILE);
            amLib.setVisibility(LocalResourceVisibility.PRIVATE);
            amLib.setResource(ConverterUtils.getYarnUrlFromPath(dst));
            amLib.setTimestamp(dstSt.getModificationTime());
            amLib.setSize(dstSt.getLen());
            DFSResourceCoordinate amLibCoord = new DFSResourceCoordinate();
            amLibCoord.res = amLib;
            amLibCoord.name = f.getName();
            if (f.getName().contains("asterix-yarn") || f.getName().contains("surefire")) {
                amLibCoord.envs.put(dst.toUri().toString(), AConstants.APPLICATIONMASTERJARLOCATION);
                amLibCoord.envs.put(Long.toString(dstSt.getLen()), AConstants.APPLICATIONMASTERJARLEN);
                amLibCoord.envs.put(Long.toString(dstSt.getModificationTime()),
                        AConstants.APPLICATIONMASTERJARTIMESTAMP);
            }
            resources.add(amLibCoord);
        }

    }
    if (resources.size() == 0) {
        throw new IOException("Required JARs are missing. Please check your directory structure");
    }
    return resources;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• »