Example usage for org.apache.hadoop.fs Path getName

List of usage examples for org.apache.hadoop.fs Path getName

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. Path
  6. getName

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path getName.

Prototype

public String getName()

Source Link

Document

Returns the final component of this path.

Usage

From source file:edu.iu.daal_cov.COVDaalCollectiveMapper.java

License:Apache License

protected void mapCollective(KeyValReader reader, Context context) throws IOException, InterruptedException {
    long startTime = System.currentTimeMillis();
    List<String> trainingDataFiles = new LinkedList<String>();

    //splitting files between mapper

    while (reader.nextKeyValue()) {
        String key = reader.getCurrentKey();
        String value = reader.getCurrentValue();
        LOG.info("Key: " + key + ", Value: " + value);
        System.out.println("file name : " + value);
        trainingDataFiles.add(value);//from   ww  w. ja v a2  s .  c o m
    }

    Configuration conf = context.getConfiguration();

    Path pointFilePath = new Path(trainingDataFiles.get(0));
    System.out.println("path = " + pointFilePath.getName());
    FileSystem fs = pointFilePath.getFileSystem(conf);
    FSDataInputStream in = fs.open(pointFilePath);

    runCOV(trainingDataFiles, conf, context);
    LOG.info("Total iterations in master view: " + (System.currentTimeMillis() - startTime));
    this.freeMemory();
    this.freeConn();
    System.gc();
}

From source file:edu.iu.daal_linreg.LinRegDaalCollectiveMapper.java

License:Apache License

protected void mapCollective(KeyValReader reader, Context context) throws IOException, InterruptedException {
    long startTime = System.currentTimeMillis();
    List<String> trainingDataFiles = new LinkedList<String>();

    //splitting files between mapper

    while (reader.nextKeyValue()) {
        String key = reader.getCurrentKey();
        String value = reader.getCurrentValue();
        LOG.info("Key: " + key + ", Value: " + value);
        System.out.println("file name : " + value);
        trainingDataFiles.add(value);//from   w w w . j av a 2  s. c om
    }

    Configuration conf = context.getConfiguration();

    Path pointFilePath = new Path(trainingDataFiles.get(0));
    System.out.println("path = " + pointFilePath.getName());
    FileSystem fs = pointFilePath.getFileSystem(conf);
    FSDataInputStream in = fs.open(pointFilePath);

    runLinReg(trainingDataFiles, conf, context);
    LOG.info("Total iterations in master view: " + (System.currentTimeMillis() - startTime));
    this.freeMemory();
    this.freeConn();
    System.gc();
}

From source file:edu.iu.daal_mom.MOMDaalCollectiveMapper.java

License:Apache License

protected void mapCollective(KeyValReader reader, Context context) throws IOException, InterruptedException {
    long startTime = System.currentTimeMillis();
    List<String> trainingDataFiles = new LinkedList<String>();

    //splitting files between mapper

    while (reader.nextKeyValue()) {
        String key = reader.getCurrentKey();
        String value = reader.getCurrentValue();
        LOG.info("Key: " + key + ", Value: " + value);
        System.out.println("file name : " + value);
        trainingDataFiles.add(value);//from  w ww.j a  v a  2  s  .c  o m
    }

    Configuration conf = context.getConfiguration();

    Path pointFilePath = new Path(trainingDataFiles.get(0));
    System.out.println("path = " + pointFilePath.getName());
    FileSystem fs = pointFilePath.getFileSystem(conf);
    FSDataInputStream in = fs.open(pointFilePath);

    runMOM(trainingDataFiles, conf, context);
    LOG.info("Total iterations in master view: " + (System.currentTimeMillis() - startTime));
    this.freeMemory();
    this.freeConn();
    System.gc();
}

From source file:edu.iu.daal_naive.NaiveDaalCollectiveMapper.java

License:Apache License

protected void mapCollective(KeyValReader reader, Context context) throws IOException, InterruptedException {

    // long startTime = System.currentTimeMillis();
    List<String> trainingDataFiles = new LinkedList<String>();

    //splitting files between mapper
    while (reader.nextKeyValue()) {
        String key = reader.getCurrentKey();
        String value = reader.getCurrentValue();
        LOG.info("Key: " + key + ", Value: " + value);
        System.out.println("file name : " + value);
        trainingDataFiles.add(value);//from www. jav a  2  s . com
    }

    Configuration conf = context.getConfiguration();

    Path pointFilePath = new Path(trainingDataFiles.get(0));
    System.out.println("path = " + pointFilePath.getName());
    FileSystem fs = pointFilePath.getFileSystem(conf);
    FSDataInputStream in = fs.open(pointFilePath);

    runNaive(trainingDataFiles, conf, context);
    // LOG.info("Total time of iterations in master view: "
    //   + (System.currentTimeMillis() - startTime));
    this.freeMemory();
    this.freeConn();
    System.gc();
}

From source file:edu.mit.jwi.data.FileProvider.java

License:Creative Commons License

/**
 * Make a File from Path// ww w  .  j  ava  2  s.  c  o  m
 * 
 * from
 * http://stackoverflow.com/questions/3444313/how-to-convert-a-hadoop-path-
 * object-into-a-java-file-object
 * 
 * @param some_path
 * @param conf
 * @author Mauro Pelucchi
 * @since JWI 2.3.3-hadoop
 * @return
 */
public static File MakeFileFromPath(Path hdfsPath, Configuration conf) {
    try {
        FileSystem fs = FileSystem.get(hdfsPath.toUri(), conf);
        File tempFolder = File.createTempFile(hdfsPath.getName(), "");
        if (!(tempFolder.delete())) {
            throw new IOException("Could not delete temp file: " + tempFolder.getAbsolutePath());
        }

        if (!(tempFolder.mkdir())) {
            throw new IOException("Could not create temp directory: " + tempFolder.getAbsolutePath());
        }
        FileStatus[] status = fs.listStatus(hdfsPath);
        for (int i = 0; i < status.length; i++) {
            System.out.println("------------------------------------");
            if (status[i].isFile()) {
                System.out.println(status[i].getPath());
                fs.copyToLocalFile(false, status[i].getPath(), new Path(tempFolder.getAbsolutePath()));
                //System.out.println(ReadFileFromHdfs(fs, status[i].getPath()));
            }
        }

        tempFolder.deleteOnExit();

        File[] files = tempFolder.listFiles();
        for (int i = 0; i < files.length; i++) {
            System.out.println("------------------------------------");
            System.out.println(files[i].getPath());
            //System.out.println(ReadFile(files[i].getPath()));
            if (files[i].getName().startsWith(".")) {
                System.out.println("Delete --> " + files[i].getPath());
                if (!(files[i].delete())) {
                    throw new IOException("Could not delete temp file: " + files[i].getAbsolutePath());
                }
            }
        }

        return tempFolder;
    } catch (Exception e) {
        e.printStackTrace();
    }
    return null;
}

From source file:edu.nyu.vida.data_polygamy.utils.FrameworkUtils.java

License:BSD License

public static void removeReducerFiles(final String fileName) throws IOException {
    PathFilter filter = new PathFilter() {

        @Override/*from  ww  w .j  ava  2  s. c  om*/
        public boolean accept(Path arg0) {
            if (arg0.getName().contains(fileName))
                return true;
            return false;
        }
    };

    FileSystem fs = FileSystem.get(new Configuration());
    Path path = new Path(fs.getHomeDirectory() + "/relationships");

    FileStatus[] status = fs.listStatus(path, filter);

    for (FileStatus fileStatus : status) {
        fs.delete(new Path(fs.getHomeDirectory() + "/relationships/" + fileStatus.getPath().getName()), true);
    }
}

From source file:edu.nyu.vida.data_polygamy.utils.FrameworkUtils.java

License:BSD License

public static String searchPreProcessing(final String name, Configuration conf, boolean s3) throws IOException {

    PathFilter filter = new PathFilter() {

        @Override// www .j  a  v a  2s .c o m
        public boolean accept(Path arg0) {
            if (arg0.getName().contains(name + "-"))
                return true;
            return false;
        }
    };

    Path path = null;
    FileSystem fs = null;

    if (s3) {
        path = new Path(conf.get("bucket") + preProcessingDir);
        fs = FileSystem.get(path.toUri(), conf);
    } else {
        fs = FileSystem.get(new Configuration());
        path = new Path(fs.getHomeDirectory() + "/" + preProcessingDir);
    }

    FileStatus[] status = fs.listStatus(path, filter);

    if (s3)
        fs.close();

    String preProcessingFile = null;
    boolean aggregatesFound = false;
    String fileName = "";
    for (FileStatus fileStatus : status) {
        fileName = fileStatus.getPath().getName();
        if (!fileName.endsWith(".aggregates"))
            preProcessingFile = fileName;
        else if (fileName.endsWith(".aggregates"))
            aggregatesFound = true;
    }

    if (!aggregatesFound)
        return null;

    return preProcessingFile;
}

From source file:edu.nyu.vida.data_polygamy.utils.FrameworkUtils.java

License:BSD License

public static String searchAggregatesHeader(final String name, Configuration conf, boolean s3)
        throws IOException {

    PathFilter filter = new PathFilter() {

        @Override// w ww.  j ava 2s  . c om
        public boolean accept(Path arg0) {
            if (arg0.getName().contains(name + "-"))
                return true;
            return false;
        }
    };

    Path path = null;
    FileSystem fs = null;

    if (s3) {
        path = new Path(conf.get("bucket") + preProcessingDir);
        fs = FileSystem.get(path.toUri(), conf);
    } else {
        fs = FileSystem.get(new Configuration());
        path = new Path(fs.getHomeDirectory() + "/" + preProcessingDir);
    }

    FileStatus[] status = fs.listStatus(path, filter);

    if (s3)
        fs.close();

    String fileName = "";
    for (FileStatus fileStatus : status) {
        fileName = fileStatus.getPath().getName();
        if (fileName.endsWith(".aggregates"))
            return fileName;
    }

    return null;
}

From source file:edu.nyu.vida.data_polygamy.utils.FrameworkUtils.java

License:BSD License

public static String[] searchAggregates(final String name, Configuration conf, boolean s3) throws IOException {

    PathFilter filter = new PathFilter() {

        @Override/*w  ww. j a v a2s . co m*/
        public boolean accept(Path arg0) {
            if (arg0.getName().contains("_SUCCESS"))
                return false;
            return true;
        }
    };

    Path path = null;
    FileSystem fs = null;

    if (s3) {
        path = new Path(conf.get("bucket") + aggregatesDir + "/" + name);
        fs = FileSystem.get(path.toUri(), conf);
    } else {
        fs = FileSystem.get(new Configuration());
        path = new Path(fs.getHomeDirectory() + "/" + aggregatesDir + "/" + name);
    }

    FileStatus[] status;

    try {
        status = fs.listStatus(path, filter);
    } catch (FileNotFoundException e) {
        return new String[0];
    }

    if (s3)
        fs.close();

    String[] names = new String[status.length];
    String fileName = "";
    for (int i = 0; i < status.length; i++) {
        fileName = status[i].getPath().getName();
        names[i] = fileName;
    }

    return names;
}

From source file:edu.nyu.vida.data_polygamy.utils.FrameworkUtils.java

License:BSD License

public static String[] searchIndex(final String name, Configuration conf, boolean s3) throws IOException {

    PathFilter filter = new PathFilter() {

        @Override//from ww  w . jav  a2s  . com
        public boolean accept(Path arg0) {
            if (arg0.getName().contains("_SUCCESS"))
                return false;
            return true;
        }
    };

    Path path = null;
    FileSystem fs = null;

    if (s3) {
        path = new Path(conf.get("bucket") + indexDir + "/" + name);
        fs = FileSystem.get(path.toUri(), conf);
    } else {
        fs = FileSystem.get(new Configuration());
        path = new Path(fs.getHomeDirectory() + "/" + indexDir + "/" + name);
    }

    FileStatus[] status;

    try {
        status = fs.listStatus(path, filter);
    } catch (FileNotFoundException e) {
        return new String[0];
    }

    if (s3)
        fs.close();

    String[] names = new String[status.length];
    String fileName = "";
    for (int i = 0; i < status.length; i++) {
        fileName = status[i].getPath().getName();
        names[i] = fileName;
    }

    return names;
}