Example usage for org.apache.hadoop.fs FileSystem open

List of usage examples for org.apache.hadoop.fs FileSystem open

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileSystem
  6. open

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem open.

Prototype

public FSDataInputStream open(PathHandle fd) throws IOException

Source Link

Document

Open an FSDataInputStream matching the PathHandle instance.

Usage

From source file:com.hhscyber.nl.tweets.impact.ImpactReducer.java

@Override
protected void setup(Context context) throws IOException, InterruptedException {
    count++;/*from  w w w  .j av  a2 s .c om*/
    System.out.println("Setup called, count = " + count);
    FileSystem hdfs = FileSystem.get(context.getConfiguration());
    try {
        InputStream model1 = hdfs.open(new Path(path + "en-ner-organization.bin"));
        tokenModel = new TokenNameFinderModel(model1);
        model1.close();
        InputStream model2 = hdfs.open(new Path(path + "en-token.bin"));
        tokenizerModel = new TokenizerModel(model2);
        model2.close();
        InputStream model3 = hdfs.open(new Path(path + "en-sent.bin"));
        sentenceModel = new SentenceModel(model3);
        model3.close();
    } catch (IllegalArgumentException | IOException ex) {
        Logger.getLogger(ImpactReducer.class.getName()).log(Level.SEVERE, null, ex);
    }
    super.setup(context); //To change body of generated methods, choose Tools | Templates.
}

From source file:com.hhscyber.nl.tweets.locationorig.LocationOriginReducer.java

@Override
protected void setup(Context context) throws IOException, InterruptedException {
    count++;//w w w . ja v  a2  s . c o m
    System.out.println("Setup called, count = " + count);
    FileSystem hdfs = FileSystem.get(context.getConfiguration());
    try {
        InputStream model1 = hdfs.open(new Path(path + "en-ner-location.bin"));
        tokenModel = new TokenNameFinderModel(model1);
        model1.close();
        InputStream model2 = hdfs.open(new Path(path + "en-token.bin"));
        tokenizerModel = new TokenizerModel(model2);
        model2.close();
        InputStream model3 = hdfs.open(new Path(path + "en-sent.bin"));
        sentenceModel = new SentenceModel(model3);
        model3.close();
    } catch (IllegalArgumentException | IOException ex) {
        Logger.getLogger(LocationOriginReducer.class.getName()).log(Level.SEVERE, null, ex);
    }
    super.setup(context); //To change body of generated methods, choose Tools | Templates.
}

From source file:com.hortonworks.pso.data.generator.mapreduce.DataGenMapper.java

License:Apache License

protected void setup(Context context) {
    // Get the conf location from the job conf.
    String config = context.getConfiguration().get("json.cfg");

    System.out.println("Config File: " + config);

    // Read the Config from the path.
    FileSystem FS1 = null;
    FSDataInputStream fsdis = null;//from  w ww .j a va  2s  .c o  m
    try {
        FS1 = FileSystem.get(context.getConfiguration());

        Path path = new Path(config);

        fsdis = FS1.open(path);

        ObjectMapper mapper = new ObjectMapper();
        JsonNode root = mapper.readValue(fsdis, JsonNode.class);

        recordGenerator = new RecordGenerator(root);

    } catch (IOException e) {
        e.printStackTrace();
        throw new RuntimeException(e);
    } finally {
        IOUtils.closeStream(fsdis);
    }

}

From source file:com.hortonworks.pso.data.generator.mapreduce.KafkaDataGenMapper.java

License:Apache License

protected void setup(Context context) {
    // Get the conf location from the job conf.
    String config = context.getConfiguration().get(CONFIG_FILE);

    System.out.println("Config File: " + config);

    // Read the Config from the path.
    FileSystem FS1 = null;
    FSDataInputStream fsdis = null;//w  w  w.j a  v a2  s.  com
    try {
        FS1 = FileSystem.get(context.getConfiguration());

        Path path = new Path(config);

        fsdis = FS1.open(path);

        ObjectMapper mapper = new ObjectMapper();
        JsonNode root = mapper.readValue(fsdis, JsonNode.class);

        recordGenerator = new RecordGenerator(root);

    } catch (IOException e) {
        e.printStackTrace();
        throw new RuntimeException(e);
    } finally {
        IOUtils.closeStream(fsdis);
    }

}

From source file:com.huayu.metis.flume.sink.hdfs.AvroEventSerializer.java

License:Apache License

private Schema loadFromUrl(String schemaUrl) throws IOException {
    Configuration conf = new Configuration();
    Schema.Parser parser = new Schema.Parser();
    if (schemaUrl.toLowerCase().startsWith("hdfs://")) {
        FileSystem fs = FileSystem.get(conf);
        FSDataInputStream input = null;/*from w  ww .  ja  v a  2s.  c  om*/
        try {
            input = fs.open(new Path(schemaUrl));
            return parser.parse(input);
        } finally {
            if (input != null) {
                input.close();
            }
        }
    } else {
        InputStream is = null;
        try {
            is = new URL(schemaUrl).openStream();
            return parser.parse(is);
        } finally {
            if (is != null) {
                is.close();
            }
        }
    }
}

From source file:com.hurence.logisland.service.cache.CSVKeyValueCacheService.java

License:Apache License

private InputStream initFromUri(String dbUri) {
    Configuration conf = new Configuration();

    String hdfsUri = conf.get("fs.defaultFS");
    getLogger().info("Default HDFS URI: " + hdfsUri);

    // Set HADOOP user to same as current suer
    String hadoopUser = System.getProperty("user.name");
    System.setProperty("HADOOP_USER_NAME", hadoopUser);
    System.setProperty("hadoop.home.dir", "/");

    // Get the HDFS filesystem
    FileSystem fs = null;
    try {/*from w  ww.  j ava  2 s .co m*/
        fs = FileSystem.get(URI.create(hdfsUri), conf);

        // Create a path to config file and init input stream
        Path hdfsReadpath = new Path(dbUri);
        getLogger().info("Reading DB file from HDFS at: " + dbUri);
        return fs.open(hdfsReadpath);
    } catch (IOException e) {
        logger.info(e.toString());
        return null;
    }

}

From source file:com.hurence.logisland.service.iptogeo.maxmind.MaxmindIpToGeoService.java

License:Apache License

private void initFromUri(String dbUri) throws Exception {
    Configuration conf = new Configuration();

    String hdfsUri = conf.get("fs.defaultFS");
    getLogger().info("Default HDFS URI: " + hdfsUri);

    // Set HADOOP user to same as current suer
    String hadoopUser = System.getProperty("user.name");
    System.setProperty("HADOOP_USER_NAME", hadoopUser);
    System.setProperty("hadoop.home.dir", "/");

    // Get the HDFS filesystem
    FileSystem fs = FileSystem.get(URI.create(hdfsUri), conf);

    // Create a path to config file and init input stream
    Path hdfsReadpath = new Path(dbUri);
    getLogger().info("Reading Maxmind DB file from HDFS at: " + dbUri);
    FSDataInputStream inputStream = fs.open(hdfsReadpath);

    long start = System.currentTimeMillis();
    final DatabaseReader databaseReader = createReader(new DatabaseReader.Builder(inputStream));
    long stop = System.currentTimeMillis();
    getLogger().info("Completed loading of Maxmind Geo Database in {} milliseconds.",
            new Object[] { stop - start });
    databaseReaderRef.set(databaseReader);
}

From source file:com.ibm.bi.dml.api.DMLScript.java

License:Open Source License

/**
 * /*from w  ww .  j  a v a2s .  c  o  m*/
 * @param argname
 * @param arg
 * @return
 * @throws IOException 
 * @throws LanguageException 
 */
protected static String readDMLScript(String argname, String script) throws IOException, LanguageException {
    boolean fromFile = argname.equals("-f") ? true : false;
    String dmlScriptStr = null;

    if (fromFile) {
        //read DML script from file
        if (script == null)
            throw new LanguageException("DML script path was not specified!");

        StringBuilder sb = new StringBuilder();
        BufferedReader in = null;
        try {
            //read from hdfs or gpfs file system
            if (script.startsWith("hdfs:") || script.startsWith("gpfs:")) {
                if (!LocalFileUtils.validateExternalFilename(script, true))
                    throw new LanguageException("Invalid (non-trustworthy) hdfs filename.");
                FileSystem fs = FileSystem.get(ConfigurationManager.getCachedJobConf());
                Path scriptPath = new Path(script);
                in = new BufferedReader(new InputStreamReader(fs.open(scriptPath)));
            }
            // from local file system
            else {
                if (!LocalFileUtils.validateExternalFilename(script, false))
                    throw new LanguageException("Invalid (non-trustworthy) local filename.");
                in = new BufferedReader(new FileReader(script));
            }

            //core script reading
            String tmp = null;
            while ((tmp = in.readLine()) != null) {
                sb.append(tmp);
                sb.append("\n");
            }
        } catch (IOException ex) {
            LOG.error("Failed to read the script from the file system", ex);
            throw ex;
        } finally {
            if (in != null)
                in.close();
        }

        dmlScriptStr = sb.toString();
    } else {
        //parse given script string 
        if (script == null)
            throw new LanguageException("DML script was not specified!");

        InputStream is = new ByteArrayInputStream(script.getBytes());
        Scanner scan = new Scanner(is);
        dmlScriptStr = scan.useDelimiter("\\A").next();
        scan.close();
    }

    return dmlScriptStr;
}

From source file:com.ibm.bi.dml.api.jmlc.Connection.java

License:Open Source License

/**
 * //from  ww  w .  j a  va 2  s . com
 * @param fname
 * @return
 * @throws IOException
 */
public String readScript(String fname) throws IOException {
    StringBuilder sb = new StringBuilder();
    BufferedReader in = null;
    try {
        //read from hdfs or gpfs file system
        if (fname.startsWith("hdfs:") || fname.startsWith("gpfs:")) {
            FileSystem fs = FileSystem.get(ConfigurationManager.getCachedJobConf());
            Path scriptPath = new Path(fname);
            in = new BufferedReader(new InputStreamReader(fs.open(scriptPath)));
        }
        // from local file system
        else {
            in = new BufferedReader(new FileReader(fname));
        }

        //core script reading
        String tmp = null;
        while ((tmp = in.readLine()) != null) {
            sb.append(tmp);
            sb.append("\n");
        }
    } catch (IOException ex) {
        throw ex;
    } finally {
        if (in != null)
            in.close();
    }

    return sb.toString();
}

From source file:com.ibm.bi.dml.conf.DMLConfig.java

License:Open Source License

/**
 * Method to parse configuration//  w  w  w.  j  a v  a 2 s .  c om
 * @throws ParserConfigurationException
 * @throws SAXException
 * @throws IOException
 */
private void parseConfig() throws ParserConfigurationException, SAXException, IOException {
    DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
    factory.setIgnoringComments(true); //ignore XML comments
    DocumentBuilder builder = factory.newDocumentBuilder();
    Document domTree = null;
    if (config_file_name.startsWith("hdfs:") || config_file_name.startsWith("gpfs:")) // config file from DFS
    {
        if (!LocalFileUtils.validateExternalFilename(config_file_name, true))
            throw new IOException("Invalid (non-trustworthy) hdfs config filename.");
        FileSystem DFS = FileSystem.get(ConfigurationManager.getCachedJobConf());
        Path configFilePath = new Path(config_file_name);
        domTree = builder.parse(DFS.open(configFilePath));
    } else // config from local file system
    {
        if (!LocalFileUtils.validateExternalFilename(config_file_name, false))
            throw new IOException("Invalid (non-trustworthy) local config filename.");
        domTree = builder.parse(config_file_name);
    }

    xml_root = domTree.getDocumentElement();
}