Example usage for org.apache.hadoop.fs FileSystem open

List of usage examples for org.apache.hadoop.fs FileSystem open

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileSystem
  6. open

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem open.

Prototype

public FSDataInputStream open(PathHandle fd) throws IOException

Source Link

Document

Open an FSDataInputStream matching the PathHandle instance.

Usage

From source file:adts.HbaseClient.java

License:Open Source License

public static void main(String[] args) throws IOException {
    String[] keys = new String[5];
    int keywords_counter = 0;
    Configuration conf = new Configuration();
    FileSystem fs = FileSystem.get(conf);
    Path inFile = new Path(args[0]);
    if (!fs.exists(inFile))
        System.out.println("Input file not found");
    if (!fs.isFile(inFile))
        System.out.println("Input should be a file");
    else {/*from   ww  w .j  av  a2s .  co m*/
        FSDataInputStream fsDataInputStream = fs.open(inFile);
        BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(fsDataInputStream));
        String line;
        while (((line = bufferedReader.readLine()) != null) && (keywords_counter < 5)) {
            String[] array = line.split("\t");
            String keyword = array[0];
            System.out.println("Record :   " + keyword);
            keys[keywords_counter] = keyword;
            keywords_counter++;
        }
        bufferedReader.close();
        fs.close();

        Configuration config = HBaseConfiguration.create();
        HTable table = new HTable(config, "index");

        Random randomGenerator = new Random();
        for (int i = 0; i < 10; i++) {
            int randomInt = randomGenerator.nextInt(5);
            System.out.println("Random chosen keyword : " + keys[randomInt]);

            FilterList list = new FilterList(FilterList.Operator.MUST_PASS_ALL);
            SingleColumnValueFilter filter_by_name = new SingleColumnValueFilter(Bytes.toBytes("keyword"),
                    Bytes.toBytes(""), CompareOp.EQUAL, Bytes.toBytes(keys[randomInt]));
            //filter_by_name.setFilterIfMissing(true);
            list.addFilter(filter_by_name);

            Scan scan = new Scan();
            scan.setFilter(list);
            //scan.addFamily(Bytes.toBytes("keyword"));
            ResultScanner scanner = table.getScanner(scan);
            try {

                for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
                    // print out the row we found and the columns we were looking for
                    byte[] cells = rr.getValue(Bytes.toBytes("article"), Bytes.toBytes(""));
                    System.out.println("Keyword " + keys[randomInt] + "belonging to article with md5 : "
                            + Bytes.toString(cells));
                }
            } catch (Exception e) {
                e.printStackTrace();
            } finally {
                scanner.close();
            }

        }
        table.close();

    }

}

From source file:apex.benchmark.RedisHelper.java

License:Apache License

public void fillDB(String fileName) throws IOException {
    Path filePath = new Path(fileName);
    Configuration configuration = new Configuration();
    FileSystem fs;
    fs = FileSystem.newInstance(filePath.toUri(), configuration);
    FSDataInputStream inputStream = fs.open(filePath);
    BufferedReader bufferedReader;

    try {//from  www.j  av  a  2s  .  c  o m

        bufferedReader = new BufferedReader(new InputStreamReader(inputStream));

        String line;
        while ((line = bufferedReader.readLine()) != null) {

            String[] mapping = line.split("\\s+");

            if (mapping.length != 2) {
                continue;
            }

            jedis.sadd("campaigns", mapping[0]);
            jedis.set(mapping[1], mapping[0]);
        }
    } catch (Exception e) {
        throw e;
    }
}

From source file:application.RecommenderEvaluator.java

License:Open Source License

/**
 *    il metodo evaluate prende in ingresso una collezione di item e media e standard error ad esso associato, 
 *    una collezione di user che ha come valore una collezione delle medie degli item per i quali ha espresso un voto,
 *    una stringa che contiene il path ad un file di testo (#user,#item,#vote) per effettuare i test. Il metodo fornisce in output la matrice di confusione cosi' formata 
 *    nella cella [0][0] ci sara' il numero di volte in cui il voto e' stato positivo e la previsione e' stata positiva (True Positive)
 *    nella cella [0][1] ci sara' il numero di volte in cui il voto e' stato positivo e la previsione e' stata negativa (False Negative)
 *    nella cella [1][0] ci sara' il numero di volte in cui il voto e' stato negativo e la previsione e' stata positiva (False Positive)
 *    nella cella [1][1] ci sara' il numero di volte in cui il voto e' stato negativo e la previsione e' stata negativa (True Negative)
 *    @param Stato_Item e' una collezione di item con media e standard error a ciascuno di esso associato
 *    @param Stato_User e' una collezione di user con valore una collezione delle medie degli item per i quali l'user ha espresso un voto
 *    @param s e' una stringa che contiene il path ad un file di testo per effettuare i test
 *    @return una matrice di confusione con il numero di TruePositive, FalsePositive, FalseNegative, TrueNegative ottenuti.
 *    @throws IOException// w w w .ja v a  2s . co m
 */
public void evaluate() {
    String test_file = GLOBALS.getTEST_FILE_NAME();
    String split = GLOBALS.getSPLIT_TOKEN();
    try {
        FileSystem fs = FileSystem.get(conf);
        BufferedReader br = new BufferedReader(new InputStreamReader(fs.open(new Path(test_file))));
        String line;
        while ((line = br.readLine()) != null) {
            StringTokenizer st = new StringTokenizer(line, split);
            int user = Integer.parseInt(st.nextToken());
            int item = Integer.parseInt(st.nextToken());
            int vote = Integer.parseInt(st.nextToken());
            UserProfile UP = USER_STATE.get(user);
            ItemProfile IP = ITEM_STATE.get(item);
            if (UP == null || IP == null) {
                skipped_evaluation++;
            } else {

                Boolean prediction = estimatePreference(UP, IP);
                if (prediction == null) {
                    skipped_evaluation++;
                    continue;
                }

                //               System.out.println("prediction:"+prediction+", vote:"+vote);
                if (prediction) {
                    if (vote == 1) {
                        CM[0][0] += 1;
                    } //TP
                    else if (vote == -1) {
                        CM[1][0] += 1;
                    } //FP
                } else {
                    if (vote == -1) {
                        CM[1][1] += 1;
                    } //TN
                    else if (vote == 1) {
                        CM[0][1] += 1;
                    } //FN
                }
            }
        }
        br.close();

        //STAT
        //         System.out.println("TP: "+CM[0][0]);
        //         System.out.println("FN: "+CM[0][1]);
        //         System.out.println("FP: "+CM[1][0]);
        //         System.out.println("TN: "+CM[1][1]);
        //         System.out.println("SKYPPED: "+skipped_evaluation);
    } catch (Exception e) {
        System.out.println("Exception " + e);
        e.printStackTrace();
        System.exit(-1);
    }
}

From source file:ark.util.HadoopUtil.java

License:Apache License

public static BufferedReader getFileReader(String path) {
    try {//from   w w  w  .ja v  a 2  s. co m
        Path filePath = new Path(path);
        FileSystem fileSystem = FileSystem.get(new Configuration());
        BufferedReader reader = new BufferedReader(new InputStreamReader(fileSystem.open(filePath)));
        return reader;
    } catch (Exception e) {
        return null;
    }
}

From source file:Assignment3_P2_MergeStockAverageCount.StockPriceMergeDriver.java

/**
 * @param args the command line arguments
 *///  w w  w  .j  a v  a2  s . c o  m
public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
    Configuration conf = new Configuration();

    // local file system handle
    FileSystem local = FileSystem.getLocal(conf);

    // hdfs file system handle
    FileSystem hdfs = FileSystem.get(conf);

    // local input directory
    Path inputDir = new Path(args[0]);

    // hdfs i/p  directory
    Path inputDir1 = new Path(args[1]);

    // local input files in local dir
    FileStatus[] inputFiles = local.listStatus(inputDir);

    // o/p stream
    FSDataOutputStream out = hdfs.create(inputDir1);

    // open each file and extract contents of file
    for (int i = 0; i < inputFiles.length; i++) {
        System.out.println("File name ----------------------------------------------------------------> "
                + inputFiles[i].getPath().getName());
        FSDataInputStream in = local.open(inputFiles[i].getPath());
        byte buffer[] = new byte[256];
        int bytesRead = 0;

        // extract all contents of file
        while ((bytesRead = in.read(buffer)) > 0) {
            out.write(buffer, 0, bytesRead);
        }

        // close input stream
        in.close();
    }

    Job job = Job.getInstance(conf, "Average Stock Price");
    job.setJarByClass(StockPriceMergeDriver.class);
    job.setMapperClass(StockPriceMerge_Mapper.class);
    job.setCombinerClass(StockPriceMerge_Reducer.class);
    job.setReducerClass(StockPriceMerge_Reducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(FloatWritable.class);
    FileInputFormat.addInputPath(job, new Path(args[1])); // above programs output will be input for mapper
    FileOutputFormat.setOutputPath(job, new Path(args[2]));
    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

From source file:at.illecker.hadoop.rootbeer.examples.matrixmultiplication.cpu.MatrixMultiplicationCpu.java

License:Apache License

static void printOutput(Configuration conf) throws IOException {
    FileSystem fs = OUTPUT_DIR.getFileSystem(conf);
    FileStatus[] files = fs.listStatus(OUTPUT_DIR);
    for (int i = 0; i < files.length; i++) {
        if (files[i].getLen() > 0) {
            System.out.println("File " + files[i].getPath());
            if (files[i].getPath().getName().endsWith(".log")) {
                FSDataInputStream in = fs.open(files[i].getPath());
                IOUtils.copyBytes(in, System.out, conf, false);
                in.close();/*w ww. j  ava 2 s .  co m*/
            }
        }
    }
    // fs.delete(FileOutputFormat.getOutputPath(job), true);
}

From source file:at.illecker.hadoop.rootbeer.examples.matrixmultiplication.MatrixMultiplicationBenchmark.java

License:Apache License

private void printOutput(Configuration conf) throws IOException {
    FileSystem fs = FileSystem.get(conf);
    FileStatus[] files = fs.listStatus(CONF_OUTPUT_DIR);
    for (int i = 0; i < files.length; i++) {
        if (files[i].getLen() > 0) {
            System.out.println("File " + files[i].getPath());
            FSDataInputStream in = fs.open(files[i].getPath());
            IOUtils.copyBytes(in, System.out, conf, false);
            in.close();/*  w w w.  ja v a  2  s .  c  o  m*/
        }
    }
    // fs.delete(FileOutputFormat.getOutputPath(job), true);
}

From source file:at.illecker.hama.hybrid.examples.hellohybrid.HelloHybridBSP.java

License:Apache License

static void printOutput(BSPJob job, Path path) throws IOException {
    FileSystem fs = path.getFileSystem(job.getConfiguration());
    FileStatus[] files = fs.listStatus(path);
    for (int i = 0; i < files.length; i++) {
        if (files[i].getLen() > 0) {
            System.out.println("File " + files[i].getPath());
            SequenceFile.Reader reader = null;
            try {
                reader = new SequenceFile.Reader(fs, files[i].getPath(), job.getConfiguration());

                IntWritable key = new IntWritable();
                NullWritable value = NullWritable.get();
                while (reader.next(key, value)) {
                    System.out.println("key: '" + key.get() + "' value: '" + value + "'\n");
                }//from  w  ww.ja  va2 s  .  c  o  m
            } catch (IOException e) {
                FSDataInputStream in = fs.open(files[i].getPath());
                IOUtils.copyBytes(in, System.out, job.getConfiguration(), false);
                in.close();
            } finally {
                if (reader != null) {
                    reader.close();
                }
            }
        }
    }
    // fs.delete(FileOutputFormat.getOutputPath(job), true);
}

From source file:at.illecker.hama.hybrid.examples.kmeans.KMeansHybridBSP.java

License:Apache License

static void printFile(Configuration conf, FileSystem fs, Path file, Writable key, Writable value)
        throws IOException {
    System.out.println("File " + file.toString());
    SequenceFile.Reader reader = null;
    try {/* w  w  w .  j  a v  a2 s .c  o  m*/
        reader = new SequenceFile.Reader(fs, file, conf);

        while (reader.next(key, value)) {
            System.out.println("key: '" + key.toString() + "' value: '" + value.toString() + "'\n");
        }
    } catch (IOException e) {
        FSDataInputStream in = fs.open(file);
        IOUtils.copyBytes(in, System.out, conf, false);
        in.close();
    } catch (NullPointerException e) {
        LOG.error(e);
    } finally {
        if (reader != null) {
            reader.close();
        }
    }
}

From source file:at.illecker.hama.hybrid.examples.matrixmultiplication.MatrixMultiplicationHybridBenchmark.java

License:Apache License

static void printOutput(Configuration conf) throws IOException {
    FileSystem fs = FileSystem.get(conf);
    FileStatus[] files = fs.listStatus(new Path(OUTPUT_DIR));
    for (int i = 0; i < files.length; i++) {
        if (files[i].getLen() > 0) {
            System.out.println("File " + files[i].getPath());
            FSDataInputStream in = fs.open(files[i].getPath());
            IOUtils.copyBytes(in, System.out, conf, false);
            in.close();//from   w  w w .  ja  v a  2 s  .c om
        }
    }
    // fs.delete(FileOutputFormat.getOutputPath(job), true);
}