Example usage for org.apache.hadoop.mapred LineRecordReader close

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred LineRecordReader close.

Prototype

public synchronized void close() throws IOException 

Source Link

Usage

From source file:edu.umn.cs.spatialHadoop.mapred.SpatialInputFormat.java

License:Open Source License

protected void listStatus(final FileSystem fs, Path dir, final List<FileStatus> result, BlockFilter filter)
        throws IOException {
    GlobalIndex<Partition> gindex = SpatialSite.getGlobalIndex(fs, dir);
    if (gindex == null) {
        FileStatus[] listStatus;//from  www . j  av  a  2  s  . com
        if (OperationsParams.isWildcard(dir)) {
            // Wild card
            listStatus = fs.globStatus(dir);
        } else {
            listStatus = fs.listStatus(dir, SpatialSite.NonHiddenFileFilter);
        }
        // Add all files under this directory
        for (FileStatus status : listStatus) {
            if (status.isDir()) {
                listStatus(fs, status.getPath(), result, filter);
            } else if (status.getPath().getName().toLowerCase().endsWith(".list")) {
                LineRecordReader in = new LineRecordReader(fs.open(status.getPath()), 0, status.getLen(),
                        Integer.MAX_VALUE);
                LongWritable key = in.createKey();
                Text value = in.createValue();
                while (in.next(key, value)) {
                    result.add(fs.getFileStatus(new Path(status.getPath().getParent(), value.toString())));
                }
                in.close();
            } else {
                result.add(status);
            }
        }
    } else {
        final Path indexDir = OperationsParams.isWildcard(dir) ? dir.getParent() : dir;
        // Use the global index to limit files
        filter.selectCells(gindex, new ResultCollector<Partition>() {
            @Override
            public void collect(Partition partition) {
                try {
                    Path cell_path = new Path(indexDir, partition.filename);
                    if (!fs.exists(cell_path))
                        LOG.warn("Matched file not found: " + cell_path);
                    result.add(fs.getFileStatus(cell_path));
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        });
    }
}

---

From source file:edu.umn.cs.spatialHadoop.operations.CatUnion.java

License:Open Source License

/**
 * Read all categories from the category file
 * @param categoryFile// w  ww  . j  ava  2s  . c o  m
 * @param categoryShapes
 * @param idToCategory
 * @throws IOException
 */
private static void readCategories(Path categoryFile, Map<Integer, Integer> idToCategory) throws IOException {
    Map<Integer, String> idToCatName = new HashMap<Integer, String>();
    FileSystem fsCategory = FileSystem.getLocal(new Configuration());
    long categoryFileSize = fsCategory.getFileStatus(categoryFile).getLen();
    if (categoryFileSize > 1024 * 1024)
        LOG.warn("Category file size is big: " + categoryFileSize);
    InputStream inCategory = fsCategory.open(categoryFile);
    LineRecordReader lineReader = new LineRecordReader(inCategory, 0, categoryFileSize, new Configuration());
    LongWritable lineOffset = lineReader.createKey();
    Text line = lineReader.createValue();

    Set<String> catNames = new TreeSet<String>();

    while (lineReader.next(lineOffset, line)) {
        int shape_id = TextSerializerHelper.consumeInt(line, ',');
        String cat_name = line.toString();
        catNames.add(cat_name);
        idToCatName.put(shape_id, cat_name);
    }

    lineReader.close();

    // Change category names to numbers
    Map<String, Integer> cat_name_to_id = new HashMap<String, Integer>();
    int cat_id = 0;
    for (String cat_name : catNames) {
        cat_name_to_id.put(cat_name, cat_id++);
    }

    for (Map.Entry<Integer, String> entry : idToCatName.entrySet()) {
        idToCategory.put(entry.getKey(), cat_name_to_id.get(entry.getValue()));
    }
}

---

From source file:kogiri.mapreduce.libra.kmersimilarity_m.KmerSimilarityMap.java

License:Open Source License

private void sumScores(Path outputPath, Configuration conf) throws IOException {
    Path[] resultFiles = KmerSimilarityHelper.getAllKmerSimilarityResultFilePath(conf, outputPath.toString());
    FileSystem fs = outputPath.getFileSystem(conf);

    KmerSimilarityOutputRecord scoreRec = null;
    for (Path resultFile : resultFiles) {
        LOG.info("Reading the scores from " + resultFile.toString());
        FSDataInputStream is = fs.open(resultFile);
        FileStatus status = fs.getFileStatus(resultFile);

        LineRecordReader reader = new LineRecordReader(is, 0, status.getLen(), conf);

        LongWritable off = new LongWritable();
        Text val = new Text();

        while (reader.next(off, val)) {
            if (scoreRec == null) {
                scoreRec = KmerSimilarityOutputRecord.createInstance(val.toString());
            } else {
                KmerSimilarityOutputRecord rec2 = KmerSimilarityOutputRecord.createInstance(val.toString());
                scoreRec.addScore(rec2.getScore());
            }/*ww w.  j  av  a  2s  . c o m*/
        }

        reader.close();
    }

    double[] accumulatedScore = scoreRec.getScore();

    String resultFilename = KmerSimilarityHelper.makeKmerSimilarityFinalResultFileName();
    Path resultFilePath = new Path(outputPath, resultFilename);

    LOG.info("Creating a final score file : " + resultFilePath.toString());

    FSDataOutputStream os = fs.create(resultFilePath);
    int n = (int) Math.sqrt(accumulatedScore.length);
    for (int i = 0; i < accumulatedScore.length; i++) {
        int x = i / n;
        int y = i % n;

        String k = x + "-" + y;
        String v = Double.toString(accumulatedScore[i]);
        String out = k + "\t" + v + "\n";
        os.write(out.getBytes());
    }

    os.close();
}

---

From source file:libra.core.kmersimilarity_m.KmerSimilarityMap.java

License:Apache License

private void sumScores(Path outputPath, Configuration conf) throws IOException {
    Path[] resultFiles = KmerSimilarityHelper.getAllKmerSimilarityResultFilePath(conf, outputPath.toString());
    FileSystem fs = outputPath.getFileSystem(conf);

    KmerSimilarityOutputRecord scoreRec = null;
    for (Path resultFile : resultFiles) {
        LOG.info("Reading the scores from " + resultFile.toString());
        FSDataInputStream is = fs.open(resultFile);
        FileStatus status = fs.getFileStatus(resultFile);

        LineRecordReader reader = new LineRecordReader(is, 0, status.getLen(), conf);

        LongWritable off = new LongWritable();
        Text val = new Text();

        while (reader.next(off, val)) {
            if (scoreRec == null) {
                scoreRec = KmerSimilarityOutputRecord.createInstance(val.toString());
            } else {
                KmerSimilarityOutputRecord rec2 = KmerSimilarityOutputRecord.createInstance(val.toString());
                scoreRec.addScore(rec2.getScore());
            }//from www . j a v a 2s. co  m
        }

        reader.close();
    }

    double[] accumulatedScore = scoreRec.getScore();

    String resultFilename = KmerSimilarityHelper.makeKmerSimilarityFinalResultFileName();
    Path resultFilePath = new Path(outputPath, resultFilename);

    LOG.info("Creating a final score file : " + resultFilePath.toString());

    FSDataOutputStream os = fs.create(resultFilePath);
    int n = (int) Math.sqrt(accumulatedScore.length);
    for (int i = 0; i < accumulatedScore.length; i++) {
        int x = i / n;
        int y = i % n;

        String k = x + "-" + y;
        String v = Double.toString(accumulatedScore[i]);
        if (x == y) {
            v = Double.toString(1.0);
        }
        String out = k + "\t" + v + "\n";
        os.write(out.getBytes());
    }

    os.close();
}

---