Example usage for org.apache.hadoop.fs Path getParent

List of usage examples for org.apache.hadoop.fs Path getParent

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. Path
  6. getParent

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path getParent.

Prototype

public Path getParent()

Source Link

Document

Returns the parent of a path or null if at root.

Usage

From source file:org.apache.blur.manager.writer.IndexImporter.java

License:Apache License

private IndexAction getIndexAction(final HdfsDirectory directory, final FileSystem fileSystem) {
    return new IndexAction() {

        @Override//www. j  a  v a 2 s . c  o  m
        public void performMutate(IndexSearcherCloseable searcher, IndexWriter writer) throws IOException {
            LOG.info("About to import [{0}] into [{1}/{2}]", directory, _shard, _table);
            boolean emitDeletes = searcher.getIndexReader().numDocs() != 0;
            applyDeletes(directory, writer, _shard, emitDeletes);
            LOG.info("Add index [{0}] [{1}/{2}]", directory, _shard, _table);
            writer.addIndexes(directory);
            LOG.info("Removing delete markers [{0}] on [{1}/{2}]", directory, _shard, _table);
            writer.deleteDocuments(new Term(BlurConstants.DELETE_MARKER, BlurConstants.DELETE_MARKER_VALUE));
            LOG.info("Finishing import [{0}], commiting on [{1}/{2}]", directory, _shard, _table);
        }

        @Override
        public void doPreCommit(IndexSearcherCloseable indexSearcher, IndexWriter writer) throws IOException {

        }

        @Override
        public void doPostCommit(IndexWriter writer) throws IOException {
            Path path = directory.getPath();
            fileSystem.delete(new Path(path, INPROGRESS), false);
            LOG.info("Import complete on [{0}/{1}]", _shard, _table);
            writer.maybeMerge();
        }

        @Override
        public void doPreRollback(IndexWriter writer) throws IOException {
            LOG.info("Starting rollback on [{0}/{1}]", _shard, _table);
        }

        @Override
        public void doPostRollback(IndexWriter writer) throws IOException {
            LOG.info("Finished rollback on [{0}/{1}]", _shard, _table);
            Path path = directory.getPath();
            String name = path.getName();
            fileSystem.rename(path, new Path(path.getParent(), rename(name, BADROWIDS)));
        }
    };
}

From source file:org.apache.blur.manager.writer.IndexImporter.java

License:Apache License

public void cleanupOldDirs() throws IOException {
    Path hdfsDirPath = _shardContext.getHdfsDirPath();
    TableContext tableContext = _shardContext.getTableContext();
    Configuration configuration = tableContext.getConfiguration();
    FileSystem fileSystem = hdfsDirPath.getFileSystem(configuration);
    FileStatus[] inuseSubDirs = fileSystem.listStatus(hdfsDirPath, new PathFilter() {
        @Override//from  ww w  .j a  v a2  s.  c om
        public boolean accept(Path path) {
            return path.getName().endsWith(INUSE);
        }
    });
    Set<Path> inuseDirs = toSet(inuseSubDirs);
    Map<Path, Path> inuseFileToDir = toMap(fileSystem, inuseDirs);
    FileStatus[] listStatus = fileSystem.listStatus(hdfsDirPath, new PathFilter() {
        @Override
        public boolean accept(Path path) {
            return path.getName().endsWith(HdfsDirectory.LNK);
        }
    });

    for (FileStatus status : listStatus) {
        Path realPath = HdfsDirectory.readRealPathDataFromSymlinkPath(fileSystem, status.getPath());
        Path inuseDir = inuseFileToDir.get(realPath);
        inuseDirs.remove(inuseDir);
        // if the inuse dir has an inprogress file then remove it because there
        // are files that reference this dir so it had to be committed.
        Path path = new Path(inuseDir, INPROGRESS);
        if (fileSystem.exists(path)) {
            fileSystem.delete(path, false);
            if (_thriftCache != null) {
                _thriftCache.clearTable(_table);
            }
        }
    }

    // Check if any inuse dirs have inprogress files.
    // If they do, rename inuse to commit to retry import.
    for (Path inuse : new HashSet<Path>(inuseDirs)) {
        Path path = new Path(inuse, INPROGRESS);
        if (fileSystem.exists(path)) {
            LOG.info("Path [{0}] is not imported but has inprogress file, retrying import.", path);
            inuseDirs.remove(inuse);
            Path commit = new Path(inuse.getParent(), rename(inuse.getName(), COMMIT));
            fileSystem.rename(inuse, commit);
        }
    }

    for (Path p : inuseDirs) {
        LOG.info("Deleting path [{0}] no longer in use.", p);
        fileSystem.delete(p, true);
    }
}

From source file:org.apache.blur.manager.writer.TestingSeqSorting.java

License:Apache License

public static void main(String[] args) throws IOException {
    Configuration conf = new Configuration();
    Path p = new Path("hdfs://localhost:9000/testsort/data.seq");
    // URI uri = new File(".").getAbsoluteFile().toURI();
    // Path local = new Path(uri);

    // Path p = new Path(new Path(local, "tmp"), "data.seq");
    // {/*from  w ww. j  a  va 2 s.c  o m*/
    // Writer writer = SequenceFile.createWriter(conf,
    // Writer.valueClass(LongWritable.class),
    // Writer.keyClass(BytesWritable.class), Writer.file(p));
    //
    // LongWritable value = new LongWritable();
    // BytesWritable key = new BytesWritable();
    // Random random = new Random(1);
    // byte[] buf = new byte[50];
    // for (int i = 0; i < 10000000; i++) {
    // value.set(random.nextLong());
    // random.nextBytes(buf);
    // key.set(buf, 0, buf.length);
    // writer.append(key, value);
    // }
    // writer.close();
    // }

    // {
    // // io.seqfile.local.dir
    // URI uri = new File(".").getAbsoluteFile().toURI();
    // Path local = new Path(uri);
    //
    // conf.set("io.seqfile.local.dir", new Path(local, "tmp").toString());
    // Path tempDir = new Path(p.getParent(), "tmp");
    // Sorter sorter = new SequenceFile.Sorter(p.getFileSystem(conf),
    // BytesWritable.class, LongWritable.class, conf);
    // sorter.setMemory(10 * 1024 * 1024);
    // // Path tempDir = new Path(p.getParent(), "tmp");
    //
    // RawKeyValueIterator iterate = sorter.sortAndIterate(new Path[] { p },
    // tempDir, false);
    // Path sortedPath = new Path(p.getParent(), "sorted.seq");
    // Writer writer = SequenceFile.createWriter(conf,
    // Writer.valueClass(LongWritable.class),
    // Writer.keyClass(BytesWritable.class), Writer.file(sortedPath));
    // while (iterate.next()) {
    // DataOutputBuffer key = iterate.getKey();
    // byte[] keyData = key.getData();
    // int keyOffset = 0;
    // int keyLength = key.getLength();
    // ValueBytes val = iterate.getValue();
    // writer.appendRaw(keyData, keyOffset, keyLength, val);
    // }
    // writer.close();
    // }
    {
        conf.setInt("io.sort.factor", 1000);
        Sorter sorter = new SequenceFile.Sorter(p.getFileSystem(conf), BytesWritable.class, LongWritable.class,
                conf);
        sorter.setMemory(10 * 1024 * 1024);
        sorter.sort(new Path[] { p }, new Path(p.getParent(), "sorted.seq"), false);
    }

}

From source file:org.apache.blur.mapreduce.lib.CsvBlurMapper.java

License:Apache License

protected boolean isParent(Path possibleParent, Path child) {
    if (child == null) {
        return false;
    }//from  ww w . j  av a  2s .com
    if (possibleParent.equals(child.getParent())) {
        return true;
    }
    return isParent(possibleParent, child.getParent());
}

From source file:org.apache.camel.component.hdfs.HdfsConsumerTest.java

License:Apache License

@Test
public void testReadStringArrayFile() throws Exception {
    if (!canTest()) {
        return;/*from   w  ww.  ja va 2s  . c  o  m*/
    }

    final Path file = new Path(new File("target/test/test-camel-string").getAbsolutePath());
    Configuration conf = new Configuration();
    FileSystem fs1 = FileSystem.get(file.toUri(), conf);
    ArrayFile.Writer writer = new ArrayFile.Writer(conf, fs1, "target/test/test-camel-string1", Text.class,
            CompressionType.NONE, new Progressable() {
                @Override
                public void progress() {
                }
            });
    Text valueWritable = new Text();
    String value = "CIAO!";
    valueWritable.set(value);
    writer.append(valueWritable);
    writer.close();

    MockEndpoint resultEndpoint = context.getEndpoint("mock:result", MockEndpoint.class);
    resultEndpoint.expectedMessageCount(1);

    context.addRoutes(new RouteBuilder() {
        public void configure() {
            from("hdfs:///" + file.getParent().toUri()
                    + "?fileSystemType=LOCAL&fileType=ARRAY_FILE&initialDelay=0").to("mock:result");
        }
    });
    context.start();

    resultEndpoint.assertIsSatisfied();
}

From source file:org.apache.camel.component.hdfs2.HdfsConsumerTest.java

License:Apache License

@Test
public void testReadStringArrayFile() throws Exception {
    if (!canTest()) {
        return;//from   ww  w .j  a v a  2s  .c o  m
    }

    final Path file = new Path(new File("target/test/test-camel-string").getAbsolutePath());
    Configuration conf = new Configuration();
    FileSystem fs1 = FileSystem.get(file.toUri(), conf);
    ArrayFile.Writer writer = new ArrayFile.Writer(conf, fs1, "target/test/test-camel-string1", Text.class,
            CompressionType.NONE, new Progressable() {
                @Override
                public void progress() {
                }
            });
    Text valueWritable = new Text();
    String value = "CIAO!";
    valueWritable.set(value);
    writer.append(valueWritable);
    writer.close();

    MockEndpoint resultEndpoint = context.getEndpoint("mock:result", MockEndpoint.class);
    resultEndpoint.expectedMessageCount(1);

    context.addRoutes(new RouteBuilder() {
        public void configure() {
            from("hdfs2:///" + file.getParent().toUri()
                    + "?fileSystemType=LOCAL&fileType=ARRAY_FILE&initialDelay=0").to("mock:result");
        }
    });
    context.start();

    resultEndpoint.assertIsSatisfied();
}

From source file:org.apache.carbondata.core.indexstore.blockletindex.BlockDataMap.java

License:Apache License

@Override
public void init(DataMapModel dataMapModel) throws IOException, MemoryException {
    long startTime = System.currentTimeMillis();
    assert (dataMapModel instanceof BlockletDataMapModel);
    BlockletDataMapModel blockletDataMapInfo = (BlockletDataMapModel) dataMapModel;
    DataFileFooterConverter fileFooterConverter = new DataFileFooterConverter(dataMapModel.getConfiguration());
    List<DataFileFooter> indexInfo = fileFooterConverter.getIndexInfo(blockletDataMapInfo.getFilePath(),
            blockletDataMapInfo.getFileData(), blockletDataMapInfo.getCarbonTable().isTransactionalTable());
    Path path = new Path(blockletDataMapInfo.getFilePath());
    // store file path only in case of partition table, non transactional table and flat folder
    // structure/*from ww  w  . jav a  2  s  .  c om*/
    byte[] filePath = null;
    boolean isPartitionTable = blockletDataMapInfo.getCarbonTable().isHivePartitionTable();
    if (isPartitionTable || !blockletDataMapInfo.getCarbonTable().isTransactionalTable()
            || blockletDataMapInfo.getCarbonTable().isSupportFlatFolder()) {
        filePath = path.getParent().toString().getBytes(CarbonCommonConstants.DEFAULT_CHARSET);
        isFilePathStored = true;
    }
    byte[] fileName = path.getName().toString().getBytes(CarbonCommonConstants.DEFAULT_CHARSET);
    byte[] segmentId = blockletDataMapInfo.getSegmentId().getBytes(CarbonCommonConstants.DEFAULT_CHARSET);
    if (!indexInfo.isEmpty()) {
        DataFileFooter fileFooter = indexInfo.get(0);
        // store for 1.1 or any prior version will not have any blocklet information in file footer
        isLegacyStore = fileFooter.getBlockletList() == null;
        // init segment properties and create schema
        SegmentProperties segmentProperties = initSegmentProperties(blockletDataMapInfo, fileFooter);
        createMemorySchema(blockletDataMapInfo);
        createSummaryDMStore(blockletDataMapInfo);
        CarbonRowSchema[] taskSummarySchema = getTaskSummarySchema();
        // check for legacy store and load the metadata
        DataMapRowImpl summaryRow = loadMetadata(taskSummarySchema, segmentProperties, blockletDataMapInfo,
                indexInfo);
        finishWriting(taskSummarySchema, filePath, fileName, segmentId, summaryRow);
    }
    if (LOGGER.isDebugEnabled()) {
        LOGGER.debug("Time taken to load blocklet datamap from file : " + dataMapModel.getFilePath() + " is "
                + (System.currentTimeMillis() - startTime));
    }
}

From source file:org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMap.java

License:Apache License

@Override
public void init(DataMapModel dataMapModel) throws IOException, MemoryException {
    long startTime = System.currentTimeMillis();
    assert (dataMapModel instanceof BlockletDataMapModel);
    BlockletDataMapModel blockletDataMapInfo = (BlockletDataMapModel) dataMapModel;
    DataFileFooterConverter fileFooterConverter = new DataFileFooterConverter();
    List<DataFileFooter> indexInfo = fileFooterConverter.getIndexInfo(blockletDataMapInfo.getFilePath(),
            blockletDataMapInfo.getFileData());
    Path path = new Path(blockletDataMapInfo.getFilePath());
    byte[] filePath = path.getParent().toString().getBytes(CarbonCommonConstants.DEFAULT_CHARSET);
    byte[] fileName = path.getName().toString().getBytes(CarbonCommonConstants.DEFAULT_CHARSET);
    byte[] segmentId = blockletDataMapInfo.getSegmentId().getBytes(CarbonCommonConstants.DEFAULT_CHARSET);
    DataMapRowImpl summaryRow = null;//from  w  w w  .  ja  v  a  2 s.  c  om
    byte[] schemaBinary = null;
    // below 2 variables will be used for fetching the relative blocklet id. Relative blocklet ID
    // is id assigned to a blocklet within a part file
    String tempFilePath = null;
    int relativeBlockletId = 0;
    for (DataFileFooter fileFooter : indexInfo) {
        if (segmentProperties == null) {
            List<ColumnSchema> columnInTable = fileFooter.getColumnInTable();
            schemaBinary = convertSchemaToBinary(columnInTable);
            blockletSchemaTime = fileFooter.getSchemaUpdatedTimeStamp();
            columnCardinality = fileFooter.getSegmentInfo().getColumnCardinality();
            segmentProperties = new SegmentProperties(columnInTable, columnCardinality);
            createSchema(segmentProperties, ((BlockletDataMapModel) dataMapModel).isAddToUnsafe());
            createSummarySchema(segmentProperties, schemaBinary, filePath, fileName, segmentId,
                    ((BlockletDataMapModel) dataMapModel).isAddToUnsafe());
        }
        TableBlockInfo blockInfo = fileFooter.getBlockInfo().getTableBlockInfo();
        BlockMetaInfo blockMetaInfo = blockletDataMapInfo.getBlockMetaInfoMap().get(blockInfo.getFilePath());
        // Here it loads info about all blocklets of index
        // Only add if the file exists physically. There are scenarios which index file exists inside
        // merge index but related carbondata files are deleted. In that case we first check whether
        // the file exists physically or not
        if (blockMetaInfo != null) {
            if (fileFooter.getBlockletList() == null) {
                // This is old store scenario, here blocklet information is not available in index file so
                // load only block info
                summaryRow = loadToUnsafeBlock(fileFooter, segmentProperties, blockInfo.getFilePath(),
                        summaryRow, blockMetaInfo);
            } else {
                // blocklet ID will start from 0 again only when part file path is changed
                if (null == tempFilePath || !tempFilePath.equals(blockInfo.getFilePath())) {
                    tempFilePath = blockInfo.getFilePath();
                    relativeBlockletId = 0;
                }
                summaryRow = loadToUnsafe(fileFooter, segmentProperties, blockInfo.getFilePath(), summaryRow,
                        blockMetaInfo, relativeBlockletId);
                // this is done because relative blocklet id need to be incremented based on the
                // total number of blocklets
                relativeBlockletId += fileFooter.getBlockletList().size();
            }
        }
    }
    if (memoryDMStore != null) {
        memoryDMStore.finishWriting();
    }
    if (null != summaryDMStore) {
        addTaskSummaryRowToUnsafeMemoryStore(summaryRow, schemaBinary, filePath, fileName, segmentId);
        summaryDMStore.finishWriting();
    }
    if (LOGGER.isDebugEnabled()) {
        LOGGER.debug("Time taken to load blocklet datamap from file : " + dataMapModel.getFilePath() + " is "
                + (System.currentTimeMillis() - startTime));
    }
}

From source file:org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMapFactory.java

License:Apache License

@Override
public List<CoarseGrainDataMap> getDataMaps(DataMapDistributable distributable) throws IOException {
    BlockletDataMapDistributable mapDistributable = (BlockletDataMapDistributable) distributable;
    List<TableBlockIndexUniqueIdentifierWrapper> identifiersWrapper = new ArrayList<>();
    Path indexPath = new Path(mapDistributable.getFilePath());
    String segmentNo = mapDistributable.getSegment().getSegmentNo();
    if (indexPath.getName().endsWith(CarbonTablePath.INDEX_FILE_EXT)) {
        String parent = indexPath.getParent().toString();
        identifiersWrapper.add(new TableBlockIndexUniqueIdentifierWrapper(
                new TableBlockIndexUniqueIdentifier(parent, indexPath.getName(), null, segmentNo),
                this.getCarbonTable()));
    } else if (indexPath.getName().endsWith(CarbonTablePath.MERGE_INDEX_FILE_EXT)) {
        SegmentIndexFileStore fileStore = new SegmentIndexFileStore();
        CarbonFile carbonFile = FileFactory.getCarbonFile(indexPath.toString());
        String parentPath = carbonFile.getParentFile().getAbsolutePath();
        List<String> indexFiles = fileStore.getIndexFilesFromMergeFile(carbonFile.getAbsolutePath());
        for (String indexFile : indexFiles) {
            identifiersWrapper.add(new TableBlockIndexUniqueIdentifierWrapper(
                    new TableBlockIndexUniqueIdentifier(parentPath, indexFile, carbonFile.getName(), segmentNo),
                    this.getCarbonTable()));
        }/*from w  w  w. j  a v  a  2  s.co  m*/
    }
    List<CoarseGrainDataMap> dataMaps = new ArrayList<>();
    try {
        List<BlockletDataMapIndexWrapper> wrappers = cache.getAll(identifiersWrapper);
        for (BlockletDataMapIndexWrapper wrapper : wrappers) {
            dataMaps.addAll(wrapper.getDataMaps());
        }
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    return dataMaps;
}

From source file:org.apache.carbondata.core.metadata.SegmentFileStore.java

License:Apache License

/**
 * Gets all carbon index files from this segment
 * @return/*from www. j a  v a  2 s. c  o m*/
 */
public List<CarbonFile> getIndexCarbonFiles() {
    Map<String, String> indexFiles = getIndexFiles();
    Set<String> files = new HashSet<>();
    for (Map.Entry<String, String> entry : indexFiles.entrySet()) {
        Path path = new Path(entry.getKey());
        files.add(entry.getKey());
        if (entry.getValue() != null) {
            files.add(new Path(path.getParent(), entry.getValue()).toString());
        }
    }
    List<CarbonFile> carbonFiles = new ArrayList<>();
    for (String indexFile : files) {
        CarbonFile carbonFile = FileFactory.getCarbonFile(indexFile);
        if (carbonFile.exists()) {
            carbonFiles.add(carbonFile);
        }
    }
    return carbonFiles;
}