List of usage examples for org.apache.solr.store.hdfs HdfsDirectory HdfsDirectory
public HdfsDirectory(Path hdfsDirPath, Configuration configuration) throws IOException
From source file:org.apache.carbondata.datamap.lucene.LuceneCoarseGrainDataMap.java
License:Apache License
/** * It is called to load the data map to memory or to initialize it. *///from ww w . ja va 2s. co m @Override public void init(DataMapModel dataMapModel) throws MemoryException, IOException { // get this path from file path Path indexPath = FileFactory.getPath(dataMapModel.getFilePath()); LOGGER.info("Lucene index read path " + indexPath.toString()); // get file system , use hdfs file system , realized in solr project FileSystem fs = FileFactory.getFileSystem(indexPath); // check this path valid if (!fs.exists(indexPath)) { String errorMessage = String.format("index directory %s not exists.", indexPath); LOGGER.error(errorMessage); throw new IOException(errorMessage); } if (!fs.isDirectory(indexPath)) { String errorMessage = String.format("error index path %s, must be directory", indexPath); LOGGER.error(errorMessage); throw new IOException(errorMessage); } // open this index path , use HDFS default configuration Directory indexDir = new HdfsDirectory(indexPath, FileFactory.getConfiguration()); IndexReader indexReader = DirectoryReader.open(indexDir); if (indexReader == null) { throw new RuntimeException("failed to create index reader object"); } // create a index searcher object indexSearcher = new IndexSearcher(indexReader); }
From source file:org.apache.carbondata.datamap.lucene.LuceneDataMapBuilder.java
License:Apache License
private IndexWriter createIndexWriter(String dataMapPath) throws IOException { Path indexPath = FileFactory.getPath(dataMapPath); FileSystem fs = FileFactory.getFileSystem(indexPath); // if index path exists, should delete it because we are // rebuilding the whole datamap for all segments if (fs.exists(indexPath)) { fs.delete(indexPath, true);//from ww w. j a v a 2s. co m } if (!fs.mkdirs(indexPath)) { LOGGER.error("Failed to create directory " + indexPath); } if (null == analyzer) { analyzer = new StandardAnalyzer(); } // create a index writer Directory indexDir = new HdfsDirectory(indexPath, FileFactory.getConfiguration()); IndexWriterConfig indexWriterConfig = new IndexWriterConfig(analyzer); if (CarbonProperties.getInstance() .getProperty(CarbonCommonConstants.CARBON_LUCENE_COMPRESSION_MODE, CarbonCommonConstants.CARBON_LUCENE_COMPRESSION_MODE_DEFAULT) .equalsIgnoreCase(CarbonCommonConstants.CARBON_LUCENE_COMPRESSION_MODE_DEFAULT)) { indexWriterConfig.setCodec(new Lucene62Codec(Lucene50StoredFieldsFormat.Mode.BEST_SPEED)); } else { indexWriterConfig.setCodec(new Lucene62Codec(Lucene50StoredFieldsFormat.Mode.BEST_COMPRESSION)); } return new IndexWriter(indexDir, new IndexWriterConfig(analyzer)); }
From source file:org.apache.carbondata.datamap.lucene.LuceneDataMapWriter.java
License:Apache License
/** * Start of new blocklet notification./*from w w w.jav a 2 s .c om*/ */ public void onBlockletStart(int blockletId) throws IOException { if (null == analyzer) { if (CarbonProperties.getInstance() .getProperty(CarbonCommonConstants.CARBON_LUCENE_INDEX_STOP_WORDS, CarbonCommonConstants.CARBON_LUCENE_INDEX_STOP_WORDS_DEFAULT) .equalsIgnoreCase("true")) { analyzer = new StandardAnalyzer(CharArraySet.EMPTY_SET); } else { analyzer = new StandardAnalyzer(); } } // save index data into ram, write into disk after one page finished ramDir = new RAMDirectory(); ramIndexWriter = new IndexWriter(ramDir, new IndexWriterConfig(analyzer)); if (indexWriter != null) { return; } // get index path, put index data into segment's path String dataMapPath; if (storeBlockletWise) { dataMapPath = this.dataMapPath + File.separator + blockletId; } else { dataMapPath = this.dataMapPath; } Path indexPath = FileFactory.getPath(dataMapPath); FileSystem fs = FileFactory.getFileSystem(indexPath); // if index path not exists, create it if (!fs.exists(indexPath)) { if (!fs.mkdirs(indexPath)) { throw new IOException("Failed to create directory " + dataMapPath); } } // the indexWriter closes the FileSystem on closing the writer, so for a new configuration // and disable the cache for the index writer, it will be closed on closing the writer Configuration conf = FileFactory.getConfiguration(); conf.set("fs.hdfs.impl.disable.cache", "true"); // create a index writer Directory indexDir = new HdfsDirectory(indexPath, conf); IndexWriterConfig indexWriterConfig = new IndexWriterConfig(analyzer); if (CarbonProperties.getInstance() .getProperty(CarbonCommonConstants.CARBON_LUCENE_COMPRESSION_MODE, CarbonCommonConstants.CARBON_LUCENE_COMPRESSION_MODE_DEFAULT) .equalsIgnoreCase(CarbonCommonConstants.CARBON_LUCENE_COMPRESSION_MODE_DEFAULT)) { indexWriterConfig.setCodec(speedCodec); } else { indexWriterConfig.setCodec(compressionCodec); } indexWriter = new IndexWriter(indexDir, indexWriterConfig); }
From source file:org.apache.carbondata.datamap.lucene.LuceneFineGrainDataMap.java
License:Apache License
private IndexSearcher createIndexSearcher(Path indexPath) throws IOException { // open this index path , use HDFS default configuration Directory indexDir = new HdfsDirectory(indexPath, FileFactory.getConfiguration()); this.indexReader = DirectoryReader.open(indexDir); if (indexReader == null) { throw new RuntimeException("failed to create index reader object"); }/*from w w w .j a v a 2 s .c o m*/ // create a index searcher object return new IndexSearcher(indexReader); }
From source file:org.apache.mahout.text.LuceneSegmentInputFormat.java
License:Apache License
@Override public List<LuceneSegmentInputSplit> getSplits(JobContext context) throws IOException, InterruptedException { Configuration configuration = context.getConfiguration(); LuceneStorageConfiguration lucene2SeqConfiguration = new LuceneStorageConfiguration(configuration); List<LuceneSegmentInputSplit> inputSplits = new ArrayList<>(); List<Path> indexPaths = lucene2SeqConfiguration.getIndexPaths(); for (Path indexPath : indexPaths) { HdfsDirectory directory = new HdfsDirectory(indexPath, configuration); SegmentInfos segmentInfos = new SegmentInfos(); segmentInfos.read(directory);//from w w w . java2 s .c o m for (SegmentCommitInfo segmentInfo : segmentInfos) { LuceneSegmentInputSplit inputSplit = new LuceneSegmentInputSplit(indexPath, segmentInfo.info.name, segmentInfo.sizeInBytes()); inputSplits.add(inputSplit); LOG.info("Created {} byte input split for index '{}' segment {}", segmentInfo.sizeInBytes(), indexPath.toUri(), segmentInfo.info.name); } } return inputSplits; }
From source file:org.apache.mahout.text.LuceneSegmentInputSplit.java
License:Apache License
/** * Get the {@link SegmentInfo} of this {@link InputSplit} via the given {@link Configuration} * * @param configuration the configuration used to locate the index * @return the segment info or throws exception if not found * @throws IOException if an error occurs when accessing the directory *//*from w w w .j av a 2s .c om*/ public SegmentCommitInfo getSegment(Configuration configuration) throws IOException { HdfsDirectory directory = new HdfsDirectory(indexPath, configuration); SegmentInfos segmentInfos = new SegmentInfos(); segmentInfos.read(directory); for (SegmentCommitInfo segmentInfo : segmentInfos) { if (segmentInfo.info.name.equals(segmentInfoName)) { return segmentInfo; } } throw new IllegalArgumentException( "No such segment: '" + segmentInfoName + "' in directory " + directory.toString()); }