Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.carbondata.core.writer; import java.io.IOException; import java.io.Serializable; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.datamap.Segment; import org.apache.carbondata.core.datastore.filesystem.CarbonFile; import org.apache.carbondata.core.datastore.impl.FileFactory; import org.apache.carbondata.core.fileoperations.FileWriteOperation; import org.apache.carbondata.core.indexstore.blockletindex.SegmentIndexFileStore; import org.apache.carbondata.core.metadata.SegmentFileStore; import org.apache.carbondata.core.metadata.schema.table.CarbonTable; import org.apache.carbondata.core.statusmanager.SegmentStatus; import org.apache.carbondata.core.util.path.CarbonTablePath; import org.apache.carbondata.format.MergedBlockIndex; import org.apache.carbondata.format.MergedBlockIndexHeader; import org.apache.hadoop.fs.Path; import org.apache.log4j.Logger; public class CarbonIndexFileMergeWriter { /** * table handle */ private CarbonTable table; /** * thrift writer object */ private ThriftWriter thriftWriter; private Logger LOGGER = LogServiceFactory.getLogService(this.getClass().getCanonicalName()); public CarbonIndexFileMergeWriter(CarbonTable table) { this.table = table; } /** * Merge all the carbonindex files of segment to a merged file * @param tablePath * @param indexFileNamesTobeAdded while merging it comsiders only these files. * If null then consider all * @param readFileFooterFromCarbonDataFile flag to read file footer information from carbondata * file. This will used in case of upgrade from version * which do not store the blocklet info to current version * @throws IOException */ private String mergeCarbonIndexFilesOfSegment(String segmentId, String tablePath, List<String> indexFileNamesTobeAdded, boolean readFileFooterFromCarbonDataFile, String uuid) { try { Segment segment = Segment.getSegment(segmentId, tablePath); String segmentPath = CarbonTablePath.getSegmentPath(tablePath, segmentId); CarbonFile[] indexFiles; SegmentFileStore sfs = null; if (segment != null && segment.getSegmentFileName() != null) { sfs = new SegmentFileStore(tablePath, segment.getSegmentFileName()); List<CarbonFile> indexCarbonFiles = sfs.getIndexCarbonFiles(); indexFiles = indexCarbonFiles.toArray(new CarbonFile[indexCarbonFiles.size()]); } else { indexFiles = SegmentIndexFileStore.getCarbonIndexFiles(segmentPath, FileFactory.getConfiguration()); } if (isCarbonIndexFilePresent(indexFiles) || indexFileNamesTobeAdded != null) { if (sfs == null) { return writeMergeIndexFileBasedOnSegmentFolder(indexFileNamesTobeAdded, readFileFooterFromCarbonDataFile, segmentPath, indexFiles, segmentId); } else { return writeMergeIndexFileBasedOnSegmentFile(segmentId, indexFileNamesTobeAdded, sfs, indexFiles, uuid); } } } catch (Exception e) { LOGGER.error("Failed to merge index files in path: " + tablePath, e); } return null; } private String writeMergeIndexFileBasedOnSegmentFolder(List<String> indexFileNamesTobeAdded, boolean readFileFooterFromCarbonDataFile, String segmentPath, CarbonFile[] indexFiles, String segmentId) throws IOException { SegmentIndexFileStore fileStore = new SegmentIndexFileStore(); if (readFileFooterFromCarbonDataFile) { // this case will be used in case of upgrade where old store will not have the blocklet // info in the index file and therefore blocklet info need to be read from the file footer // in the carbondata file fileStore.readAllIndexAndFillBolckletInfo(segmentPath); } else { fileStore.readAllIIndexOfSegment(segmentPath); } Map<String, byte[]> indexMap = fileStore.getCarbonIndexMap(); writeMergeIndexFile(indexFileNamesTobeAdded, segmentPath, indexMap, segmentId); for (CarbonFile indexFile : indexFiles) { indexFile.delete(); } return null; } private String writeMergeIndexFileBasedOnSegmentFile(String segmentId, List<String> indexFileNamesTobeAdded, SegmentFileStore segmentFileStore, CarbonFile[] indexFiles, String uuid) throws IOException { SegmentIndexFileStore fileStore = new SegmentIndexFileStore(); fileStore.readAllIIndexOfSegment(segmentFileStore.getSegmentFile(), segmentFileStore.getTablePath(), SegmentStatus.SUCCESS, true); Map<String, byte[]> indexMap = fileStore.getCarbonIndexMapWithFullPath(); Map<String, Map<String, byte[]>> indexLocationMap = new HashMap<>(); for (Map.Entry<String, byte[]> entry : indexMap.entrySet()) { Path path = new Path(entry.getKey()); Map<String, byte[]> map = indexLocationMap.get(path.getParent().toString()); if (map == null) { map = new HashMap<>(); indexLocationMap.put(path.getParent().toString(), map); } map.put(path.getName(), entry.getValue()); } for (Map.Entry<String, Map<String, byte[]>> entry : indexLocationMap.entrySet()) { String mergeIndexFile = writeMergeIndexFile(indexFileNamesTobeAdded, entry.getKey(), entry.getValue(), segmentId); for (Map.Entry<String, SegmentFileStore.FolderDetails> segentry : segmentFileStore.getLocationMap() .entrySet()) { String location = segentry.getKey(); if (segentry.getValue().isRelative()) { location = segmentFileStore.getTablePath() + CarbonCommonConstants.FILE_SEPARATOR + location; } if (new Path(entry.getKey()).equals(new Path(location))) { segentry.getValue().setMergeFileName(mergeIndexFile); segentry.getValue().setFiles(new HashSet<String>()); break; } } } String newSegmentFileName = SegmentFileStore.genSegmentFileName(segmentId, uuid) + CarbonTablePath.SEGMENT_EXT; String path = CarbonTablePath.getSegmentFilesLocation(table.getTablePath()) + CarbonCommonConstants.FILE_SEPARATOR + newSegmentFileName; SegmentFileStore.writeSegmentFile(segmentFileStore.getSegmentFile(), path); SegmentFileStore.updateSegmentFile(table, segmentId, newSegmentFileName, table.getCarbonTableIdentifier().getTableId(), segmentFileStore); for (CarbonFile file : indexFiles) { file.delete(); } return uuid; } private String writeMergeIndexFile(List<String> indexFileNamesTobeAdded, String segmentPath, Map<String, byte[]> indexMap, String segment_id) throws IOException { MergedBlockIndexHeader indexHeader = new MergedBlockIndexHeader(); MergedBlockIndex mergedBlockIndex = new MergedBlockIndex(); List<String> fileNames = new ArrayList<>(indexMap.size()); List<ByteBuffer> data = new ArrayList<>(indexMap.size()); for (Map.Entry<String, byte[]> entry : indexMap.entrySet()) { if (indexFileNamesTobeAdded == null || indexFileNamesTobeAdded.contains(entry.getKey())) { fileNames.add(entry.getKey()); data.add(ByteBuffer.wrap(entry.getValue())); } } if (fileNames.size() > 0) { String mergeIndexName = segment_id + '_' + System.currentTimeMillis() + CarbonTablePath.MERGE_INDEX_FILE_EXT; openThriftWriter(segmentPath + "/" + mergeIndexName); indexHeader.setFile_names(fileNames); mergedBlockIndex.setFileData(data); writeMergedBlockIndexHeader(indexHeader); writeMergedBlockIndex(mergedBlockIndex); close(); return mergeIndexName; } return null; } /** * Merge all the carbonindex files of segment to a merged file * * @param segmentId * @throws IOException */ public String mergeCarbonIndexFilesOfSegment(String segmentId, String uuid, String tablePath) throws IOException { return mergeCarbonIndexFilesOfSegment(segmentId, tablePath, null, false, uuid); } /** * Merge all the carbonindex files of segment to a merged file * * @param segmentId * @param readFileFooterFromCarbonDataFile * @throws IOException */ public String mergeCarbonIndexFilesOfSegment(String segmentId, String tablePath, boolean readFileFooterFromCarbonDataFile, String uuid) throws IOException { return mergeCarbonIndexFilesOfSegment(segmentId, tablePath, null, readFileFooterFromCarbonDataFile, uuid); } private boolean isCarbonIndexFilePresent(CarbonFile[] indexFiles) { for (CarbonFile file : indexFiles) { if (file.getName().endsWith(CarbonTablePath.INDEX_FILE_EXT)) { return true; } } return false; } /** * It writes thrift object to file * * @throws IOException */ private void writeMergedBlockIndexHeader(MergedBlockIndexHeader indexObject) throws IOException { thriftWriter.write(indexObject); } /** * It writes thrift object to file * * @throws IOException */ private void writeMergedBlockIndex(MergedBlockIndex indexObject) throws IOException { thriftWriter.write(indexObject); } /** * Below method will be used to open the thrift writer * * @param filePath file path where data need to be written * @throws IOException throws io exception in case of any failure */ private void openThriftWriter(String filePath) throws IOException { // create thrift writer instance thriftWriter = new ThriftWriter(filePath, false); // open the file stream thriftWriter.open(FileWriteOperation.OVERWRITE); } /** * Below method will be used to close the thrift object */ private void close() throws IOException { thriftWriter.close(); } public static class SegmentIndexFIleMergeStatus implements Serializable { private SegmentFileStore.SegmentFile segmentFile; private List<String> filesTobeDeleted; public SegmentIndexFIleMergeStatus(SegmentFileStore.SegmentFile segmentFile, List<String> filesTobeDeleted) { this.segmentFile = segmentFile; this.filesTobeDeleted = filesTobeDeleted; } public SegmentFileStore.SegmentFile getSegmentFile() { return segmentFile; } public List<String> getFilesTobeDeleted() { return filesTobeDeleted; } } }