Example usage for org.apache.cassandra.io.sstable Component DATA

List of usage examples for org.apache.cassandra.io.sstable Component DATA

  1. HOME
  2. Java
  3. org.apache.cassandra
  4. org.apache.cassandra.io.sstable.*
  5. Component
  6. DATA

Introduction

In this page you can find the example usage for org.apache.cassandra.io.sstable Component DATA.

Prototype

Component DATA

To view the source code for org.apache.cassandra.io.sstable Component DATA.

Click Source Link

Usage

From source file:com.chaordicsystems.sstableconverter.SSTableConverter.java

License:Apache License

public static Collection<SSTableReader> readSSTables(final IPartitioner srcPartitioner, File srcDir,
        final CFMetaData metadata, final OutputHandler outputHandler) {
    final List<SSTableReader> sstables = new ArrayList<SSTableReader>();
    srcDir.list(new FilenameFilter() {
        public boolean accept(File dir, String name) {
            if (new File(dir, name).isDirectory())
                return false;
            Pair<Descriptor, Component> p = SSTable.tryComponentFromFilename(dir, name);
            Descriptor desc = p == null ? null : p.left;
            if (p == null || !p.right.equals(Component.DATA) || desc.temporary)
                return false;

            if (!new File(desc.filenameFor(Component.PRIMARY_INDEX)).exists()) {
                outputHandler.output(String.format("Skipping file %s because index is missing", name));
                return false;
            }//from   w ww .ja  va2s  .  c o m

            Set<Component> components = new HashSet<Component>();
            components.add(Component.DATA);
            components.add(Component.PRIMARY_INDEX);
            if (new File(desc.filenameFor(Component.COMPRESSION_INFO)).exists())
                components.add(Component.COMPRESSION_INFO);
            if (new File(desc.filenameFor(Component.STATS)).exists())
                components.add(Component.STATS);

            try {
                sstables.add(SSTableReader.openForBatch(desc, components, srcPartitioner, metadata));
            } catch (IOException e) {
                outputHandler
                        .output(String.format("Skipping file %s, error opening it: %s", name, e.getMessage()));
            }
            return false;
        }
    });
    return sstables;
}

From source file:com.datastax.brisk.BriskServer.java

License:Apache License

/**
 * Retrieves a local subBlock//  ww w . j  a v a 2  s  .  co m
 * 
 * @param blockId row key
 * @param sblockId SubBlock column name
 * @param offset inside the sblock
 * @return a local sublock
 * @throws TException
 */
private LocalBlock getLocalSubBlock(String subBlockCFName, ByteBuffer blockId, ByteBuffer sblockId, int offset)
        throws TException {
    DecoratedKey<Token<?>> decoratedKey = new DecoratedKey<Token<?>>(
            StorageService.getPartitioner().getToken(blockId), blockId);

    Table table = Table.open(cfsKeyspace);
    ColumnFamilyStore sblockStore = table.getColumnFamilyStore(subBlockCFName);

    Collection<SSTableReader> sstables = sblockStore.getSSTables();

    for (SSTableReader sstable : sstables) {

        long position = sstable.getPosition(decoratedKey, Operator.EQ);

        if (position == -1)
            continue;

        String filename = sstable.descriptor.filenameFor(Component.DATA);
        RandomAccessFile raf = null;
        int mappedLength = -1;
        MappedByteBuffer mappedData = null;
        MappedFileDataInput file = null;
        try {
            raf = new RandomAccessFile(filename, "r");
            assert position < raf.length();

            mappedLength = (raf.length() - position) < Integer.MAX_VALUE ? (int) (raf.length() - position)
                    : Integer.MAX_VALUE;

            mappedData = raf.getChannel().map(FileChannel.MapMode.READ_ONLY, position, mappedLength);

            file = new MappedFileDataInput(mappedData, filename, 0);

            if (file == null)
                continue;

            //Verify key was found in data file
            DecoratedKey keyInDisk = SSTableReader.decodeKey(sstable.partitioner, sstable.descriptor,
                    ByteBufferUtil.readWithShortLength(file));
            assert keyInDisk.equals(decoratedKey) : String.format("%s != %s in %s", keyInDisk, decoratedKey,
                    file.getPath());

            long rowSize = SSTableReader.readRowSize(file, sstable.descriptor);

            assert rowSize > 0;
            assert rowSize < mappedLength;

            Filter bf = IndexHelper.defreezeBloomFilter(file, sstable.descriptor.usesOldBloomFilter);

            //verify this column in in this version of the row.
            if (!bf.isPresent(sblockId))
                continue;

            List<IndexHelper.IndexInfo> indexList = IndexHelper.deserializeIndex(file);

            // we can stop early if bloom filter says none of the
            // columns actually exist -- but,
            // we can't stop before initializing the cf above, in
            // case there's a relevant tombstone
            ColumnFamilySerializer serializer = ColumnFamily.serializer();
            try {
                ColumnFamily cf = serializer
                        .deserializeFromSSTableNoColumns(ColumnFamily.create(sstable.metadata), file);

                if (cf.isMarkedForDelete())
                    continue;

            } catch (Exception e) {
                e.printStackTrace();

                throw new IOException(serializer + " failed to deserialize " + sstable.getColumnFamilyName()
                        + " with " + sstable.metadata + " from " + file, e);
            }

            Integer sblockLength = null;

            if (indexList == null)
                sblockLength = seekToSubColumn(sstable.metadata, file, sblockId);
            else
                sblockLength = seekToSubColumn(sstable.metadata, file, sblockId, indexList);

            if (sblockLength == null || sblockLength < 0)
                continue;

            int bytesReadFromStart = mappedLength - (int) file.bytesRemaining();

            if (logger.isDebugEnabled())
                logger.debug("BlockLength = " + sblockLength + " Availible " + file.bytesRemaining());

            assert offset <= sblockLength : String.format("%d > %d", offset, sblockLength);

            long dataOffset = position + bytesReadFromStart;

            if (file.bytesRemaining() == 0 || sblockLength == 0)
                continue;

            return new LocalBlock(file.getPath(), dataOffset + offset, sblockLength - offset);

        } catch (IOException e) {
            throw new TException(e);
        } finally {
            FileUtils.closeQuietly(raf);
        }
    }

    return null;
}

From source file:com.knewton.mapreduce.SSTableRecordReader.java

License:Apache License

/**
 * Moves all the minimum required tables for the table reader to work to local disk.
 *
 * @param split The table to work on./*from   ww w .  ja v a  2  s. c  om*/
 */
@VisibleForTesting
void copyTablesToLocal(FileSystem remoteFS, FileSystem localFS, Path dataTablePath, TaskAttemptContext context)
        throws IOException {
    Configuration conf = context.getConfiguration();
    String hdfsDataTablePathStr = dataTablePath.toUri().getPath();
    String localDataTablePathStr = dataTablePath.toUri().getHost() + File.separator
            + dataTablePath.toUri().getPath();
    // Make path relative due to EMR permissions
    if (localDataTablePathStr.startsWith("/")) {
        String mapTaskId = conf.get("mapreduce.task.attempt.id");
        String mapTempDir = conf.get("mapreduce.cluster.temp.dir");
        String taskWorkDir = mapTempDir + File.separator + mapTaskId;
        LOG.info("Appending {} to {}", taskWorkDir, localDataTablePathStr);
        localDataTablePathStr = taskWorkDir + localDataTablePathStr;
    }
    Path localDataTablePath = new Path(localDataTablePathStr);
    LOG.info("Copying hdfs file from {} to local disk at {}.", dataTablePath.toUri(),
            localDataTablePath.toUri());
    copyToLocalFile(remoteFS, localFS, dataTablePath, localDataTablePath);
    boolean isCompressed = conf.getBoolean(PropertyConstants.COMPRESSION_ENABLED.txt, false);
    if (isCompressed) {
        decompress(localDataTablePath, context);
    }
    components.add(Component.DATA);
    desc = Descriptor.fromFilename(localDataTablePathStr);
    Descriptor hdfsDesc = Descriptor.fromFilename(hdfsDataTablePathStr);
    String indexPathStr = hdfsDesc.filenameFor(Component.PRIMARY_INDEX);
    components.add(Component.PRIMARY_INDEX);
    Path localIdxPath = new Path(desc.filenameFor(Component.PRIMARY_INDEX));
    LOG.info("Copying hdfs file from {} to local disk at {}.", indexPathStr, localIdxPath);
    copyToLocalFile(remoteFS, localFS, new Path(indexPathStr), localIdxPath);
    if (isCompressed) {
        decompress(localIdxPath, context);
    }
    String compressionTablePathStr = hdfsDesc.filenameFor(Component.COMPRESSION_INFO.name());
    Path compressionTablePath = new Path(compressionTablePathStr);
    if (remoteFS.exists(compressionTablePath)) {
        Path localCompressionPath = new Path(desc.filenameFor(Component.COMPRESSION_INFO.name()));
        LOG.info("Copying hdfs file from {} to local disk at {}.", compressionTablePath.toUri(),
                localCompressionPath);
        copyToLocalFile(remoteFS, localFS, compressionTablePath, localCompressionPath);
        if (isCompressed) {
            decompress(localCompressionPath, context);
        }
        components.add(Component.COMPRESSION_INFO);
    }
}

From source file:com.scylladb.tools.SSTableToCQL.java

License:Open Source License

private void addFile(final List<SSTableReader> sstables, File dir, String name) {
    Pair<Descriptor, Component> p = SSTable.tryComponentFromFilename(dir, name);
    Descriptor desc = p == null ? null : p.left;
    if (p == null || !p.right.equals(Component.DATA) || desc.type.isTemporary) {
        return;/*from  w  w  w . j a  v  a2 s . c  o  m*/
    }

    if (!new File(desc.filenameFor(Component.PRIMARY_INDEX)).exists()) {
        logger.info("Skipping file {} because index is missing", name);
        return;
    }

    CFMetaData metadata = getCFMetaData(keyspace, desc.cfname);
    if (metadata == null) {
        logger.info("Skipping file {}: column family {}.{} doesn't exist", name, keyspace, desc.cfname);
        return;
    }

    Set<Component> components = new HashSet<>();
    components.add(Component.DATA);
    components.add(Component.PRIMARY_INDEX);
    if (new File(desc.filenameFor(Component.SUMMARY)).exists()) {
        components.add(Component.SUMMARY);
    }
    if (new File(desc.filenameFor(Component.COMPRESSION_INFO)).exists()) {
        components.add(Component.COMPRESSION_INFO);
    }
    if (new File(desc.filenameFor(Component.STATS)).exists()) {
        components.add(Component.STATS);
    }

    try {
        // To conserve memory, open SSTableReaders without bloom
        // filters and discard
        // the index summary after calculating the file sections to
        // stream and the estimated
        // number of keys for each endpoint. See CASSANDRA-5555 for
        // details.
        SSTableReader sstable = openForBatch(desc, components, metadata, getPartitioner());
        sstables.add(sstable);
    } catch (IOException e) {
        logger.warn("Skipping file {}, error opening it: {}", name, e.getMessage());
    }
}

From source file:com.wenyu.cassandra.tools.SSTableLevelToZero.java

License:Apache License

/**
 * @param args a list of sstables whose metadata we are changing
 *///  w  ww  .  java  2 s .c  o  m
public static void main(String[] args) throws IOException {
    PrintStream out = System.out;
    if (args.length < 3) {
        out.println("This command should be run with Cassandra stopped!");
        out.println("Usage: sstableleveltozero <keyspace> <columnfamily> <sstablefiles_fullpath>");
        System.exit(1);
    }

    // TODO several daemon threads will run from here.
    // So we have to explicitly call System.exit.
    try {
        // load keyspace descriptions.
        DatabaseDescriptor.loadSchemas();

        String keyspaceName = args[0];
        String columnfamily = args[1];
        String sstables = args[2];

        Set<String> sstableSet = new HashSet<>();
        for (String sstable : sstables.split(",")) {
            sstable = sstable.trim();
            if (sstable.length() == 0) {
                continue;
            }
            sstableSet.add(sstable);
        }

        // validate columnfamily
        if (Schema.instance.getCFMetaData(keyspaceName, columnfamily) == null) {
            System.err.println("ColumnFamily not found: " + keyspaceName + "/" + columnfamily);
            System.exit(1);
        }

        Keyspace keyspace = Keyspace.openWithoutSSTables(keyspaceName);
        ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(columnfamily);
        boolean foundSSTable = false;
        for (Map.Entry<Descriptor, Set<Component>> sstable : cfs.directories.sstableLister().list()
                .entrySet()) {
            if (sstable.getValue().contains(Component.STATS)) {
                foundSSTable = true;
                Descriptor descriptor = sstable.getKey();

                StatsMetadata metadata = (StatsMetadata) descriptor.getMetadataSerializer()
                        .deserialize(descriptor, MetadataType.STATS);
                String path = descriptor.filenameFor(Component.DATA);
                if (metadata.sstableLevel > 0 && sstableSet.contains(path)) {
                    out.println("Changing level from " + metadata.sstableLevel + " to 0 on "
                            + descriptor.filenameFor(Component.DATA));
                    descriptor.getMetadataSerializer().mutateLevel(descriptor, 0);
                } else if (metadata.sstableLevel <= 0 && sstableSet.contains(path)) {
                    out.println("Skipped " + descriptor.filenameFor(Component.DATA)
                            + " since it is already on level 0");
                }
            }
        }

        if (!foundSSTable) {
            out.println("Found no sstables, did you give the correct keyspace/columnfamily?");
        }
    } catch (Throwable t) {
        JVMStabilityInspector.inspectThrowable(t);
        t.printStackTrace();
        System.exit(1);
    }
    System.exit(0);
}