Example usage for org.apache.cassandra.io.sstable Component COMPRESSION_INFO

Introduction

In this page you can find the example usage for org.apache.cassandra.io.sstable Component COMPRESSION_INFO.

Prototype

Component COMPRESSION_INFO

To view the source code for org.apache.cassandra.io.sstable Component COMPRESSION_INFO.

Click Source Link

Usage

From source file:com.chaordicsystems.sstableconverter.SSTableConverter.java

License:Apache License

public static Collection<SSTableReader> readSSTables(final IPartitioner srcPartitioner, File srcDir,
        final CFMetaData metadata, final OutputHandler outputHandler) {
    final List<SSTableReader> sstables = new ArrayList<SSTableReader>();
    srcDir.list(new FilenameFilter() {
        public boolean accept(File dir, String name) {
            if (new File(dir, name).isDirectory())
                return false;
            Pair<Descriptor, Component> p = SSTable.tryComponentFromFilename(dir, name);
            Descriptor desc = p == null ? null : p.left;
            if (p == null || !p.right.equals(Component.DATA) || desc.temporary)
                return false;

            if (!new File(desc.filenameFor(Component.PRIMARY_INDEX)).exists()) {
                outputHandler.output(String.format("Skipping file %s because index is missing", name));
                return false;
            }//from  ww w . j a v  a  2  s .c  o  m

            Set<Component> components = new HashSet<Component>();
            components.add(Component.DATA);
            components.add(Component.PRIMARY_INDEX);
            if (new File(desc.filenameFor(Component.COMPRESSION_INFO)).exists())
                components.add(Component.COMPRESSION_INFO);
            if (new File(desc.filenameFor(Component.STATS)).exists())
                components.add(Component.STATS);

            try {
                sstables.add(SSTableReader.openForBatch(desc, components, srcPartitioner, metadata));
            } catch (IOException e) {
                outputHandler
                        .output(String.format("Skipping file %s, error opening it: %s", name, e.getMessage()));
            }
            return false;
        }
    });
    return sstables;
}

---

From source file:com.knewton.mapreduce.SSTableRecordReader.java

License:Apache License

/**
 * Moves all the minimum required tables for the table reader to work to local disk.
 *
 * @param split The table to work on./*from   w w  w . j a va2 s.  c o m*/
 */
@VisibleForTesting
void copyTablesToLocal(FileSystem remoteFS, FileSystem localFS, Path dataTablePath, TaskAttemptContext context)
        throws IOException {
    Configuration conf = context.getConfiguration();
    String hdfsDataTablePathStr = dataTablePath.toUri().getPath();
    String localDataTablePathStr = dataTablePath.toUri().getHost() + File.separator
            + dataTablePath.toUri().getPath();
    // Make path relative due to EMR permissions
    if (localDataTablePathStr.startsWith("/")) {
        String mapTaskId = conf.get("mapreduce.task.attempt.id");
        String mapTempDir = conf.get("mapreduce.cluster.temp.dir");
        String taskWorkDir = mapTempDir + File.separator + mapTaskId;
        LOG.info("Appending {} to {}", taskWorkDir, localDataTablePathStr);
        localDataTablePathStr = taskWorkDir + localDataTablePathStr;
    }
    Path localDataTablePath = new Path(localDataTablePathStr);
    LOG.info("Copying hdfs file from {} to local disk at {}.", dataTablePath.toUri(),
            localDataTablePath.toUri());
    copyToLocalFile(remoteFS, localFS, dataTablePath, localDataTablePath);
    boolean isCompressed = conf.getBoolean(PropertyConstants.COMPRESSION_ENABLED.txt, false);
    if (isCompressed) {
        decompress(localDataTablePath, context);
    }
    components.add(Component.DATA);
    desc = Descriptor.fromFilename(localDataTablePathStr);
    Descriptor hdfsDesc = Descriptor.fromFilename(hdfsDataTablePathStr);
    String indexPathStr = hdfsDesc.filenameFor(Component.PRIMARY_INDEX);
    components.add(Component.PRIMARY_INDEX);
    Path localIdxPath = new Path(desc.filenameFor(Component.PRIMARY_INDEX));
    LOG.info("Copying hdfs file from {} to local disk at {}.", indexPathStr, localIdxPath);
    copyToLocalFile(remoteFS, localFS, new Path(indexPathStr), localIdxPath);
    if (isCompressed) {
        decompress(localIdxPath, context);
    }
    String compressionTablePathStr = hdfsDesc.filenameFor(Component.COMPRESSION_INFO.name());
    Path compressionTablePath = new Path(compressionTablePathStr);
    if (remoteFS.exists(compressionTablePath)) {
        Path localCompressionPath = new Path(desc.filenameFor(Component.COMPRESSION_INFO.name()));
        LOG.info("Copying hdfs file from {} to local disk at {}.", compressionTablePath.toUri(),
                localCompressionPath);
        copyToLocalFile(remoteFS, localFS, compressionTablePath, localCompressionPath);
        if (isCompressed) {
            decompress(localCompressionPath, context);
        }
        components.add(Component.COMPRESSION_INFO);
    }
}

---

From source file:com.knewton.mapreduce.SSTableRecordReaderTest.java

License:Apache License

/**
 * Tests to see if tables can be correctly copied locally including the compression info table
 *//*from   w w w.  j  av a  2s .  c o  m*/
@Test
public void testCopyTablesToLocalWithCompressionInfo() throws Exception {
    TaskAttemptContext context = getTaskAttemptContext(true, true, true);
    ssTableColumnRecordReader.initialize(inputSplit, context);

    doCallRealMethod().when(ssTableColumnRecordReader).copyTablesToLocal(any(FileSystem.class),
            any(FileSystem.class), any(Path.class), any(TaskAttemptContext.class));

    FileSystem remoteFS = mock(FileSystem.class);
    FileSystem localFS = mock(FileSystem.class);

    byte[] data = new byte[] { 0xA };
    FSDataInputStream fsIn = new FSDataInputStream(new MemoryDataInputStream(data));
    FSDataOutputStream fsOut = mock(FSDataOutputStream.class);

    when(remoteFS.open(any(Path.class))).thenReturn(fsIn);
    when(localFS.create(any(Path.class), anyBoolean())).thenReturn(fsOut);

    Path dataTablePath = inputSplit.getPath();
    FileStatus fileStatus = mock(FileStatus.class);
    when(fileStatus.getLen()).thenReturn(10L);
    when(fileStatus.isDirectory()).thenReturn(false);
    when(remoteFS.getFileStatus(any(Path.class))).thenReturn(fileStatus);

    String str = ssTableColumnRecordReader.getDescriptor().filenameFor(Component.COMPRESSION_INFO);
    when(remoteFS.exists(new Path(str))).thenReturn(true);

    ssTableColumnRecordReader.copyTablesToLocal(remoteFS, localFS, dataTablePath, context);
    verify(remoteFS).getFileStatus(dataTablePath);
    ssTableColumnRecordReader.close();
    verify(fsOut).write(any(byte[].class), eq(0), eq(data.length));
    assertEquals(3, ssTableColumnRecordReader.getComponentSize());
}

---

From source file:com.scylladb.tools.SSTableToCQL.java

License:Open Source License

private void addFile(final List<SSTableReader> sstables, File dir, String name) {
    Pair<Descriptor, Component> p = SSTable.tryComponentFromFilename(dir, name);
    Descriptor desc = p == null ? null : p.left;
    if (p == null || !p.right.equals(Component.DATA) || desc.type.isTemporary) {
        return;/*w  w  w .j  a  va  2  s .c  o  m*/
    }

    if (!new File(desc.filenameFor(Component.PRIMARY_INDEX)).exists()) {
        logger.info("Skipping file {} because index is missing", name);
        return;
    }

    CFMetaData metadata = getCFMetaData(keyspace, desc.cfname);
    if (metadata == null) {
        logger.info("Skipping file {}: column family {}.{} doesn't exist", name, keyspace, desc.cfname);
        return;
    }

    Set<Component> components = new HashSet<>();
    components.add(Component.DATA);
    components.add(Component.PRIMARY_INDEX);
    if (new File(desc.filenameFor(Component.SUMMARY)).exists()) {
        components.add(Component.SUMMARY);
    }
    if (new File(desc.filenameFor(Component.COMPRESSION_INFO)).exists()) {
        components.add(Component.COMPRESSION_INFO);
    }
    if (new File(desc.filenameFor(Component.STATS)).exists()) {
        components.add(Component.STATS);
    }

    try {
        // To conserve memory, open SSTableReaders without bloom
        // filters and discard
        // the index summary after calculating the file sections to
        // stream and the estimated
        // number of keys for each endpoint. See CASSANDRA-5555 for
        // details.
        SSTableReader sstable = openForBatch(desc, components, metadata, getPartitioner());
        sstables.add(sstable);
    } catch (IOException e) {
        logger.warn("Skipping file {}, error opening it: {}", name, e.getMessage());
    }
}

---