List of usage examples for org.apache.cassandra.io.sstable Descriptor filenameFor
public String filenameFor(Component component)
From source file:com.chaordicsystems.sstableconverter.SSTableConverter.java
License:Apache License
public static Collection<SSTableReader> readSSTables(final IPartitioner srcPartitioner, File srcDir, final CFMetaData metadata, final OutputHandler outputHandler) { final List<SSTableReader> sstables = new ArrayList<SSTableReader>(); srcDir.list(new FilenameFilter() { public boolean accept(File dir, String name) { if (new File(dir, name).isDirectory()) return false; Pair<Descriptor, Component> p = SSTable.tryComponentFromFilename(dir, name); Descriptor desc = p == null ? null : p.left; if (p == null || !p.right.equals(Component.DATA) || desc.temporary) return false; if (!new File(desc.filenameFor(Component.PRIMARY_INDEX)).exists()) { outputHandler.output(String.format("Skipping file %s because index is missing", name)); return false; }//from w w w .j a v a 2s. c om Set<Component> components = new HashSet<Component>(); components.add(Component.DATA); components.add(Component.PRIMARY_INDEX); if (new File(desc.filenameFor(Component.COMPRESSION_INFO)).exists()) components.add(Component.COMPRESSION_INFO); if (new File(desc.filenameFor(Component.STATS)).exists()) components.add(Component.STATS); try { sstables.add(SSTableReader.openForBatch(desc, components, srcPartitioner, metadata)); } catch (IOException e) { outputHandler .output(String.format("Skipping file %s, error opening it: %s", name, e.getMessage())); } return false; } }); return sstables; }
From source file:com.knewton.mapreduce.SSTableRecordReader.java
License:Apache License
/** * Moves all the minimum required tables for the table reader to work to local disk. * * @param split The table to work on.//www .j a v a 2s. co m */ @VisibleForTesting void copyTablesToLocal(FileSystem remoteFS, FileSystem localFS, Path dataTablePath, TaskAttemptContext context) throws IOException { Configuration conf = context.getConfiguration(); String hdfsDataTablePathStr = dataTablePath.toUri().getPath(); String localDataTablePathStr = dataTablePath.toUri().getHost() + File.separator + dataTablePath.toUri().getPath(); // Make path relative due to EMR permissions if (localDataTablePathStr.startsWith("/")) { String mapTaskId = conf.get("mapreduce.task.attempt.id"); String mapTempDir = conf.get("mapreduce.cluster.temp.dir"); String taskWorkDir = mapTempDir + File.separator + mapTaskId; LOG.info("Appending {} to {}", taskWorkDir, localDataTablePathStr); localDataTablePathStr = taskWorkDir + localDataTablePathStr; } Path localDataTablePath = new Path(localDataTablePathStr); LOG.info("Copying hdfs file from {} to local disk at {}.", dataTablePath.toUri(), localDataTablePath.toUri()); copyToLocalFile(remoteFS, localFS, dataTablePath, localDataTablePath); boolean isCompressed = conf.getBoolean(PropertyConstants.COMPRESSION_ENABLED.txt, false); if (isCompressed) { decompress(localDataTablePath, context); } components.add(Component.DATA); desc = Descriptor.fromFilename(localDataTablePathStr); Descriptor hdfsDesc = Descriptor.fromFilename(hdfsDataTablePathStr); String indexPathStr = hdfsDesc.filenameFor(Component.PRIMARY_INDEX); components.add(Component.PRIMARY_INDEX); Path localIdxPath = new Path(desc.filenameFor(Component.PRIMARY_INDEX)); LOG.info("Copying hdfs file from {} to local disk at {}.", indexPathStr, localIdxPath); copyToLocalFile(remoteFS, localFS, new Path(indexPathStr), localIdxPath); if (isCompressed) { decompress(localIdxPath, context); } String compressionTablePathStr = hdfsDesc.filenameFor(Component.COMPRESSION_INFO.name()); Path compressionTablePath = new Path(compressionTablePathStr); if (remoteFS.exists(compressionTablePath)) { Path localCompressionPath = new Path(desc.filenameFor(Component.COMPRESSION_INFO.name())); LOG.info("Copying hdfs file from {} to local disk at {}.", compressionTablePath.toUri(), localCompressionPath); copyToLocalFile(remoteFS, localFS, compressionTablePath, localCompressionPath); if (isCompressed) { decompress(localCompressionPath, context); } components.add(Component.COMPRESSION_INFO); } }
From source file:com.scylladb.tools.SSTableToCQL.java
License:Open Source License
private void addFile(final List<SSTableReader> sstables, File dir, String name) { Pair<Descriptor, Component> p = SSTable.tryComponentFromFilename(dir, name); Descriptor desc = p == null ? null : p.left; if (p == null || !p.right.equals(Component.DATA) || desc.type.isTemporary) { return;//from w ww.j a v a2 s. co m } if (!new File(desc.filenameFor(Component.PRIMARY_INDEX)).exists()) { logger.info("Skipping file {} because index is missing", name); return; } CFMetaData metadata = getCFMetaData(keyspace, desc.cfname); if (metadata == null) { logger.info("Skipping file {}: column family {}.{} doesn't exist", name, keyspace, desc.cfname); return; } Set<Component> components = new HashSet<>(); components.add(Component.DATA); components.add(Component.PRIMARY_INDEX); if (new File(desc.filenameFor(Component.SUMMARY)).exists()) { components.add(Component.SUMMARY); } if (new File(desc.filenameFor(Component.COMPRESSION_INFO)).exists()) { components.add(Component.COMPRESSION_INFO); } if (new File(desc.filenameFor(Component.STATS)).exists()) { components.add(Component.STATS); } try { // To conserve memory, open SSTableReaders without bloom // filters and discard // the index summary after calculating the file sections to // stream and the estimated // number of keys for each endpoint. See CASSANDRA-5555 for // details. SSTableReader sstable = openForBatch(desc, components, metadata, getPartitioner()); sstables.add(sstable); } catch (IOException e) { logger.warn("Skipping file {}, error opening it: {}", name, e.getMessage()); } }
From source file:com.wenyu.cassandra.tools.SSTableLevelToZero.java
License:Apache License
/** * @param args a list of sstables whose metadata we are changing *//* w w w . j av a2s. c o m*/ public static void main(String[] args) throws IOException { PrintStream out = System.out; if (args.length < 3) { out.println("This command should be run with Cassandra stopped!"); out.println("Usage: sstableleveltozero <keyspace> <columnfamily> <sstablefiles_fullpath>"); System.exit(1); } // TODO several daemon threads will run from here. // So we have to explicitly call System.exit. try { // load keyspace descriptions. DatabaseDescriptor.loadSchemas(); String keyspaceName = args[0]; String columnfamily = args[1]; String sstables = args[2]; Set<String> sstableSet = new HashSet<>(); for (String sstable : sstables.split(",")) { sstable = sstable.trim(); if (sstable.length() == 0) { continue; } sstableSet.add(sstable); } // validate columnfamily if (Schema.instance.getCFMetaData(keyspaceName, columnfamily) == null) { System.err.println("ColumnFamily not found: " + keyspaceName + "/" + columnfamily); System.exit(1); } Keyspace keyspace = Keyspace.openWithoutSSTables(keyspaceName); ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(columnfamily); boolean foundSSTable = false; for (Map.Entry<Descriptor, Set<Component>> sstable : cfs.directories.sstableLister().list() .entrySet()) { if (sstable.getValue().contains(Component.STATS)) { foundSSTable = true; Descriptor descriptor = sstable.getKey(); StatsMetadata metadata = (StatsMetadata) descriptor.getMetadataSerializer() .deserialize(descriptor, MetadataType.STATS); String path = descriptor.filenameFor(Component.DATA); if (metadata.sstableLevel > 0 && sstableSet.contains(path)) { out.println("Changing level from " + metadata.sstableLevel + " to 0 on " + descriptor.filenameFor(Component.DATA)); descriptor.getMetadataSerializer().mutateLevel(descriptor, 0); } else if (metadata.sstableLevel <= 0 && sstableSet.contains(path)) { out.println("Skipped " + descriptor.filenameFor(Component.DATA) + " since it is already on level 0"); } } } if (!foundSSTable) { out.println("Found no sstables, did you give the correct keyspace/columnfamily?"); } } catch (Throwable t) { JVMStabilityInspector.inspectThrowable(t); t.printStackTrace(); System.exit(1); } System.exit(0); }