org.apache.cassandra.io.sstable.Descriptor Java Examples

public static Iterator<OnDiskAtom> onDiskIterator(final DataInput in,
                                                  final ColumnSerializer.Flag flag,
                                                  final int expireBefore,
                                                  final Descriptor.Version version,
                                                  final CellNameType type)
{
    return new AbstractIterator<OnDiskAtom>()
    {
        protected OnDiskAtom computeNext()
        {
            OnDiskAtom atom;
            try
            {
                atom = type.onDiskAtomSerializer().deserializeFromSSTable(in, flag, expireBefore, version);
            }
            catch (IOException e)
            {
                throw new IOError(e);
            }
            if (atom == null)
                return endOfData();

            return atom;
        }
    };
}
 

public RowIndexEntry deserialize(DataInput in, Descriptor.Version version) throws IOException
{
    long position = in.readLong();

    int size = in.readInt();
    if (size > 0)
    {
        DeletionTime deletionTime = DeletionTime.serializer.deserialize(in);

        int entries = in.readInt();
        ISerializer<IndexHelper.IndexInfo> idxSerializer = type.indexSerializer();
        List<IndexHelper.IndexInfo> columnsIndex = new ArrayList<IndexHelper.IndexInfo>(entries);
        for (int i = 0; i < entries; i++)
            columnsIndex.add(idxSerializer.deserialize(in));

        return new IndexedEntry(position, deletionTime, columnsIndex);
    }
    else
    {
        return new RowIndexEntry(position);
    }
}
 

public Map<MetadataType, MetadataComponent> deserialize(Descriptor descriptor, EnumSet<MetadataType> types) throws IOException
{
    Map<MetadataType, MetadataComponent> components;
    logger.debug("Load metadata for {}", descriptor);
    File statsFile = new File(descriptor.filenameFor(Component.STATS));
    if (!statsFile.exists())
    {
        logger.debug("No sstable stats for {}", descriptor);
        components = Maps.newHashMap();
        components.put(MetadataType.STATS, MetadataCollector.defaultStatsMetadata());
    }
    else
    {
        try (RandomAccessReader r = RandomAccessReader.open(statsFile))
        {
            components = deserialize(descriptor, r, types);
        }
    }
    return components;
}
 

public Map<MetadataType, MetadataComponent> deserialize(Descriptor descriptor, FileDataInput in, EnumSet<MetadataType> types) throws IOException
{
    Map<MetadataType, MetadataComponent> components = Maps.newHashMap();
    // read number of components
    int numComponents = in.readInt();
    // read toc
    Map<MetadataType, Integer> toc = new HashMap<>(numComponents);
    for (int i = 0; i < numComponents; i++)
    {
        toc.put(MetadataType.values()[in.readInt()], in.readInt());
    }
    for (MetadataType type : types)
    {
        MetadataComponent component = null;
        if (toc.containsKey(type))
        {
            in.seek(toc.get(type));
            component = type.serializer.deserialize(descriptor.version, in);
        }
        components.put(type, component);
    }
    return components;
}
 

@VisibleForTesting
protected synchronized void compact()
{
    ArrayList<Descriptor> descriptors = new ArrayList<>();
    for (SSTable sstable : hintStore.getDataTracker().getUncompactingSSTables())
        descriptors.add(sstable.descriptor);

    if (descriptors.isEmpty())
        return;

    try
    {
        CompactionManager.instance.submitUserDefined(hintStore, descriptors, (int) (System.currentTimeMillis() / 1000)).get();
    }
    catch (InterruptedException | ExecutionException e)
    {
        throw new RuntimeException(e);
    }
}
 

public void cleanup()
{
    List<String> files = readLockfile(lockfile);
    for (String file : files)
    {
        try
        {
            Descriptor desc = Descriptor.fromFilename(file, true);
            SSTable.delete(desc, SSTable.componentsFor(desc));
        }
        catch (Exception e)
        {
            JVMStabilityInspector.inspectThrowable(e);
            logger.warn("failed to delete a potentially stale sstable {}", file);
        }
    }
}
 

public void create(Collection<SSTableWriter> sstables)
{
    List<String> sstablePaths = new ArrayList<>(sstables.size());
    for (SSTableWriter writer : sstables)
    {
        /* write out the file names *without* the 'tmp-file' flag in the file name.
           this class will not need to clean up tmp files (on restart), CassandraDaemon does that already,
           just make sure we delete the fully-formed SSTRs. */
        sstablePaths.add(writer.descriptor.asType(Descriptor.Type.FINAL).baseFilename());
    }

    try
    {
        Files.write(lockfile.toPath(), sstablePaths, Charsets.UTF_8,
                StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE, StandardOpenOption.DSYNC);
    }
    catch (IOException e)
    {
        logger.warn(String.format("Could not create lockfile %s for stream session, nothing to worry too much about", lockfile), e);
    }
}
 

private static synchronized ColumnFamilyStore createColumnFamilyStore(Keyspace keyspace,
                                                                     String columnFamily,
                                                                     IPartitioner partitioner,
                                                                     CFMetaData metadata,
                                                                     boolean loadSSTables)
{
    // get the max generation number, to prevent generation conflicts
    Directories directories = new Directories(metadata);
    Directories.SSTableLister lister = directories.sstableLister().includeBackups(true);
    List<Integer> generations = new ArrayList<Integer>();
    for (Map.Entry<Descriptor, Set<Component>> entry : lister.list().entrySet())
    {
        Descriptor desc = entry.getKey();
        generations.add(desc.generation);
        if (!desc.isCompatible())
            throw new RuntimeException(String.format("Incompatible SSTable found. Current version %s is unable to read file: %s. Please run upgradesstables.",
                                                      Descriptor.Version.CURRENT, desc));
    }
    Collections.sort(generations);
    int value = (generations.size() > 0) ? (generations.get(generations.size() - 1)) : 0;

    return new ColumnFamilyStore(keyspace, columnFamily, partitioner, value, metadata, directories, loadSSTables);
}
 

public void writeFullChecksum(Descriptor descriptor)
{
    File outFile = new File(descriptor.filenameFor(Component.DIGEST));
    BufferedWriter out = null;
    try
    {
        out = Files.newBufferedWriter(outFile.toPath(), Charsets.UTF_8);
        out.write(String.valueOf(fullChecksum.getValue()));
    }
    catch (IOException e)
    {
        throw new FSWriteError(e, outFile);
    }
    finally
    {
        FileUtils.closeQuietly(out);
    }
}
 

@Test
public void testStandardDirs()
{
    for (CFMetaData cfm : CFM)
    {
        Directories directories = new Directories(cfm);
        assertEquals(cfDir(cfm), directories.getDirectoryForNewSSTables());

        Descriptor desc = new Descriptor(cfDir(cfm), KS, cfm.cfName, 1, Descriptor.Type.FINAL);
        File snapshotDir = new File(cfDir(cfm),  File.separator + Directories.SNAPSHOT_SUBDIR + File.separator + "42");
        assertEquals(snapshotDir, Directories.getSnapshotDirectory(desc, "42"));

        File backupsDir = new File(cfDir(cfm),  File.separator + Directories.BACKUPS_SUBDIR);
        assertEquals(backupsDir, Directories.getBackupsDirectory(desc));
    }
}
 

@Test
public void testMTSnapshots() throws Exception
{
    for (final CFMetaData cfm : CFM)
    {
        final Directories directories = new Directories(cfm);
        assertEquals(cfDir(cfm), directories.getDirectoryForNewSSTables());
        final String n = Long.toString(System.nanoTime());
        Callable<File> directoryGetter = new Callable<File>() {
            public File call() throws Exception {
                Descriptor desc = new Descriptor(cfDir(cfm), KS, cfm.cfName, 1, Descriptor.Type.FINAL);
                return Directories.getSnapshotDirectory(desc, n);
            }
        };
        List<Future<File>> invoked = Executors.newFixedThreadPool(2).invokeAll(Arrays.asList(directoryGetter, directoryGetter));
        for(Future<File> fut:invoked) {
            assertTrue(fut.get().exists());
        }
    }
}
 

@Test
/* 
 *  The schema is 
 *      CREATE TABLE cql_keyspace.table1 (k int PRIMARY KEY, v1 text, v2 int)
 * */
public void shouldImportCqlTable() throws IOException, URISyntaxException
{
    String cql_keyspace = "cql_keyspace";
    String cql_table = "table1";
    String jsonUrl = resourcePath("CQLTable.json");
    File tempSS = tempSSTableFile(cql_keyspace, cql_table);
    new SSTableImport(true).importJson(jsonUrl, cql_keyspace, cql_table, tempSS.getPath());
    SSTableReader reader = SSTableReader.open(Descriptor.fromFilename(tempSS.getPath()));
    Keyspace.open(cql_keyspace).getColumnFamilyStore(cql_table).addSSTable(reader);
    
    UntypedResultSet result = QueryProcessor.executeOnceInternal(String.format("SELECT * FROM %s.%s", cql_keyspace, cql_table));
    assertThat(result.size(), is(2));
    assertThat(result, hasItem(withElements(1, "NY", 1980)));
    assertThat(result, hasItem(withElements(2, "CA", 2014)));
    reader.selfRef().release();
}
 

private static Stream.TableProgress getTableProgressFromFile(ProgressInfo progressInfo) {
  // the value of ProgressInfo.fileName is different depending on the direction of the stream
  // when sending, the file name is an absolute path of a SSTable
  // when receiving, it's just keyspace/table (probably because C* doesn't know the final file name yet)
  String ksTable;
  if (progressInfo.direction == ProgressInfo.Direction.OUT) {
    Descriptor descriptor = Descriptor.fromFilename(progressInfo.fileName);
    ksTable = String.format("%s.%s", descriptor.ksname, descriptor.cfname);
  } else {
    ksTable = progressInfo.fileName.replace('/', '.');
  }

  long currentBytes = progressInfo.currentBytes;
  long totalBytes = progressInfo.totalBytes;

  return Stream.TableProgress.builder()
      .withTable(ksTable)
      .withCurrent(currentBytes)
      .withTotal(totalBytes)
      .build();
}
 

public StreamReader(FileMessageHeader header, StreamSession session)
{
    this.session = session;
    this.cfId = header.cfId;
    this.estimatedKeys = header.estimatedKeys;
    this.sections = header.sections;
    this.inputVersion = new Descriptor.Version(header.version);
    this.repairedAt = header.repairedAt;
}
 

public KeyCacheKey(UUID cfId, Descriptor desc, ByteBuffer key)
{
    this.cfId = cfId;
    this.desc = desc;
    this.key = ByteBufferUtil.getArray(key);
    assert this.key != null;
}
 

public View(ColumnIndex index, AbstractType<?> keyValidator,
            Collection<SSTableIndex> currentView,
            Collection<SSTableReader> oldSSTables,
            Set<SSTableIndex> newIndexes)
{
    Map<Descriptor, SSTableIndex> newView = new HashMap<>();

    AbstractType<?> validator = index.getValidator();
    TermTree.Builder termTreeBuilder = (validator instanceof AsciiType || validator instanceof UTF8Type)
                                        ? new PrefixTermTree.Builder(index.getMode().mode, validator)
                                        : new RangeTermTree.Builder(index.getMode().mode, validator);

    List<Interval<ByteBuffer, SSTableIndex>> keyIntervals = new ArrayList<>();
    for (SSTableIndex sstableIndex : Iterables.concat(currentView, newIndexes))
    {
        SSTableReader sstable = sstableIndex.getSSTable();
        if (oldSSTables.contains(sstable) || sstable.isMarkedCompacted() || newView.containsKey(sstable.descriptor))
        {
            sstableIndex.release();
            continue;
        }

        newView.put(sstable.descriptor, sstableIndex);

        termTreeBuilder.add(sstableIndex);
        keyIntervals.add(Interval.create(sstableIndex.minKey(), sstableIndex.maxKey(), sstableIndex));
    }

    this.view = newView;
    this.termTree = termTreeBuilder.build();
    this.keyIntervalTree = IntervalTree.build(keyIntervals, keyValidator);

    if (keyIntervalTree.intervalCount() != termTree.intervalCount())
        throw new IllegalStateException(String.format("mismatched sizes for intervals tree for keys vs terms: %d != %d", keyIntervalTree.intervalCount(), termTree.intervalCount()));
}
 

protected SSTableWriter createWriter(ColumnFamilyStore cfs, long totalSize, long repairedAt) throws IOException
{
    Directories.DataDirectory localDir = cfs.directories.getWriteableLocation(totalSize);
    if (localDir == null)
        throw new IOException("Insufficient disk space to store " + totalSize + " bytes");
    desc = Descriptor.fromFilename(cfs.getTempSSTablePath(cfs.directories.getLocationForDisk(localDir)));

    return new SSTableWriter(desc.filenameFor(Component.DATA), estimatedKeys, repairedAt);
}
 

public PerSSTableIndexWriter(AbstractType<?> keyValidator,
                             Descriptor descriptor,
                             Source source,
                             Map<ByteBuffer, ColumnIndex> supportedIndexes)
{
    this.keyValidator = keyValidator;
    this.descriptor = descriptor;
    this.source = source;
    this.supportedIndexes = supportedIndexes;
    this.indexes = new HashMap<>();
}
 

private void cleanup() throws ExecutionException, InterruptedException
{
    ColumnFamilyStore cfs = Keyspace.open(Keyspace.SYSTEM_KS).getColumnFamilyStore(SystemKeyspace.BATCHLOG_CF);
    cfs.forceBlockingFlush();
    Collection<Descriptor> descriptors = new ArrayList<>();
    for (SSTableReader sstr : cfs.getSSTables())
        descriptors.add(sstr.descriptor);
    if (!descriptors.isEmpty()) // don't pollute the logs if there is nothing to compact.
        CompactionManager.instance.submitUserDefined(cfs, descriptors, Integer.MAX_VALUE).get();
}
 

public Compact(String... args) {
    for(String path : args) {
        try {
            for (File f : CassandraUtils.sstablesFromPath(path)) {
                if (metadata == null) {
                    metadata = CassandraUtils.tableFromSSTable(f);
                }
                Descriptor d = Descriptor.fromFilename(f.getAbsolutePath());
                sstables.add(SSTableReader.openNoValidation(d, metadata));
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}
 

public CompactionMetadata deserialize(Descriptor.Version version, DataInput in) throws IOException
{
    int nbAncestors = in.readInt();
    Set<Integer> ancestors = new HashSet<>(nbAncestors);
    for (int i = 0; i < nbAncestors; i++)
        ancestors.add(in.readInt());
    ICardinality cardinality = HyperLogLogPlus.Builder.build(ByteBufferUtil.readBytes(in, in.readInt()));
    return new CompactionMetadata(ancestors, cardinality);
}
 

public void mutateLevel(Descriptor descriptor, int newLevel) throws IOException
{
    logger.debug("Mutating {} to level {}", descriptor.filenameFor(Component.STATS), newLevel);
    Map<MetadataType, MetadataComponent> currentComponents = deserialize(descriptor, EnumSet.allOf(MetadataType.class));
    StatsMetadata stats = (StatsMetadata) currentComponents.remove(MetadataType.STATS);
    // mutate level
    currentComponents.put(MetadataType.STATS, stats.mutateLevel(newLevel));
    rewriteSSTableMetadata(descriptor, currentComponents);
}
 

public void mutateRepairedAt(Descriptor descriptor, long newRepairedAt) throws IOException
{
    logger.debug("Mutating {} to repairedAt time {}", descriptor.filenameFor(Component.STATS), newRepairedAt);
    Map<MetadataType, MetadataComponent> currentComponents = deserialize(descriptor, EnumSet.allOf(MetadataType.class));
    StatsMetadata stats = (StatsMetadata) currentComponents.remove(MetadataType.STATS);
    // mutate level
    currentComponents.put(MetadataType.STATS, stats.mutateRepairedAt(newRepairedAt));
    rewriteSSTableMetadata(descriptor, currentComponents);
}
 

private void rewriteSSTableMetadata(Descriptor descriptor, Map<MetadataType, MetadataComponent> currentComponents) throws IOException
{
    Descriptor tmpDescriptor = descriptor.asType(Descriptor.Type.TEMP);

    try (DataOutputStreamAndChannel out = new DataOutputStreamAndChannel(new FileOutputStream(tmpDescriptor.filenameFor(Component.STATS))))
    {
        serialize(currentComponents, out);
        out.flush();
    }
    // we cant move a file on top of another file in windows:
    if (FBUtilities.isWindows())
        FileUtils.delete(descriptor.filenameFor(Component.STATS));
    FileUtils.renameWithConfirm(tmpDescriptor.filenameFor(Component.STATS), descriptor.filenameFor(Component.STATS));

}
 

public ChecksumValidator(Descriptor descriptor) throws IOException
{
    this.descriptor = descriptor;
    checksum = descriptor.version.hasAllAdlerChecksums ? new Adler32() : new PureJavaCrc32();
    reader = RandomAccessReader.open(new File(descriptor.filenameFor(Component.CRC)));
    chunkSize = reader.readInt();
}
 

public static Future<?> compactAll(ColumnFamilyStore cfs, int gcBefore)
{
    List<Descriptor> descriptors = new ArrayList<>();
    for (SSTableReader sstable : cfs.getSSTables())
        descriptors.add(sstable.descriptor);
    return CompactionManager.instance.submitUserDefined(cfs, descriptors, gcBefore);
}
 

@Test
public void testBackupAfterFlush() throws Throwable
{
    ColumnFamilyStore cfs = insertKey1Key2();

    for (int version = 1; version <= 2; ++version)
    {
        Descriptor existing = new Descriptor(cfs.directories.getDirectoryForNewSSTables(), "Keyspace2", "Standard1", version, Descriptor.Type.FINAL);
        Descriptor desc = new Descriptor(Directories.getBackupsDirectory(existing), "Keyspace2", "Standard1", version, Descriptor.Type.FINAL);
        for (Component c : new Component[]{ Component.DATA, Component.PRIMARY_INDEX, Component.FILTER, Component.STATS })
            assertTrue("can not find backedup file:" + desc.filenameFor(c), new File(desc.filenameFor(c)).exists());
    }
}
 

private static void createFakeSSTable(File dir, String cf, int gen, boolean temp, List<File> addTo) throws IOException
{
    Descriptor desc = new Descriptor(dir, KS, cf, gen, temp ? Descriptor.Type.TEMP : Descriptor.Type.FINAL);
    for (Component c : new Component[]{ Component.DATA, Component.PRIMARY_INDEX, Component.FILTER })
    {
        File f = new File(desc.filenameFor(c));
        f.createNewFile();
        addTo.add(f);
    }
}
 

private void checkVersionFromFilename(String filename) {
    Descriptor descriptor = Descriptor.fromFilename(filename);

    if (this.version == null) {
        this.version = descriptor.version;
    } else if (!this.version.equals(descriptor.version)) {
        throw new IllegalStateException("All files must have the same sstable version.  File '" + filename
                + "' has version '" + descriptor.version + "' and we have already seen a file with version '"
                + version + "'");
    }
}
 

public SSTableColumnScanner(InputStream is, long start, long end, Descriptor.Version version) throws IOException {
    this.version = version;
    this.start = start;
    this.end = end;
    this.input = new DataInputStream(is);
    if (this.start > 0) {
        LOG.info("skipping to start: {}", start);
        skipUnsafe(start);
    }
    this.pos = start;
}