org.apache.cassandra.db.DecoratedKey Java Examples

private ColumnFamilyStore prepareColumnFamilyStore()
{
    Keyspace keyspace = Keyspace.open(KEYSPACE1);
    ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF);
    store.truncateBlocking();
    store.disableAutoCompaction();
    long timestamp = System.currentTimeMillis();
    for (int i = 0; i < 10; i++)
    {
        DecoratedKey key = Util.dk(Integer.toString(i));
        Mutation rm = new Mutation(KEYSPACE1, key.getKey());
        for (int j = 0; j < 10; j++)
            rm.add("Standard1", Util.cellname(Integer.toString(j)),
                   ByteBufferUtil.EMPTY_BYTE_BUFFER,
                   timestamp,
                   0);
        rm.apply();
    }
    store.forceBlockingFlush();
    return store;
}
 

@Override
public void add(ByteBuffer value, ByteBuffer key)
{
    final DecoratedKey dk = StorageService.getPartitioner().decorateKey(key);
    ConcurrentSkipListSet<DecoratedKey> keys = index.get(value);

    if (keys == null)
    {
        ConcurrentSkipListSet<DecoratedKey> newKeys = new ConcurrentSkipListSet<>(DecoratedKey.comparator);
        keys = index.putIfAbsent(value, newKeys);
        if (keys == null)
            keys = newKeys;
    }

    keys.add(dk);
}
 

@Override
public void add(ByteBuffer value, ByteBuffer key)
{
    final DecoratedKey dk = StorageService.getPartitioner().decorateKey(key);

    AbstractAnalyzer analyzer = columnIndex.getAnalyzer();
    analyzer.reset(value.duplicate());

    while (analyzer.hasNext())
    {
        ByteBuffer term = analyzer.next();

        if (term.remaining() >= OnDiskIndexBuilder.MAX_TERM_SIZE)
        {
            logger.info("Can't add term of column {} to index for key: {}, term size {} bytes, max allowed size {} bytes, use analyzed = true (if not yet set) for that column.",
                        columnIndex.getColumnName(),
                        keyValidator.getString(key),
                        term.remaining(),
                        OnDiskIndexBuilder.MAX_TERM_SIZE);
            continue;
        }

        index.add(columnIndex.getValidator().getString(term), dk);
    }
}
 

/**
 *
 * @param decoratedKey the key for this record
 * @param indexStart the position in the index file this record begins
 * @param indexEnd the position in the index file we need to be able to read to (exclusive) to read this record
 * @param dataEnd the position in the data file we need to be able to read to (exclusive) to read this record
 *                a value of 0 indicates we are not tracking readable boundaries
 */
public IndexSummaryBuilder maybeAddEntry(DecoratedKey decoratedKey, long indexStart, long indexEnd, long dataEnd)
{
    if (keysWritten == nextSamplePosition)
    {
        assert entries.length() <= Integer.MAX_VALUE;
        offsets.writeInt((int) entries.length());
        entries.write(decoratedKey.getKey());
        entries.writeLong(indexStart);
        setNextSamplePosition(keysWritten);
    }
    else if (dataEnd != 0 && keysWritten + 1 == nextSamplePosition)
    {
        // this is the last key in this summary interval, so stash it
        ReadableBoundary boundary = new ReadableBoundary(decoratedKey, indexEnd, dataEnd, (int)(offsets.length() / 4), entries.length());
        lastReadableByData.put(dataEnd, boundary);
        lastReadableByIndex.put(indexEnd, boundary);
    }
    keysWritten++;

    return this;
}
 

/**
 * @return list of Token ranges (_not_ keys!) together with estimated key count,
 *      breaking up the data this node is responsible for into pieces of roughly keysPerSplit
 */
public List<Pair<Range<Token>, Long>> getSplits(String keyspaceName, String cfName, Range<Token> range, int keysPerSplit)
{
    Keyspace t = Keyspace.open(keyspaceName);
    ColumnFamilyStore cfs = t.getColumnFamilyStore(cfName);
    List<DecoratedKey> keys = keySamples(Collections.singleton(cfs), range);

    long totalRowCountEstimate = cfs.estimatedKeysForRange(range);

    // splitCount should be much smaller than number of key samples, to avoid huge sampling error
    int minSamplesPerSplit = 4;
    int maxSplitCount = keys.size() / minSamplesPerSplit + 1;
    int splitCount = Math.max(1, Math.min(maxSplitCount, (int)(totalRowCountEstimate / keysPerSplit)));

    List<Token> tokens = keysToTokens(range, keys);
    return getSplits(tokens, splitCount, cfs);
}
 

public void append(DecoratedKey decoratedKey, ColumnFamily cf)
{
    if (decoratedKey.getKey().remaining() > FBUtilities.MAX_UNSIGNED_SHORT)
    {
        logger.error("Key size {} exceeds maximum of {}, skipping row",
                     decoratedKey.getKey().remaining(),
                     FBUtilities.MAX_UNSIGNED_SHORT);
        return;
    }

    long startPosition = beforeAppend(decoratedKey);
    long endPosition;
    try
    {
        RowIndexEntry entry = rawAppend(cf, startPosition, decoratedKey, dataFile.stream);
        endPosition = dataFile.getFilePointer();
        afterAppend(decoratedKey, endPosition, entry);
    }
    catch (IOException e)
    {
        throw new FSWriteError(e, dataFile.getPath());
    }
    sstableMetadataCollector.update(endPosition - startPosition, cf.getColumnStats());
}
 

public void append(DecoratedKey key, RowIndexEntry indexEntry, long dataEnd)
{
    bf.add(key.getKey());
    long indexStart = indexFile.getFilePointer();
    try
    {
        ByteBufferUtil.writeWithShortLength(key.getKey(), indexFile.stream);
        metadata.comparator.rowIndexEntrySerializer().serialize(indexEntry, indexFile.stream);
    }
    catch (IOException e)
    {
        throw new FSWriteError(e, indexFile.getPath());
    }
    long indexEnd = indexFile.getFilePointer();

    if (logger.isTraceEnabled())
        logger.trace("wrote index entry: " + indexEntry + " at " + indexStart);

    summary.maybeAddEntry(key, indexStart, indexEnd, dataEnd);
    builder.addPotentialBoundary(indexStart);
}
 

public int binarySearch(RowPosition key)
{
    int low = 0, mid = offsetCount, high = mid - 1, result = -1;
    while (low <= high)
    {
        mid = (low + high) >> 1;
        result = -DecoratedKey.compareTo(partitioner, ByteBuffer.wrap(getKey(mid)), key);
        if (result > 0)
        {
            low = mid + 1;
        }
        else if (result == 0)
        {
            return mid;
        }
        else
        {
            high = mid - 1;
        }
    }

    return -mid - (result < 0 ? 1 : 2);
}
 

/**
 * @return a QueryFilter object to satisfy the given slice criteria:
 * @param key the row to slice
 * @param cfName column family to query
 * @param start column to start slice at, inclusive; empty for "the first column"
 * @param finish column to stop slice at, inclusive; empty for "the last column"
 * @param reversed true to start with the largest column (as determined by configured sort order) instead of smallest
 * @param limit maximum number of non-deleted columns to return
 * @param timestamp time to use for determining expiring columns' state
 */
public static QueryFilter getSliceFilter(DecoratedKey key,
                                         String cfName,
                                         Composite start,
                                         Composite finish,
                                         boolean reversed,
                                         int limit,
                                         long timestamp)
{
    return new QueryFilter(key, cfName, new SliceQueryFilter(start, finish, reversed, limit), timestamp);
}
 

@Override
public Iterator<DecoratedKey> iterator()
{
    List<DecoratedKey> keys = new ArrayList<>(offsets.size());
    for (LongCursor offset : offsets)
         keys.add(dk(offset.value));

    return keys.iterator();
}
 

private static Set<DecoratedKey> convert(RangeIterator<Long, Token> results)
{
    if (results == null)
        return Collections.emptySet();

    Set<DecoratedKey> keys = new TreeSet<>(DecoratedKey.comparator);

    while (results.hasNext())
    {
        for (DecoratedKey key : results.next())
            keys.add(key);
    }

    return keys;
}
 

private static Set<DecoratedKey> convert(TokenTreeBuilder offsets)
{
    Set<DecoratedKey> result = new HashSet<>();

    Iterator<Pair<Long, LongSet>> offsetIter = offsets.iterator();
    while (offsetIter.hasNext())
    {
        LongSet v = offsetIter.next().right;

        for (LongCursor offset : v)
            result.add(keyAt(offset.value));
    }
    return result;
}
 

/**
 * Finds and returns the first key beyond a given token in this SSTable or null if no such key exists.
 */
public DecoratedKey firstKeyBeyond(RowPosition token)
{
    if (token.compareTo(first) < 0)
        return first;

    long sampledPosition = getIndexScanPosition(token);

    Iterator<FileDataInput> segments = ifile.iterator(sampledPosition);
    while (segments.hasNext())
    {
        FileDataInput in = segments.next();
        try
        {
            while (!in.isEOF())
            {
                ByteBuffer indexKey = ByteBufferUtil.readWithShortLength(in);
                DecoratedKey indexDecoratedKey = partitioner.decorateKey(indexKey);
                if (indexDecoratedKey.compareTo(token) > 0)
                    return indexDecoratedKey;

                RowIndexEntry.Serializer.skip(in);
            }
        }
        catch (IOException e)
        {
            markSuspect();
            throw new CorruptSSTableException(e, in.getPath());
        }
        finally
        {
            FileUtils.closeQuietly(in);
        }
    }

    return null;
}
 

private void seekToCurrentRangeStart()
{
    long indexPosition = sstable.getIndexScanPosition(currentRange.left);
    ifile.seek(indexPosition);
    try
    {

        while (!ifile.isEOF())
        {
            indexPosition = ifile.getFilePointer();
            DecoratedKey indexDecoratedKey = sstable.partitioner.decorateKey(ByteBufferUtil.readWithShortLength(ifile));
            if (indexDecoratedKey.compareTo(currentRange.left) > 0 || currentRange.contains(indexDecoratedKey))
            {
                // Found, just read the dataPosition and seek into index and data files
                long dataPosition = ifile.readLong();
                ifile.seek(indexPosition);
                dfile.seek(dataPosition);
                break;
            }
            else
            {
                RowIndexEntry.Serializer.skip(ifile);
            }
        }
    }
    catch (IOException e)
    {
        sstable.markSuspect();
        throw new CorruptSSTableException(e, sstable.getFilename());
    }
}
 

@Test
public void testDescriptor() throws Exception
{
    final Map<ByteBuffer, Pair<DecoratedKey, Long>> data = new HashMap<ByteBuffer, Pair<DecoratedKey, Long>>()
    {{
            put(Int32Type.instance.decompose(5), Pair.create(keyAt(1L), 1L));
    }};

    OnDiskIndexBuilder builder1 = new OnDiskIndexBuilder(UTF8Type.instance, Int32Type.instance, OnDiskIndexBuilder.Mode.ORIGINAL);
    OnDiskIndexBuilder builder2 = new OnDiskIndexBuilder(UTF8Type.instance, Int32Type.instance, OnDiskIndexBuilder.Mode.ORIGINAL);
    for (Map.Entry<ByteBuffer, Pair<DecoratedKey, Long>> e : data.entrySet())
    {
        DecoratedKey key = e.getValue().left;
        Long position = e.getValue().right;

        builder1.add(e.getKey(), key, position);
        builder2.add(e.getKey(), key, position);
    }

    File index1 = File.createTempFile("on-disk-sa-int", "db");
    File index2 = File.createTempFile("on-disk-sa-int2", "db");
    index1.deleteOnExit();
    index2.deleteOnExit();

    builder1.finish(index1);
    builder2.finish(new Descriptor(Descriptor.VERSION_AA), index2);

    OnDiskIndex onDisk1 = new OnDiskIndex(index1, Int32Type.instance, new KeyConverter());
    OnDiskIndex onDisk2 = new OnDiskIndex(index2, Int32Type.instance, new KeyConverter());

    ByteBuffer number = Int32Type.instance.decompose(5);

    Assert.assertEquals(Collections.singleton(data.get(number).left), convert(onDisk1.search(expressionFor(Int32Type.instance, number))));
    Assert.assertEquals(Collections.singleton(data.get(number).left), convert(onDisk2.search(expressionFor(Int32Type.instance, number))));

    Assert.assertEquals(onDisk1.descriptor.version.version, Descriptor.CURRENT_VERSION);
    Assert.assertEquals(onDisk2.descriptor.version.version, Descriptor.VERSION_AA);
}
 

@Test
public void testMultiSuffixMatches() throws Exception
{
    OnDiskIndexBuilder builder = new OnDiskIndexBuilder(UTF8Type.instance, UTF8Type.instance, OnDiskIndexBuilder.Mode.SUFFIX)
    {{
            addAll(this, UTF8Type.instance.decompose("Eliza"), keyBuilder(1L, 2L));
            addAll(this, UTF8Type.instance.decompose("Elizabeth"), keyBuilder(3L, 4L));
            addAll(this, UTF8Type.instance.decompose("Aliza"), keyBuilder(5L, 6L));
            addAll(this, UTF8Type.instance.decompose("Taylor"), keyBuilder(7L, 8L));
            addAll(this, UTF8Type.instance.decompose("Pavel"), keyBuilder(9L, 10L));
    }};

    File index = File.createTempFile("on-disk-sa-multi-suffix-match", ".db");
    index.deleteOnExit();

    builder.finish(index);

    OnDiskIndex onDisk = new OnDiskIndex(index, UTF8Type.instance, new KeyConverter());

    Assert.assertEquals(convert(1, 2, 3, 4, 5, 6), convert(onDisk.search(expressionFor("liz"))));
    Assert.assertEquals(convert(1, 2, 3, 4, 5, 6, 7, 8, 9, 10), convert(onDisk.search(expressionFor("a"))));
    Assert.assertEquals(convert(5, 6), convert(onDisk.search(expressionFor("A"))));
    Assert.assertEquals(convert(1, 2, 3, 4), convert(onDisk.search(expressionFor("E"))));
    Assert.assertEquals(convert(1, 2, 3, 4, 5, 6, 7, 8, 9, 10), convert(onDisk.search(expressionFor("l"))));
    Assert.assertEquals(convert(3, 4), convert(onDisk.search(expressionFor("bet"))));
    Assert.assertEquals(convert(3, 4, 9, 10), convert(onDisk.search(expressionFor("e"))));
    Assert.assertEquals(convert(7, 8), convert(onDisk.search(expressionFor("yl"))));
    Assert.assertEquals(convert(7, 8), convert(onDisk.search(expressionFor("T"))));
    Assert.assertEquals(convert(1, 2, 3, 4, 5, 6), convert(onDisk.search(expressionFor("za"))));
    Assert.assertEquals(convert(3, 4), convert(onDisk.search(expressionFor("ab"))));

    Assert.assertEquals(Collections.<DecoratedKey>emptySet(), convert(onDisk.search(expressionFor("Pi"))));
    Assert.assertEquals(Collections.<DecoratedKey>emptySet(), convert(onDisk.search(expressionFor("ethz"))));
    Assert.assertEquals(Collections.<DecoratedKey>emptySet(), convert(onDisk.search(expressionFor("liw"))));
    Assert.assertEquals(Collections.<DecoratedKey>emptySet(), convert(onDisk.search(expressionFor("Taw"))));
    Assert.assertEquals(Collections.<DecoratedKey>emptySet(), convert(onDisk.search(expressionFor("Av"))));

    onDisk.close();
}
 

@Override
public Iterator<DecoratedKey> iterator()
{
    List<Iterator<DecoratedKey>> keys = new ArrayList<>(info.size());

    for (TokenInfo i : info)
        keys.add(i.iterator());

    if (!loadedKeys.isEmpty())
        keys.add(loadedKeys.iterator());

    return MergeIterator.get(keys, DecoratedKey.comparator, new MergeIterator.Reducer<DecoratedKey, DecoratedKey>()
    {
        DecoratedKey reduced = null;

        @Override
        public boolean trivialReduceIsTrivial()
        {
            return true;
        }

        @Override
        public void reduce(DecoratedKey current)
        {
            reduced = current;
        }

        @Override
        protected DecoratedKey getReduced()
        {
            return reduced;
        }
    });
}
 

/**
 * Returns the CQL3 {@link Row} identified by the specified key pair, using the specified time stamp to ignore
 * deleted columns. The {@link Row} is retrieved from the storage engine, so it involves IO operations.
 *
 * @param partitionKey The partition key.
 * @param timestamp    The time stamp to ignore deleted columns.
 * @return The CQL3 {@link Row} identified by the specified key pair.
 */
private Row row(DecoratedKey partitionKey, long timestamp) {
    QueryFilter queryFilter = QueryFilter.getIdentityFilter(partitionKey, metadata.cfName, timestamp);
    ColumnFamily columnFamily = baseCfs.getColumnFamily(queryFilter);
    if (columnFamily != null) {
        ColumnFamily cleanColumnFamily = cleanExpired(columnFamily, timestamp);
        return new Row(partitionKey, cleanColumnFamily);
    }
    return null;
}
 

public long getBytesRead()
{
    long m = 0;
    for (Iterator<DecoratedKey> iter : mi.iterators())
    {
        m += ((KeyIterator) iter).getBytesRead();
    }
    return m;
}
 

public DigestMismatchException(DecoratedKey key, ByteBuffer digest1, ByteBuffer digest2)
{
    super(String.format("Mismatch for key %s (%s vs %s)",
                        key.toString(),
                        ByteBufferUtil.bytesToHex(digest1),
                        ByteBufferUtil.bytesToHex(digest2)));
}
 

private List<Token> keysToTokens(Range<Token> range, List<DecoratedKey> keys)
{
    List<Token> tokens = Lists.newArrayListWithExpectedSize(keys.size() + 2);
    tokens.add(range.left);
    for (DecoratedKey key : keys)
        tokens.add(key.getToken());
    tokens.add(range.right);
    return tokens;
}
 

private List<DecoratedKey> keySamples(Iterable<ColumnFamilyStore> cfses, Range<Token> range)
{
    List<DecoratedKey> keys = new ArrayList<>();
    for (ColumnFamilyStore cfs : cfses)
        Iterables.addAll(keys, cfs.keySamples(range));
    FBUtilities.sortSampledKeys(keys, range);
    return keys;
}
 

/**
 * #{@inheritDoc}
 */
public List<String> sampleKeyRange() // do not rename to getter - see CASSANDRA-4452 for details
{
    List<DecoratedKey> keys = new ArrayList<>();
    for (Keyspace keyspace : Keyspace.nonSystem())
    {
        for (Range<Token> range : getPrimaryRangesForEndpoint(keyspace.getName(), FBUtilities.getBroadcastAddress()))
            keys.addAll(keySamples(keyspace.getColumnFamilyStores(), range));
    }

    List<String> sampledKeys = new ArrayList<>(keys.size());
    for (DecoratedKey key : keys)
        sampledKeys.add(key.getToken().toString());
    return sampledKeys;
}
 

public ColumnFamily prune(DecoratedKey rowKey, ColumnFamily data)
{
    if (optimizedFilter == null)
        return data;

    ColumnFamily pruned = data.cloneMeShallow();
    IDiskAtomFilter filter = dataRange.columnFilter(rowKey.getKey());
    Iterator<Cell> iter = filter.getColumnIterator(data);
    filter.collectReducedColumns(pruned, QueryFilter.gatherTombstones(pruned, iter), cfs.gcBefore(timestamp), timestamp);
    return pruned;
}
 

@Test
public void testFilterOldSSTables()
{
    Keyspace keyspace = Keyspace.open(KEYSPACE1);
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_STANDARD1);
    cfs.truncateBlocking();
    cfs.disableAutoCompaction();

    ByteBuffer value = ByteBuffer.wrap(new byte[100]);

    // create 3 sstables
    int numSSTables = 3;
    for (int r = 0; r < numSSTables; r++)
    {
        DecoratedKey key = Util.dk(String.valueOf(r));
        Mutation rm = new Mutation(KEYSPACE1, key.getKey());
        rm.add(CF_STANDARD1, Util.cellname("column"), value, r);
        rm.apply();
        cfs.forceBlockingFlush();
    }
    cfs.forceBlockingFlush();

    Iterable<SSTableReader> filtered;
    List<SSTableReader> sstrs = new ArrayList<>(cfs.getSSTables());

    filtered = filterOldSSTables(sstrs, 0, 2);
    assertEquals("when maxSSTableAge is zero, no sstables should be filtered", sstrs.size(), Iterables.size(filtered));

    filtered = filterOldSSTables(sstrs, 1, 2);
    assertEquals("only the newest 2 sstables should remain", 2, Iterables.size(filtered));

    filtered = filterOldSSTables(sstrs, 1, 3);
    assertEquals("only the newest sstable should remain", 1, Iterables.size(filtered));

    filtered = filterOldSSTables(sstrs, 1, 4);
    assertEquals("no sstables should remain when all are too old", 0, Iterables.size(filtered));
}
 

public ReadableBoundary(DecoratedKey lastKey, long indexLength, long dataLength, int summaryCount, long entriesLength)
{
    this.lastKey = lastKey;
    this.indexLength = indexLength;
    this.dataLength = dataLength;
    this.summaryCount = summaryCount;
    this.entriesLength = entriesLength;
}
 

public IDiskAtomFilter getExtraFilter(DecoratedKey rowKey, ColumnFamily data)
{
    /*
     * This method assumes the IndexExpression names are valid column names, which is not the
     * case with composites. This is ok for now however since:
     * 1) CompositeSearcher doesn't use it.
     * 2) We don't yet allow non-indexed range slice with filters in CQL3 (i.e. this will never be
     * called by CFS.filter() for composites).
     */
    assert !(cfs.getComparator().isCompound()) : "Sequential scan with filters is not supported (if you just created an index, you "
                                                 + "need to wait for the creation to be propagated to all nodes before querying it)";

    if (!needsExtraQuery(rowKey.getKey(), data))
        return null;

    // Note: for counters we must be careful to not add a column that was already there (to avoid overcount). That is
    // why we do the dance of avoiding to query any column we already have (it's also more efficient anyway)
    SortedSet<CellName> columns = new TreeSet<CellName>(cfs.getComparator());
    for (IndexExpression expr : clause)
    {
        CellName name = data.getComparator().cellFromByteBuffer(expr.column);
        if (data.getColumn(name) == null)
            columns.add(name);
    }
    assert !columns.isEmpty();
    return new NamesQueryFilter(columns);
}
 

public static RowIndexEntry rawAppend(ColumnFamily cf, long startPosition, DecoratedKey key, DataOutputPlus out) throws IOException
{
    assert cf.hasColumns() || cf.isMarkedForDelete();

    ColumnIndex.Builder builder = new ColumnIndex.Builder(cf, key.getKey(), out);
    ColumnIndex index = builder.build(cf);

    out.writeShort(END_OF_ROW);
    return RowIndexEntry.create(startPosition, cf.deletionInfo().getTopLevelDeletion(), index);
}
 

public QueryFilter(DecoratedKey key, String cfName, IDiskAtomFilter filter, long timestamp)
{
    this.key = key;
    this.cfName = cfName;
    this.filter = filter;
    this.timestamp = timestamp;
}
 

private SSTableIdentityIterator getIdentityIterator(final ByteBuffer keyBytes, final long dataSize) {
    final DecoratedKey decoratedKey = getDecoratedKey(keyBytes);
    final CFMetaData cfMetaData = getCfMetaData();

    return new SSTableIdentityIterator(cfMetaData, getReader(), getDataPath().toString(), decoratedKey,
            getReader().getFilePointer(), dataSize, IColumnSerializer.Flag.LOCAL);
}