org.apache.cassandra.service.StorageService Java Examples

private SSTableReader writeFile(ColumnFamilyStore cfs, int count)
{
    ArrayBackedSortedColumns cf = ArrayBackedSortedColumns.factory.create(cfs.metadata);
    for (int i = 0; i < count / 100; i++)
        cf.addColumn(Util.cellname(i), random(0, 1000), 1);
    File dir = cfs.directories.getDirectoryForNewSSTables();
    String filename = cfs.getTempSSTablePath(dir);

    SSTableWriter writer = new SSTableWriter(filename,
            0,
            0,
            cfs.metadata,
            StorageService.getPartitioner(),
            new MetadataCollector(cfs.metadata.comparator));

    for (int i = 0; i < count * 5; i++)
        writer.append(StorageService.getPartitioner().decorateKey(ByteBufferUtil.bytes(i)), cf);
    return writer.closeAndOpenReader();
}
 

@Test
public void testRowCacheCleanup() throws Exception
{
    StorageService.instance.initServer(0);
    CacheService.instance.setRowCacheCapacityInMB(1);
    rowCacheLoad(100, Integer.MAX_VALUE, 1000);

    ColumnFamilyStore store = Keyspace.open(KEYSPACE).getColumnFamilyStore(COLUMN_FAMILY);
    assertEquals(CacheService.instance.rowCache.getKeySet().size(), 100);
    store.cleanupCache();
    assertEquals(CacheService.instance.rowCache.getKeySet().size(), 100);
    TokenMetadata tmd = StorageService.instance.getTokenMetadata();
    byte[] tk1, tk2;
    tk1 = "key1000".getBytes();
    tk2 = "key1050".getBytes();
    tmd.updateNormalToken(new BytesToken(tk1), InetAddress.getByName("127.0.0.1"));
    tmd.updateNormalToken(new BytesToken(tk2), InetAddress.getByName("127.0.0.2"));
    store.cleanupCache();
    assertEquals(CacheService.instance.rowCache.getKeySet().size(), 50);
    CacheService.instance.setRowCacheCapacityInMB(0);
}
 

/**
 * be careful about just blindly updating ApplicationState.INTERNAL_IP everytime we read the yaml file,
 * as that can cause connections to get unnecessarily reset (via IESCS.onChange()).
 */
private void maybeSetApplicationState()
{
    if (localNodeData.dcLocalAddress == null)
        return;
    final EndpointState es = Gossiper.instance.getEndpointStateForEndpoint(FBUtilities.getBroadcastAddress());
    if (es == null)
        return;
    final VersionedValue vv = es.getApplicationState(ApplicationState.INTERNAL_IP);
    if ((vv != null && !vv.value.equals(localNodeData.dcLocalAddress.getHostAddress()))
        || vv == null)
    {
        Gossiper.instance.addLocalApplicationState(ApplicationState.INTERNAL_IP,
            StorageService.instance.valueFactory.internalIP(localNodeData.dcLocalAddress.getHostAddress()));
    }
}
 

@Test
public void testCommitFailurePolicy_stop() throws ConfigurationException
{
    // Need storage service active so stop policy can shutdown gossip
    StorageService.instance.initServer();
    Assert.assertTrue(Gossiper.instance.isEnabled());

    Config.CommitFailurePolicy oldPolicy = DatabaseDescriptor.getCommitFailurePolicy();
    try
    {
        DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.stop);
        CommitLog.handleCommitError("Test stop error", new Throwable());
        Assert.assertFalse(Gossiper.instance.isEnabled());
    }
    finally
    {
        DatabaseDescriptor.setCommitFailurePolicy(oldPolicy);
    }
}
 

public static void handleFSError(FSError e)
{
    JVMStabilityInspector.inspectThrowable(e);
    switch (DatabaseDescriptor.getDiskFailurePolicy())
    {
        case stop_paranoid:
        case stop:
            StorageService.instance.stopTransports();
            break;
        case best_effort:
            // for both read and write errors mark the path as unwritable.
            BlacklistedDirectories.maybeMarkUnwritable(e.path);
            if (e instanceof FSReadError)
            {
                File directory = BlacklistedDirectories.maybeMarkUnreadable(e.path);
                if (directory != null)
                    Keyspace.removeUnreadableSSTables(directory);
            }
            break;
        case ignore:
            // already logged, so left nothing to do
            break;
        default:
            throw new IllegalStateException();
    }
}
 

@Test
public void testSatisfiedByWithMultipleTerms()
{
    final ByteBuffer comment = UTF8Type.instance.decompose("comment");
    final ColumnFamilyStore store = Keyspace.open("sasecondaryindex").getColumnFamilyStore("saindexed1");
    final IPartitioner<?> partitioner = StorageService.getPartitioner();

    ColumnFamily cf = ArrayBackedSortedColumns.factory.create(store.metadata);
    cf.addColumn(new Column(comment, UTF8Type.instance.decompose("software engineer is working on a project"), System.currentTimeMillis()));

    Operation.Builder builder = new Operation.Builder(OperationType.AND, UTF8Type.instance, controller,
                                        new IndexExpression(comment, IndexOperator.EQ, UTF8Type.instance.decompose("eng is a work")));
    Operation op = builder.complete();

    Assert.assertTrue(op.satisfiedBy(new Row(partitioner.decorateKey(UTF8Type.instance.decompose("key1")), cf), null, false));

    builder = new Operation.Builder(OperationType.AND, UTF8Type.instance, controller,
                                        new IndexExpression(comment, IndexOperator.EQ, UTF8Type.instance.decompose("soft works fine")));
    op = builder.complete();

    Assert.assertTrue(op.satisfiedBy(new Row(partitioner.decorateKey(UTF8Type.instance.decompose("key1")), cf), null, false));
}
 

public void doVerb(MessageIn<ReadCommand> message, int id)
{
    if (StorageService.instance.isBootstrapMode())
    {
        throw new RuntimeException("Cannot service reads while bootstrapping!");
    }

    ReadCommand command = message.payload;
    Keyspace keyspace = Keyspace.open(command.ksName);
    Row row;
    try
    {
        row = command.getRow(keyspace);
    }
    catch (TombstoneOverwhelmingException e)
    {
        // error already logged.  Drop the request
        return;
    }

    MessageOut<ReadResponse> reply = new MessageOut<ReadResponse>(MessagingService.Verb.REQUEST_RESPONSE,
                                                                  getResponse(command, row),
                                                                  ReadResponse.serializer);
    Tracing.trace("Enqueuing response to {}", message.from);
    MessagingService.instance().sendReply(reply, id, message.from);
}
 

@Test
public void testRacks() throws IOException, ConfigurationException
{
    az = "ch-gva-1";
    CloudstackSnitch snitch = new TestCloudstackSnitch();
    InetAddress local = InetAddress.getByName("127.0.0.1");
    InetAddress nonlocal = InetAddress.getByName("127.0.0.7");

    Gossiper.instance.addSavedEndpoint(nonlocal);
    Map<ApplicationState,VersionedValue> stateMap = Gossiper.instance.getEndpointStateForEndpoint(nonlocal).getApplicationStateMap();
    stateMap.put(ApplicationState.DC, StorageService.instance.valueFactory.datacenter("ch-zrh"));
    stateMap.put(ApplicationState.RACK, StorageService.instance.valueFactory.rack("2"));

    assertEquals("ch-zrh", snitch.getDatacenter(nonlocal));
    assertEquals("2", snitch.getRack(nonlocal));

    assertEquals("ch-gva", snitch.getDatacenter(local));
    assertEquals("1", snitch.getRack(local));

}
 

@Override
public void add(ByteBuffer value, ByteBuffer key)
{
    final DecoratedKey dk = StorageService.getPartitioner().decorateKey(key);
    ConcurrentSkipListSet<DecoratedKey> keys = index.get(value);

    if (keys == null)
    {
        ConcurrentSkipListSet<DecoratedKey> newKeys = new ConcurrentSkipListSet<>(DecoratedKey.comparator);
        keys = index.putIfAbsent(value, newKeys);
        if (keys == null)
            keys = newKeys;
    }

    keys.add(dk);
}
 

public List<CfSplit> describe_splits_ex(String cfName, String start_token, String end_token, int keys_per_split)
throws InvalidRequestException, TException
{
    try
    {
        Token.TokenFactory tf = StorageService.getPartitioner().getTokenFactory();
        Range<Token> tr = new Range<Token>(tf.fromString(start_token), tf.fromString(end_token));
        List<Pair<Range<Token>, Long>> splits =
                StorageService.instance.getSplits(state().getKeyspace(), cfName, tr, keys_per_split);
        List<CfSplit> result = new ArrayList<CfSplit>(splits.size());
        for (Pair<Range<Token>, Long> split : splits)
            result.add(new CfSplit(split.left.left.toString(), split.left.right.toString(), split.right));
        return result;
    }
    catch (RequestValidationException e)
    {
        throw ThriftConversion.toThrift(e);
    }
}
 

@VisibleForTesting
public static boolean handleCommitError(String message, Throwable t)
{
    JVMStabilityInspector.inspectCommitLogThrowable(t);
    switch (DatabaseDescriptor.getCommitFailurePolicy())
    {
        // Needed here for unit tests to not fail on default assertion
        case die:
        case stop:
            StorageService.instance.stopTransports();
        case stop_commit:
            logger.error(String.format("%s. Commit disk failure policy is %s; terminating thread", message, DatabaseDescriptor.getCommitFailurePolicy()), t);
            return false;
        case ignore:
            logger.error(message, t);
            return true;
        default:
            throw new AssertionError(DatabaseDescriptor.getCommitFailurePolicy());
    }
}
 

private final boolean hasNextInternal() throws BackendException {
    ensureOpen();

    if (keys == null)
        return false;

    boolean hasNext = keys.hasNext();

    if (!hasNext && lastSeenKey != null) {
        Token lastSeenToken = StorageService.getPartitioner().getToken(lastSeenKey.duplicate());

        // let's check if we reached key upper bound already so we can skip one useless call to Cassandra
        if (maximumToken != getMinimumToken() && lastSeenToken.equals(maximumToken)) {
            return false;
        }

        List<Row> newKeys = getKeySlice(StorageService.getPartitioner().getToken(lastSeenKey), maximumToken, sliceQuery, pageSize, nowMillis);

        keys = getRowsIterator(newKeys, lastSeenKey);
        hasNext = keys.hasNext();
    }

    return hasNext;
}
 

RangeSliceQueryPager(RangeSliceCommand command, ConsistencyLevel consistencyLevel, boolean localQuery, PagingState state)
{
    this(command, consistencyLevel, localQuery);

    if (state != null)
    {
        lastReturnedKey = StorageService.getPartitioner().decorateKey(state.partitionKey);
        lastReturnedName = cfm.comparator.cellFromByteBuffer(state.cellName);
        restoreState(state.remaining, true);
    }
}
 

public KSMetaData validate() throws ConfigurationException
{
    if (!CFMetaData.isNameValid(name))
        throw new ConfigurationException(String.format("Keyspace name must not be empty, more than %s characters long, or contain non-alphanumeric-underscore characters (got \"%s\")", Schema.NAME_LENGTH, name));

    // Attempt to instantiate the ARS, which will throw a ConfigException if the strategy_options aren't fully formed
    TokenMetadata tmd = StorageService.instance.getTokenMetadata();
    IEndpointSnitch eps = DatabaseDescriptor.getEndpointSnitch();
    AbstractReplicationStrategy.validateReplicationStrategy(name, strategyClass, tmd, eps, strategyOptions);

    for (CFMetaData cfm : cfMetaData.values())
        cfm.validate();

    return this;
}
 

private void transferSSTables(SSTableReader sstable) throws Exception
{
    IPartitioner p = StorageService.getPartitioner();
    List<Range<Token>> ranges = new ArrayList<>();
    ranges.add(new Range<>(p.getMinimumToken(), p.getToken(ByteBufferUtil.bytes("key1"))));
    ranges.add(new Range<>(p.getToken(ByteBufferUtil.bytes("key2")), p.getMinimumToken()));
    transfer(sstable, ranges);
}
 

RangeNamesQueryPager(RangeSliceCommand command, ConsistencyLevel consistencyLevel, boolean localQuery, PagingState state)
{
    this(command, consistencyLevel, localQuery);

    if (state != null)
    {
        lastReturnedKey = StorageService.getPartitioner().decorateKey(state.partitionKey);
        restoreState(state.remaining, true);
    }
}
 

/**
 * @return a list of unreachable token owners
 */
public Set<InetAddress> getUnreachableTokenOwners()
{
    Set<InetAddress> tokenOwners = new HashSet<>();
    for (InetAddress endpoint : unreachableEndpoints.keySet())
    {
        if (StorageService.instance.getTokenMetadata().isMember(endpoint))
            tokenOwners.add(endpoint);
    }

    return tokenOwners;
}
 

private void testBloomFilterWrite(boolean offheap) throws IOException
{
    IFilter bf = FilterFactory.getFilter(1000000, 0.0001, offheap);
    for (int i = 0; i < 100; i++)
        bf.add(StorageService.getPartitioner().getTokenFactory().toByteArray(StorageService.getPartitioner().getRandomToken()));
    DataOutputStreamAndChannel out = getOutput("utils.BloomFilter.bin");
    FilterFactory.serialize(bf, out);
    out.close();
    bf.close();
}
 

@Before
public void setUp() throws Exception
{
    TokenMetadata metadata = StorageService.instance.getTokenMetadata();
    InetAddress localhost = InetAddress.getByName("127.0.0.1");
    metadata.updateNormalToken(Util.token("A"), localhost);
    metadata.updateHostId(UUIDGen.getTimeUUID(), localhost);
}
 

@VisibleForTesting
public void initializeNodeUnsafe(InetAddress addr, UUID uuid, int generationNbr)
{
    HeartBeatState hbState = new HeartBeatState(generationNbr);
    EndpointState newState = new EndpointState(hbState);
    newState.markAlive();
    EndpointState oldState = endpointStateMap.putIfAbsent(addr, newState);
    EndpointState localState = oldState == null ? newState : oldState;

    // always add the version state
    localState.addApplicationState(ApplicationState.NET_VERSION, StorageService.instance.valueFactory.networkVersion());
    localState.addApplicationState(ApplicationState.HOST_ID, StorageService.instance.valueFactory.hostId(uuid));
}
 

/**
 * This method appends a row to the global CommitLog, then updates memtables and indexes.
 *
 * @param mutation       the row to write.  Must not be modified after calling apply, since commitlog append
 *                       may happen concurrently, depending on the CL Executor type.
 * @param writeCommitLog false to disable commitlog append entirely
 * @param updateIndexes  false to disable index updates (used by CollationController "defragmenting")
 */
public void apply(Mutation mutation, boolean writeCommitLog, boolean updateIndexes)
{
    try (OpOrder.Group opGroup = writeOrder.start())
    {
        // write the mutation to the commitlog and memtables
        ReplayPosition replayPosition = null;
        if (writeCommitLog)
        {
            Tracing.trace("Appending to commitlog");
            replayPosition = CommitLog.instance.add(mutation);
        }

        DecoratedKey key = StorageService.getPartitioner().decorateKey(mutation.key());
        for (ColumnFamily cf : mutation.getColumnFamilies())
        {
            ColumnFamilyStore cfs = columnFamilyStores.get(cf.id());
            if (cfs == null)
            {
                logger.error("Attempting to mutate non-existant column family {}", cf.id());
                continue;
            }

            Tracing.trace("Adding to {} memtable", cf.metadata().cfName);
            SecondaryIndexManager.Updater updater = updateIndexes
                                                  ? cfs.indexManager.updaterFor(key, cf, opGroup)
                                                  : SecondaryIndexManager.nullUpdater;
            cfs.apply(key, cf, updater, opGroup, replayPosition);
        }
    }
}
 

/**
 * Gets compaction rate limiter. When compaction_throughput_mb_per_sec is 0 or node is bootstrapping,
 * this returns rate limiter with the rate of Double.MAX_VALUE bytes per second.
 * Rate unit is bytes per sec.
 *
 * @return RateLimiter with rate limit set
 */
public RateLimiter getRateLimiter()
{
    double currentThroughput = DatabaseDescriptor.getCompactionThroughputMbPerSec() * 1024.0 * 1024.0;
    // if throughput is set to 0, throttling is disabled
    if (currentThroughput == 0 || StorageService.instance.isBootstrapMode())
        currentThroughput = Double.MAX_VALUE;
    if (compactionRateLimiter.getRate() != currentThroughput)
        compactionRateLimiter.setRate(currentThroughput);
    return compactionRateLimiter;
}
 

public boolean isFatClient(InetAddress endpoint)
{
    EndpointState epState = endpointStateMap.get(endpoint);
    if (epState == null)
    {
        return false;
    }
    return !isDeadState(epState) && !StorageService.instance.getTokenMetadata().isMember(endpoint);
}
 

public RowPosition deserialize(DataInput in) throws IOException
{
    Kind kind = Kind.fromOrdinal(in.readByte());
    if (kind == Kind.ROW_KEY)
    {
        ByteBuffer k = ByteBufferUtil.readWithShortLength(in);
        return StorageService.getPartitioner().decorateKey(k);
    }
    else
    {
        Token t = Token.serializer.deserialize(in);
        return kind == Kind.MIN_BOUND ? t.minKeyBound() : t.maxKeyBound();
    }
}
 

@Override
public KeyIterator getKeys(KeyRangeQuery keyRangeQuery, StoreTransaction txh) throws BackendException {
    IPartitioner partitioner = StorageService.getPartitioner();

    // see rant about this in Astyanax implementation
    if (partitioner instanceof RandomPartitioner || partitioner instanceof Murmur3Partitioner)
        throw new PermanentBackendException("This operation is only supported when byte-ordered partitioner is used.");

    return new RowIterator(keyRangeQuery, storeManager.getPageSize(), txh);
}
 

/**
 * if initialtoken was specified, use that (split on comma).
 * otherwise, if num_tokens == 1, pick a token to assume half the load of the most-loaded node.
 * else choose num_tokens tokens at random
 */
public static Collection<Token> getBootstrapTokens(final TokenMetadata metadata) throws ConfigurationException
{
    Collection<String> initialTokens = DatabaseDescriptor.getInitialTokens();
    // if user specified tokens, use those
    if (initialTokens.size() > 0)
    {
        logger.debug("tokens manually specified as {}",  initialTokens);
        List<Token> tokens = new ArrayList<Token>(initialTokens.size());
        for (String tokenString : initialTokens)
        {
            Token token = StorageService.getPartitioner().getTokenFactory().fromString(tokenString);
            if (metadata.getEndpoint(token) != null)
                throw new ConfigurationException("Bootstrapping to existing token " + tokenString + " is not allowed (decommission/removenode the old node first).");
            tokens.add(token);
        }
        return tokens;
    }

    int numTokens = DatabaseDescriptor.getNumTokens();
    if (numTokens < 1)
        throw new ConfigurationException("num_tokens must be >= 1");

    if (numTokens == 1)
        logger.warn("Picking random token for a single vnode.  You should probably add more vnodes; failing that, you should probably specify the token manually");

    return getRandomTokens(metadata, numTokens);
}
 

@BeforeClass
public static void setup() throws Exception
{
    SchemaLoader.mkdirs();
    SchemaLoader.cleanup();
    Keyspace.setInitialized();
    StorageService.instance.initServer(0);
}
 

private void transferRanges(ColumnFamilyStore cfs) throws Exception
{
    IPartitioner p = StorageService.getPartitioner();
    List<Range<Token>> ranges = new ArrayList<>();
    // wrapped range
    ranges.add(new Range<Token>(p.getToken(ByteBufferUtil.bytes("key1")), p.getToken(ByteBufferUtil.bytes("key0"))));
    new StreamPlan("StreamingTransferTest").transferRanges(LOCAL, cfs.keyspace.getName(), ranges, cfs.getColumnFamilyName()).execute().get();
}
 

public Map<Token, Float> describeOwnership(List<Token> sortedTokens)
{
    // allTokens will contain the count and be returned, sorted_ranges is shorthand for token<->token math.
    Map<Token, Float> allTokens = new HashMap<Token, Float>();
    List<Range<Token>> sortedRanges = new ArrayList<Range<Token>>(sortedTokens.size());

    // this initializes the counts to 0 and calcs the ranges in order.
    Token lastToken = sortedTokens.get(sortedTokens.size() - 1);
    for (Token node : sortedTokens)
    {
        allTokens.put(node, new Float(0.0));
        sortedRanges.add(new Range<Token>(lastToken, node));
        lastToken = node;
    }

    for (String ks : Schema.instance.getKeyspaces())
    {
        for (CFMetaData cfmd : Schema.instance.getKSMetaData(ks).cfMetaData().values())
        {
            for (Range<Token> r : sortedRanges)
            {
                // Looping over every KS:CF:Range, get the splits size and add it to the count
                allTokens.put(r.right, allTokens.get(r.right) + StorageService.instance.getSplits(ks, cfmd.cfName, r, 1).size());
            }
        }
    }

    // Sum every count up and divide count/total for the fractional ownership.
    Float total = new Float(0.0);
    for (Float f : allTokens.values())
        total += f;
    for (Map.Entry<Token, Float> row : allTokens.entrySet())
        allTokens.put(row.getKey(), row.getValue() / total);

    return allTokens;
}
 

private static Set<String> tokensAsSet(Collection<Token> tokens)
{
    Token.TokenFactory factory = StorageService.getPartitioner().getTokenFactory();
    Set<String> s = new HashSet<>(tokens.size());
    for (Token tk : tokens)
        s.add(factory.toString(tk));
    return s;
}