org.apache.cassandra.dht.Token Java Examples

public int forceRepairAsync(String keyspace, RepairParallelism parallelismDegree, Collection<String> dataCenters, Collection<String> hosts, Collection<Range<Token>> ranges, boolean fullRepair, String... columnFamilies)
{
    if (ranges.isEmpty() || Keyspace.open(keyspace).getReplicationStrategy().getReplicationFactor() < 2)
        return 0;

    int cmd = nextRepairCommand.incrementAndGet();
    if (ranges.size() > 0)
    {
        if (FBUtilities.isWindows() && parallelismDegree != RepairParallelism.PARALLEL)
        {
            logger.warn("Snapshot-based repair is not yet supported on Windows.  Reverting to parallel repair.");
            parallelismDegree = RepairParallelism.PARALLEL;
        }
        new Thread(createRepairTask(cmd, keyspace, ranges, parallelismDegree, dataCenters, hosts, fullRepair, columnFamilies)).start();
    }
    return cmd;
}
 

/**
 * get the (possibly cached) endpoints that should store the given Token.
 * Note that while the endpoints are conceptually a Set (no duplicates will be included),
 * we return a List to avoid an extra allocation when sorting by proximity later
 * @param searchPosition the position the natural endpoints are requested for
 * @return a copy of the natural endpoints for the given token
 */
public ArrayList<InetAddress> getNaturalEndpoints(RingPosition searchPosition)
{
    Token searchToken = searchPosition.getToken();
    Token keyToken = TokenMetadata.firstToken(tokenMetadata.sortedTokens(), searchToken);
    ArrayList<InetAddress> endpoints = getCachedEndpoints(keyToken);
    if (endpoints == null)
    {
        TokenMetadata tm = tokenMetadata.cachedOnlyTokenMap();
        // if our cache got invalidated, it's possible there is a new token to account for too
        keyToken = TokenMetadata.firstToken(tm.sortedTokens(), searchToken);
        endpoints = new ArrayList<InetAddress>(calculateNaturalEndpoints(searchToken, tm));
        cachedEndpoints.put(keyToken, endpoints);
    }

    return new ArrayList<InetAddress>(endpoints);
}
 

@Test
public void testGetNeighborsTimesTwoInSpecifiedHosts() throws Throwable
{
    TokenMetadata tmd = StorageService.instance.getTokenMetadata();

    // generate rf*2 nodes, and ensure that only neighbors specified by the hosts are returned
    addTokens(2 * Keyspace.open(keyspaceName).getReplicationStrategy().getReplicationFactor());
    AbstractReplicationStrategy ars = Keyspace.open(keyspaceName).getReplicationStrategy();
    List<InetAddress> expected = new ArrayList<>();
    for (Range<Token> replicaRange : ars.getAddressRanges().get(FBUtilities.getBroadcastAddress()))
    {
        expected.addAll(ars.getRangeAddresses(tmd.cloneOnlyTokenMap()).get(replicaRange));
    }

    expected.remove(FBUtilities.getBroadcastAddress());
    Collection<String> hosts = Arrays.asList(FBUtilities.getBroadcastAddress().getCanonicalHostName(),expected.get(0).getCanonicalHostName());

   assertEquals(expected.get(0), ActiveRepairService.getNeighbors(keyspaceName, StorageService.instance.getLocalRanges(keyspaceName).iterator().next(), null, hosts).iterator().next());
}
 

/**
 * Create a copy of TokenMetadata with tokenToEndpointMap reflecting situation after all
 * current leave, and move operations have finished.
 *
 * @return new token metadata
 */
public TokenMetadata cloneAfterAllSettled()
{
    lock.readLock().lock();

    try
    {
        TokenMetadata metadata = cloneOnlyTokenMap();

        for (InetAddress endpoint : leavingEndpoints)
            metadata.removeEndpoint(endpoint);


        for (Pair<Token, InetAddress> pair : movingEndpoints)
            metadata.updateNormalToken(pair.left, pair.right);

        return metadata;
    }
    finally
    {
        lock.readLock().unlock();
    }
}
 

public int forceRepairAsync(String keyspace, int parallelismDegree, Collection<String> dataCenters, Collection<String> hosts, boolean primaryRange, boolean fullRepair, String... columnFamilies)
{
    if (parallelismDegree < 0 || parallelismDegree > RepairParallelism.values().length - 1)
    {
        throw new IllegalArgumentException("Invalid parallelism degree specified: " + parallelismDegree);
    }
    Collection<Range<Token>> ranges;
    if (primaryRange)
    {
        // when repairing only primary range, neither dataCenters nor hosts can be set
        if (dataCenters == null && hosts == null)
            ranges = getPrimaryRanges(keyspace);
        // except dataCenters only contain local DC (i.e. -local)
        else if (dataCenters != null && dataCenters.size() == 1 && dataCenters.contains(DatabaseDescriptor.getLocalDataCenter()))
            ranges = getPrimaryRangesWithinDC(keyspace);
        else
            throw new IllegalArgumentException("You need to run primary range repair on all nodes in the cluster.");
    }
    else
    {
         ranges = getLocalRanges(keyspace);
    }

    return forceRepairAsync(keyspace, RepairParallelism.values()[parallelismDegree], dataCenters, hosts, ranges, fullRepair, columnFamilies);
}
 

@Test
public void antiCompactionSizeTest() throws ExecutionException, InterruptedException, IOException
{
    Keyspace keyspace = Keyspace.open(KEYSPACE1);
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
    cfs.disableAutoCompaction();
    SSTableReader s = writeFile(cfs, 1000);
    cfs.addSSTable(s);
    long origSize = s.bytesOnDisk();
    Range<Token> range = new Range<Token>(new BytesToken(ByteBufferUtil.bytes(0)), new BytesToken(ByteBufferUtil.bytes(500)));
    Collection<SSTableReader> sstables = cfs.getSSTables();
    CompactionManager.instance.performAnticompaction(cfs, Arrays.asList(range), Refs.tryRef(sstables), 12345);
    long sum = 0;
    for (SSTableReader x : cfs.getSSTables())
        sum += x.bytesOnDisk();
    assertEquals(sum, cfs.metric.liveDiskSpaceUsed.count());
    assertEquals(origSize, cfs.metric.liveDiskSpaceUsed.count(), 100000);

}
 

/**
 * Checks if a token is included in the current split.
 *
 * @param token {@link Token} to be checked.
 * @return true, if the token is included in the interval; false, otherwise.
 */
public static boolean isTokenIncludedInRange(DeepTokenRange deepTokenRange, Token<Comparable> token) {

    boolean isIncluded = false;

    if (((Comparable) deepTokenRange.getStartTokenAsComparable())
            .compareTo(deepTokenRange.getEndTokenAsComparable()) <= 0) {
        isIncluded = token.token.compareTo(deepTokenRange.getStartTokenAsComparable()) > 0;

        if (isIncluded) {
            isIncluded = token.token.compareTo(deepTokenRange.getEndTokenAsComparable()) <= 0;
        }
    } else {
        isIncluded = token.token.compareTo(deepTokenRange.getStartTokenAsComparable()) > 0;

        if (!isIncluded) {
            isIncluded = token.token.compareTo(deepTokenRange.getEndTokenAsComparable()) <= 0;
        }
    }

    return isIncluded;
}
 

/** @return an endpoint to token multimap representation of tokenToEndpointMap (a copy) */
public Multimap<InetAddress, Token> getEndpointToTokenMapForReading()
{
    lock.readLock().lock();
    try
    {
        Multimap<InetAddress, Token> cloned = HashMultimap.create();
        for (Map.Entry<Token, InetAddress> entry : tokenToEndpointMap.entrySet())
            cloned.put(entry.getValue(), entry.getKey());
        return cloned;
    }
    finally
    {
        lock.readLock().unlock();
    }
}
 

@Test
public void shouldSkipAntiCompactionForNonIntersectingRange() throws InterruptedException, ExecutionException, IOException
{
    ColumnFamilyStore store = prepareColumnFamilyStore();
    Collection<SSTableReader> sstables = store.getUnrepairedSSTables();
    assertEquals(store.getSSTables().size(), sstables.size());
    Range<Token> range = new Range<Token>(new BytesToken("-10".getBytes()), new BytesToken("-1".getBytes()));
    List<Range<Token>> ranges = Arrays.asList(range);

    Refs<SSTableReader> refs = Refs.tryRef(sstables);
    if (refs == null)
        throw new IllegalStateException();
    CompactionManager.instance.performAnticompaction(store, ranges, refs, 1);
    assertThat(store.getSSTables().size(), is(1));
    assertThat(Iterables.get(store.getSSTables(), 0).isRepaired(), is(false));
    assertThat(Iterables.get(store.getSSTables(), 0).selfRef().globalCount(), is(1));
    assertThat(store.getDataTracker().getCompacting().size(), is(0));
}
 

@Test
public void testGetNeighborsPlusOneInLocalDC() throws Throwable
{
    TokenMetadata tmd = StorageService.instance.getTokenMetadata();
    
    // generate rf+1 nodes, and ensure that all nodes are returned
    Set<InetAddress> expected = addTokens(1 + Keyspace.open(keyspaceName).getReplicationStrategy().getReplicationFactor());
    expected.remove(FBUtilities.getBroadcastAddress());
    // remove remote endpoints
    TokenMetadata.Topology topology = tmd.cloneOnlyTokenMap().getTopology();
    HashSet<InetAddress> localEndpoints = Sets.newHashSet(topology.getDatacenterEndpoints().get(DatabaseDescriptor.getLocalDataCenter()));
    expected = Sets.intersection(expected, localEndpoints);

    Collection<Range<Token>> ranges = StorageService.instance.getLocalRanges(keyspaceName);
    Set<InetAddress> neighbors = new HashSet<InetAddress>();
    for (Range<Token> range : ranges)
    {
        neighbors.addAll(ActiveRepairService.getNeighbors(keyspaceName, range, Arrays.asList(DatabaseDescriptor.getLocalDataCenter()), null));
    }
    assertEquals(expected, neighbors);
}
 

/** @return the number of nodes bootstrapping into source's primary range */
public int pendingRangeChanges(InetAddress source)
{
    int n = 0;
    Collection<Range<Token>> sourceRanges = getPrimaryRangesFor(getTokens(source));
    lock.readLock().lock();
    try
    {
        for (Token token : bootstrapTokens.keySet())
            for (Range<Token> range : sourceRanges)
                if (range.contains(token))
                    n++;
    }
    finally
    {
        lock.readLock().unlock();
    }
    return n;
}
 

public StreamRequest deserialize(DataInput in, int version) throws IOException
{
    String keyspace = in.readUTF();
    long repairedAt = in.readLong();
    int rangeCount = in.readInt();
    List<Range<Token>> ranges = new ArrayList<>(rangeCount);
    for (int i = 0; i < rangeCount; i++)
    {
        Token left = Token.serializer.deserialize(in);
        Token right = Token.serializer.deserialize(in);
        ranges.add(new Range<>(left, right));
    }
    int cfCount = in.readInt();
    List<String> columnFamilies = new ArrayList<>(cfCount);
    for (int i = 0; i < cfCount; i++)
        columnFamilies.add(in.readUTF());
    return new StreamRequest(keyspace, ranges, columnFamilies, repairedAt);
}
 

public Collection<Range<Token>> getPrimaryRangesFor(Collection<Token> tokens)
{
    Collection<Range<Token>> ranges = new ArrayList<Range<Token>>(tokens.size());
    for (Token right : tokens)
        ranges.add(new Range<Token>(getPredecessor(right), right));
    return ranges;
}
 

public static Collection<Token> getSavedTokens()
{
    String req = "SELECT tokens FROM system.%s WHERE key='%s'";
    UntypedResultSet result = executeInternal(String.format(req, LOCAL_CF, LOCAL_KEY));
    return result.isEmpty() || !result.one().has("tokens")
         ? Collections.<Token>emptyList()
         : deserializeTokens(result.one().<String>getSet("tokens", UTF8Type.instance));
}
 

protected void addRangeForEndpoint(Range<Token> range, InetAddress endpoint)
{
    Collection<Range<Token>> ranges = endpointToRanges.get(endpoint);
    if (ranges == null)
    {
        ranges = new HashSet<>();
        endpointToRanges.put(endpoint, ranges);
    }
    ranges.add(range);
}
 

public void serialize(MerkleTree mt, DataOutputPlus out, int version) throws IOException
{
    out.writeByte(mt.hashdepth);
    out.writeLong(mt.maxsize);
    out.writeLong(mt.size);
    out.writeUTF(mt.partitioner.getClass().getCanonicalName());
    // full range
    Token.serializer.serialize(mt.fullRange.left, out);
    Token.serializer.serialize(mt.fullRange.right, out);
    Hashable.serializer.serialize(mt.root, out, version);
}
 

@Test
public void testDifference() throws Throwable
{
    Range<Token> range = new Range<>(partirioner.getMinimumToken(), partirioner.getRandomToken());
    UUID parentRepairSession = UUID.randomUUID();
    Keyspace keyspace = Keyspace.open("Keyspace1");
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Standard1");

    ActiveRepairService.instance.registerParentRepairSession(parentRepairSession, Arrays.asList(cfs), Arrays.asList(range));

    RepairJobDesc desc = new RepairJobDesc(parentRepairSession, UUID.randomUUID(), "Keyspace1", "Standard1", range);

    MerkleTree tree1 = createInitialTree(desc);
    MerkleTree tree2 = createInitialTree(desc);

    // change a range in one of the trees
    Token token = partirioner.midpoint(range.left, range.right);
    tree1.invalidate(token);
    MerkleTree.TreeRange changed = tree1.get(token);
    changed.hash("non-empty hash!".getBytes());

    Set<Range<Token>> interesting = new HashSet<>();
    interesting.add(changed);

    // difference the trees
    // note: we reuse the same endpoint which is bogus in theory but fine here
    TreeResponse r1 = new TreeResponse(InetAddress.getByName("127.0.0.1"), tree1);
    TreeResponse r2 = new TreeResponse(InetAddress.getByName("127.0.0.2"), tree2);
    Differencer diff = new Differencer(desc, r1, r2);
    diff.run();

    // ensure that the changed range was recorded
    assertEquals("Wrong differing ranges", interesting, new HashSet<>(diff.differences));
}
 

public Range<Token> getRange(ByteBuffer key)
{
    // TODO: naive linear search of the token map
    Token t = partitioner.getToken(key);
    for (Range<Token> range : rangeMap.keySet())
        if (range.contains(t))
            return range;

    throw new RuntimeException("Invalid token information returned by describe_ring: " + rangeMap);
}
 

public long serializedSize(MerkleTree mt, int version)
{
    long size = 1 // mt.hashdepth
         + TypeSizes.NATIVE.sizeof(mt.maxsize)
         + TypeSizes.NATIVE.sizeof(mt.size)
         + TypeSizes.NATIVE.sizeof(mt.partitioner.getClass().getCanonicalName());

    // full range
    size += Token.serializer.serializedSize(mt.fullRange.left, TypeSizes.NATIVE);
    size += Token.serializer.serializedSize(mt.fullRange.right, TypeSizes.NATIVE);

    size += Hashable.serializer.serializedSize(mt.root, version);
    return size;
}
 

public int forceRepairRangeAsync(String beginToken, String endToken, String keyspaceName, int parallelismDegree, Collection<String> dataCenters, Collection<String> hosts, boolean fullRepair, String... columnFamilies)
{
    if (parallelismDegree < 0 || parallelismDegree > RepairParallelism.values().length - 1)
    {
        throw new IllegalArgumentException("Invalid parallelism degree specified: " + parallelismDegree);
    }
    Collection<Range<Token>> repairingRange = createRepairRangeFrom(beginToken, endToken);

    logger.info("starting user-requested repair of range {} for keyspace {} and column families {}",
                       repairingRange, keyspaceName, columnFamilies);

    RepairParallelism parallelism = RepairParallelism.values()[parallelismDegree];
    return forceRepairAsync(keyspaceName, parallelism, dataCenters, hosts, repairingRange, fullRepair, columnFamilies);
}
 

public void run()
{
    if (StorageService.instance.isStarting())
    {
        logger.debug("Node has not yet joined; not recording size estimates");
        return;
    }

    logger.debug("Recording size estimates");

    // find primary token ranges for the local node.
    Collection<Token> localTokens = StorageService.instance.getLocalTokens();
    Collection<Range<Token>> localRanges = StorageService.instance.getTokenMetadata().getPrimaryRangesFor(localTokens);

    for (Keyspace keyspace : Keyspace.nonSystem())
    {
        for (ColumnFamilyStore table : keyspace.getColumnFamilyStores())
        {
            long start = System.nanoTime();
            recordSizeEstimates(table, localRanges);
            long passed = System.nanoTime() - start;
            logger.debug("Spent {} milliseconds on estimating {}.{} size",
                         TimeUnit.NANOSECONDS.toMillis(passed),
                         table.metadata.ksName,
                         table.metadata.cfName);
        }
    }
}
 

private Collection<Token> getTokensFor(InetAddress endpoint)
{
    try
    {
        return TokenSerializer.deserialize(getPartitioner(), new DataInputStream(new ByteArrayInputStream(getApplicationStateValue(endpoint, ApplicationState.TOKENS))));
    }
    catch (IOException e)
    {
        throw new RuntimeException(e);
    }
}
 

/**
 * Returns the Lucene {@link BytesRef} represented by the specified Cassandra {@link Token}.
 *
 * @param token A Cassandra {@link Token}.
 * @return The Lucene {@link BytesRef} represented by the specified Cassandra {@link Token}.
 */
@SuppressWarnings("unchecked")
public BytesRef bytesRef(Token token) {
    ByteBuffer bb = factory.toByteArray(token);
    byte[] bytes = ByteBufferUtils.asArray(bb);
    return new BytesRef(bytes);
}
 

public Collection<Token> getTokens(InetAddress endpoint)
{
    assert endpoint != null;
    assert isMember(endpoint); // don't want to return nulls

    lock.readLock().lock();
    try
    {
        return new ArrayList<Token>(tokenToEndpointMap.inverse().get(endpoint));
    }
    finally
    {
        lock.readLock().unlock();
    }
}
 

public int forceRepairRangeAsync(String beginToken, String endToken, String keyspaceName, boolean isSequential, boolean isLocal, boolean fullRepair, String... columnFamilies)
{
    Collection<Range<Token>> repairingRange = createRepairRangeFrom(beginToken, endToken);

    logger.info("starting user-requested repair of range {} for keyspace {} and column families {}",
                       repairingRange, keyspaceName, columnFamilies);
    return forceRepairAsync(keyspaceName, isSequential, isLocal, repairingRange, fullRepair, columnFamilies);
}
 

public static List<SSTableReader> intersecting(Collection<SSTableReader> sstables, Range<Token> range)
{
    ArrayList<SSTableReader> filtered = new ArrayList<SSTableReader>();
    for (SSTableReader sstable : sstables)
    {
        Range<Token> sstableRange = new Range<Token>(sstable.first.getToken(), sstable.last.getToken(), sstable.partitioner);
        if (range == null || sstableRange.intersects(range))
            filtered.add(sstable);
    }
    return filtered;
}
 

public void removeBootstrapTokens(Collection<Token> tokens)
{
    assert tokens != null && !tokens.isEmpty();

    lock.writeLock().lock();
    try
    {
        for (Token token : tokens)
            bootstrapTokens.remove(token);
    }
    finally
    {
        lock.writeLock().unlock();
    }
}
 

private List<String> getTokens(InetAddress endpoint)
{
    List<String> strTokens = new ArrayList<>();
    for (Token tok : getTokenMetadata().getTokens(endpoint))
        strTokens.add(tok.toString());
    return strTokens;
}
 

private void excise(Collection<Token> tokens, InetAddress endpoint)
{
    logger.info("Removing tokens {} for {}", tokens, endpoint);
    HintedHandOffManager.instance.deleteHintsForEndpoint(endpoint);
    removeEndpoint(endpoint);
    tokenMetadata.removeEndpoint(endpoint);
    tokenMetadata.removeBootstrapTokens(tokens);

    if (!isClientMode)
    {
        for (IEndpointLifecycleSubscriber subscriber : lifecycleSubscribers)
            subscriber.onLeaveCluster(endpoint);
    }
    PendingRangeCalculatorService.instance.update();
}
 

/**
 * Constructs an Inner with the given token and children, and a null hash.
 */
public Inner(Token token, Hashable lchild, Hashable rchild)
{
    super(null);
    this.token = token;
    this.lchild = lchild;
    this.rchild = rchild;
}