org.apache.ignite.cache.query.ScanQuery Java Examples

/**
 * @throws Exception If failed.
 */
@Test
public void testScanLocalQuery() throws Exception {
    IgniteCache<Integer, String> cache = grid().getOrCreateCache(String.class.getSimpleName());

    try {
        cache.query(new ScanQuery<>(new IgniteBiPredicate<Integer, String>() {
            @Override public boolean apply(Integer id, String s) {
                return s.equals("");
            }
        }).setLocal(true)).getAll();
    }
    catch (IgniteException ignored) {
        assertTrue("Scan query should work with disable query indexing.", false);
    }
}
 

/** {@inheritDoc} */
@Override public void run() {
    IgniteCache cache = node.cache(cacheName);

    // Getting a list of the partitions owned by this node.
    List<Integer> myPartitions = cachePart.get(node.cluster().localNode().id());

    for (Integer part : myPartitions) {

        ScanQuery scanQry = new ScanQuery();

        scanQry.setPartition(part);

        scanQry.setFilter(igniteBiPred);

        try (QueryCursor cursor = cache.query(scanQry)) {
            for (Object obj : cursor) {
                // No-op.
            }
        }

    }
}
 

/**
 * Reads value from cache for the given key using given read mode.
 *
 * // TODO IGNITE-6938 remove inTx flag
 * // TODO IGNITE-6739 add SQL-get support "select _key, _val from cache where _key = key"
 * @param inTx Flag whether current read is inside transaction.
 * This is because reads can't see writes made in current transaction.
 * @param cache Cache.
 * @param key Key.
 * @param readMode Read mode.
 * @return Value.
 */
private Object getByReadMode(boolean inTx, IgniteCache cache, final Object key, ReadMode readMode) {

    // TODO Remove in IGNITE-6938
    if (inTx)
        readMode = GET;

    switch (readMode) {
        case GET:
            return cache.get(key);

        case SCAN:
            List res = cache.query(new ScanQuery(new IgniteBiPredicate() {
                @Override public boolean apply(Object k, Object v) {
                    return k.equals(key);
                }
            })).getAll();

            assertTrue(res.size() <= 1);

            return res.isEmpty() ? null : ((IgniteBiTuple)res.get(0)).getValue();

        default:
            throw new IgniteException("Unsupported read mode: " + readMode);
    }
}
 

/**
 * Tests that Scan queries are executed in dedicated pool
 * @throws Exception If failed.
 */
@Test
public void testScanQueryUsesDedicatedThreadPool() throws Exception {
    startGrid("server");

    try (Ignite client = startClientGrid("client")) {
        IgniteCache<Integer, Integer> cache = client.cache(CACHE_NAME);

        cache.put(0, 0);

        QueryCursor<Cache.Entry<Object, Object>> cursor = cache.query(
            new ScanQuery<>(new IgniteBiPredicate<Object, Object>() {
                @Override public boolean apply(Object o, Object o2) {
                    return F.eq(GridIoManager.currentPolicy(), GridIoPolicy.QUERY_POOL);
                }
            }));

        assertEquals(1, cursor.getAll().size());

        cursor.close();
    }
}
 

/**
 * @param cache Cache.
 */
private void cacheQuery(IgniteCache cache) {
    int keys = 100;

    Map<Integer, Integer> data = generateDataMap(keys);

    cache.putAll(data);

    ScanQuery<Integer, Integer> qry = new ScanQuery<>(new IgniteBiPredicate<Integer, Integer>() {
        @Override public boolean apply(Integer key, Integer val) {
            return key % 2 == 0;
        }
    });

    List<Cache.Entry<Integer, Integer>> all = cache.query(qry).getAll();

    assertEquals(all.size(), data.size() / 2);

    for (Cache.Entry<Integer, Integer> entry : all) {
        assertEquals(0, entry.getKey() % 2);
        assertEquals(entry.getValue(), data.get(entry.getKey()));
    }

    tearDown(cache);
}
 

/**
 * @throws Exception If failed.
 */
@Test
public void testScanQuery() throws Exception {
    IgniteCache<Integer, String> cache = grid().getOrCreateCache(String.class.getSimpleName());

    try {
        cache.query(new ScanQuery<>(new IgniteBiPredicate<Integer, String>() {
            @Override public boolean apply(Integer id, String s) {
                return s.equals("");
            }
        })).getAll();
    }
    catch (IgniteException ignored) {
        assertTrue("Scan query should work with disabled query indexing.", false);
    }
}
 

/** */
@Test
public void testScanQuery() throws Exception {
    Ignite ignite = G.allGrids().stream().findFirst().orElseThrow(IllegalStateException::new);

    try (IgniteDataStreamer<String, Integer> strAllowedCache = ignite.dataStreamer(CACHE_NAME);
         IgniteDataStreamer<String, Integer> strForbiddenCache = ignite.dataStreamer(FORBIDDEN_CACHE)) {
        for (int i = 1; i <= 10; i++) {
            strAllowedCache.addData(Integer.toString(i), i);
            strForbiddenCache.addData(Integer.toString(i), i);
        }
    }

    Ignite node = startGrid(loginPrefix(clientMode) + "_test_node",
        SecurityPermissionSetBuilder.create()
            .appendCachePermissions(CACHE_NAME, CACHE_READ)
            .appendCachePermissions(FORBIDDEN_CACHE, EMPTY_PERMS).build(), clientMode);

    assertFalse(node.cache(CACHE_NAME).query(new ScanQuery<String, Integer>()).getAll().isEmpty());

    assertThrowsWithCause(() -> node.cache(FORBIDDEN_CACHE).query(new ScanQuery<String, Integer>()).getAll(),
        SecurityException.class);
}
 

/**
 * @throws Exception In case of error.
 */
@Test
public void testDeployScanPredicate() throws Exception {
    startGrids(4);

    awaitPartitionMapExchange();

    try {
        IgniteCache<Object, Object> cache = grid(3).cache(DEFAULT_CACHE_NAME);

        // It is important that there are no too many keys.
        for (int i = 0; i < 1; i++)
            cache.put(i, i);

        Class predCls = grid(3).configuration().getClassLoader().loadClass(TEST_PREDICATE);

        IgniteBiPredicate<Object, Object> pred = (IgniteBiPredicate<Object, Object>)predCls.newInstance();

        List<Cache.Entry<Object, Object>> all = cache.query(new ScanQuery<>(pred)).getAll();

        assertEquals(1, all.size());
    }
    finally {
        stopAllGrids();
    }
}
 

/**
 * Scan (with explicit {@code setLocal(true)}, no partition specified) should perform on the local node.
 *
 * @throws Exception If failed.
 */
@Test
public void testScanLocalExplicitNoPart() throws Exception {
    cacheMode = CacheMode.PARTITIONED;
    backups = 0;
    commSpiFactory = new TestLocalCommunicationSpiFactory();

    try {
        Ignite ignite = startGrids(GRID_CNT);

        IgniteCacheProxy<Integer, Integer> cache = fillCache(ignite);

        QueryCursor<Cache.Entry<Integer, Integer>> qry =
            cache.query(new ScanQuery<Integer, Integer>().setLocal(true));

        assertFalse(qry.getAll().isEmpty());
    }
    finally {
        stopAllGrids();
    }
}
 

/**
 * Example for scan query based on a predicate using binary objects.
 */
private static void scanQuery() {
    IgniteCache<BinaryObject, BinaryObject> cache = Ignition.ignite()
        .cache(PERSON_CACHE).withKeepBinary();

    ScanQuery<BinaryObject, BinaryObject> scan = new ScanQuery<>(
        new IgniteBiPredicate<BinaryObject, BinaryObject>() {
            @Override public boolean apply(BinaryObject key, BinaryObject person) {
                return person.<Double>field("salary") <= 1000;
            }
        }
    );

    // Execute queries for salary ranges.
    print("People with salaries between 0 and 1000 (queried with SCAN query): ", cache.query(scan).getAll());
}
 

/**
 * @param cache Ignite cache.
 * @throws Exception If failed.
 */
private void scanByCopositeFirstSecondPredicate(IgniteCache cache) throws Exception {
    IgniteBiPredicate firstFilter = (IgniteBiPredicate)TEST_CLASS_LOADER.loadClass(FIRST_FILTER_NAME)
        .getConstructor().newInstance();
    IgniteBiPredicate secondFilter = (IgniteBiPredicate)TEST_CLASS_LOADER.loadClass(SECOND_FILTER_NAME)
        .getConstructor().newInstance();
    IgniteBiPredicate compositeFilter = (IgniteBiPredicate)TEST_CLASS_LOADER.loadClass(COMPOSITE_FILTER_NAME)
        .getConstructor().newInstance();

    U.invoke(TEST_CLASS_LOADER.loadClass(COMPOSITE_FILTER_NAME), compositeFilter, "addPredicate", firstFilter);
    U.invoke(TEST_CLASS_LOADER.loadClass(COMPOSITE_FILTER_NAME), compositeFilter, "addPredicate", secondFilter);

    List list = cache.query(new ScanQuery().setPageSize(10).setFilter(compositeFilter)).getAll();

    assertEquals(list.size(), 0);
}
 

/**
 * @param total Expected total entries.
 */
private void checkScan(int total) {
    for (int i = 0; i < gridCount(); i++) {
        Set<DbKey> allKeys = new HashSet<>();

        Ignite ignite0 = grid(i);

        IgniteCache<DbKey, DbValue> cache0 = ignite0.cache("non-primitive");

        ScanQuery<DbKey, DbValue> qry = new ScanQuery<>();

        QueryCursor<Cache.Entry<DbKey, DbValue>> cur = cache0.query(qry);

        for (Cache.Entry<DbKey, DbValue> e : cur) {
            allKeys.add(e.getKey());
            assertEquals(e.getKey().val, e.getValue().iVal);
        }

        assertEquals(total, allKeys.size());
    }
}
 

/**
 * Scan should perform on the local node.
 *
 * @throws Exception If failed.
 */
@Test
public void testScanLocal() throws Exception {
    cacheMode = CacheMode.PARTITIONED;
    backups = 0;
    commSpiFactory = new TestLocalCommunicationSpiFactory();

    try {
        Ignite ignite = startGrids(GRID_CNT);

        IgniteCacheProxy<Integer, Integer> cache = fillCache(ignite);

        int part = anyLocalPartition(cache.context());

        QueryCursor<Cache.Entry<Integer, Integer>> qry =
            cache.query(new ScanQuery<Integer, Integer>().setPartition(part));

        doTestScanQuery(qry, part);
    }
    finally {
        stopAllGrids();
    }
}
 

/** */
@Test
public void testInitialQuery() {
    checkContinuousQuery(() -> {
        ContinuousQueryWithTransformer<Integer, Integer, Integer> q = new ContinuousQueryWithTransformer<>();

        q.setInitialQuery(new ScanQuery<>(INIT_QRY_FILTER));
        q.setRemoteTransformerFactory(() -> Cache.Entry::getValue);
        q.setLocalListener(e -> {/* No-op. */});

        return q;
    }, true);
}
 

/** {@inheritDoc} */
@Override public Integer call() throws Exception {
    Thread loadThread = new Thread() {
        @Override public void run() {
            log.info("result = " + simulateLoad());
        }
    };

    loadThread.setName("load-thread");
    loadThread.start();

    int cnt = 0;

    final Iterator<Cache.Entry<Integer, byte[]>> it = ignite.cache(DEFAULT_CACHE_NAME).
        query(new ScanQuery<Integer, byte[]>().setPageSize(100000)).iterator();

    while (it.hasNext()) {
        Cache.Entry<Integer, byte[]> entry = it.next();

        // Trigger STW.
        final long[] tids = ManagementFactory.getThreadMXBean().findDeadlockedThreads();

        cnt++;
    }

    loadThread.join();

    return cnt;
}
 

/**
 * Tests initial query of {@link ContinuousQueryWithTransformer}.
 */
@Test
public void testInitialQuery() {
    Consumer<ContinuousQueryWithTransformer<Integer, Integer, Integer>> consumer =
        new Consumer<ContinuousQueryWithTransformer<Integer, Integer, Integer>>() {
            @Override public void accept(ContinuousQueryWithTransformer<Integer, Integer, Integer> q) {
                q.setInitialQuery(new ScanQuery<>(INITIAL_QUERY_FILTER));
                q.setRemoteTransformerFactory(() -> Cache.Entry::getValue);
                q.setLocalListener(e -> {/* No-op. */});
            }
        };

    runAndCheck(operation(consumer, true));
}
 

/**
 * Stream of runnables to call query methods.
 */
private Stream<IgniteRunnable> operations() {
    return Stream.of(
        () -> localIgnite().cache(CACHE_NAME).query(new ScanQuery<>(operationCheck(SRV_CHECK))).getAll(),
        () -> localIgnite().cache(CACHE_NAME).query(new ScanQuery<>((k, v) -> true), operationCheck(SRV_CHECK)).getAll()
    );
}
 

/** */
@Test
public void testLocalCacheQueryMapping() throws Exception {
    IgniteEx ign0 = startGrid(0);

    IgniteCache<Object, Object> cache = ign0.createCache(new CacheConfiguration<>(GridAbstractTest.DEFAULT_CACHE_NAME)
        .setCacheMode(CacheMode.LOCAL));

    cache.put(1, 2);

    startGrid(1);

    ScanQuery<Object, Object> qry = new ScanQuery<>();

    {
        List<Cache.Entry<Object, Object>> res0 = grid(0).cache(GridAbstractTest.DEFAULT_CACHE_NAME).query(qry).getAll();

        assertEquals(1, res0.size());
        assertEquals(1, res0.get(0).getKey());
        assertEquals(2, res0.get(0).getValue());
    }

    {
        List<Cache.Entry<Object, Object>> res1 = grid(1).cache(GridAbstractTest.DEFAULT_CACHE_NAME).query(qry).getAll();

        assertTrue(res1.isEmpty());
    }
}
 

/**
 * @param partCntrs Expected per-partition entries count.
 */
private void checkScanPartition(Ignite ignite,
    IgniteCache<DbKey, DbValue> cache,
    Map<Integer, Integer> partCntrs,
    boolean loc) {
    Affinity<Object> aff = ignite.affinity(cache.getName());

    int parts = aff.partitions();

    for (int p = 0; p < parts; p++) {
        ScanQuery<DbKey, DbValue> qry = new ScanQuery<>();

        qry.setPartition(p);
        qry.setLocal(loc);

        if (loc && !ignite.cluster().localNode().equals(aff.mapPartitionToNode(p)))
            continue;

        QueryCursor<Cache.Entry<DbKey, DbValue>> cur = cache.query(qry);

        Set<DbKey> allKeys = new HashSet<>();

        for (Cache.Entry<DbKey, DbValue> e : cur) {
            allKeys.add(e.getKey());
            assertEquals(e.getKey().val, e.getValue().iVal);
        }

        Integer exp = partCntrs.get(p);

        if (exp == null)
            exp = 0;

        assertEquals(exp, (Integer)allKeys.size());
    }
}
 

/** */
@Test
public void testScanQueryAllowed() {
    performAction(cache -> {
        try (QueryCursor qry = cache.query(new ScanQuery<>())) {
            for (Object o : qry.getAll()) {
                IgniteBiTuple<Integer, Integer> tuple = (IgniteBiTuple<Integer, Integer>)o;

                assertEquals(o.toString(), kvMap.get(tuple.getKey()), tuple.getValue());
            }
        }
    });
}
 

/** */
@Test
public void testInitialQueryFilter() {
    checkContinuousQuery(() -> {
        ContinuousQuery<Integer, Integer> cq = new ContinuousQuery<>();

        cq.setInitialQuery(new ScanQuery<>(INIT_QRY_FILTER));
        cq.setLocalListener(e -> {/* No-op. */});

        return cq;
    }, true);
}
 

/**
 * Checks query.
 *
 * @param qry Query
 * @throws CacheException If query indexing disabled for sql query.
 */
private void validate(Query qry) {
    GridCacheContext<K, V> ctx = getContextSafe();

    if (!QueryUtils.isEnabled(ctx.config()) && !(qry instanceof ScanQuery) &&
        !(qry instanceof ContinuousQuery) && !(qry instanceof ContinuousQueryWithTransformer) &&
        !(qry instanceof SpiQuery) && !(qry instanceof SqlQuery) && !(qry instanceof SqlFieldsQuery))
        throw new CacheException("Indexing is disabled for cache: " + cacheName +
                ". Use setIndexedTypes or setTypeMetadata methods on CacheConfiguration to enable.");

    if (!ctx.kernalContext().query().moduleEnabled() &&
        (qry instanceof SqlQuery || qry instanceof SqlFieldsQuery || qry instanceof TextQuery))
        throw new CacheException("Failed to execute query. Add module 'ignite-indexing' to the classpath " +
                "of all Ignite nodes.");
}
 

/**
 * @throws Exception If failed.
 */
@Test
public void testGetObjectField() throws Exception {
    IgniteCache<Integer, Value> cache = grid().createCache("test-cache");

    try {
        for (int i = 0; i < 50; i++)
            cache.put(i, new Value("str" + i, i * 100));

        IgniteClosure<Cache.Entry<Integer, Value>, Integer> transformer =
            new IgniteClosure<Cache.Entry<Integer, Value>, Integer>() {
                @Override public Integer apply(Cache.Entry<Integer, Value> e) {
                    return e.getValue().idx;
                }
            };

        List<Integer> res = cache.query(new ScanQuery<Integer, Value>(), transformer).getAll();

        assertEquals(50, res.size());

        Collections.sort(res);

        for (int i = 0; i < 50; i++)
            assertEquals(i * 100, res.get(i).intValue());
    }
    finally {
        cache.destroy();
    }
}
 

/**
 * @throws Exception If failed.
 */
@Test
public void testGetKeys() throws Exception {
    IgniteCache<Integer, String> cache = grid().createCache("test-cache");

    try {
        for (int i = 0; i < 50; i++)
            cache.put(i, "val" + i);

        IgniteClosure<Cache.Entry<Integer, String>, Integer> transformer =
            new IgniteClosure<Cache.Entry<Integer, String>, Integer>() {
                @Override public Integer apply(Cache.Entry<Integer, String> e) {
                    return e.getKey();
                }
            };

        List<Integer> keys = cache.query(new ScanQuery<Integer, String>(), transformer).getAll();

        assertEquals(50, keys.size());

        Collections.sort(keys);

        for (int i = 0; i < 50; i++)
            assertEquals(i, keys.get(i).intValue());
    }
    finally {
        cache.destroy();
    }
}
 

/**
 * @throws Exception If failed.
 */
@Test
public void testScanFilters() throws Exception {
    IgniteCache<Integer, Integer> cache = jcache(Integer.class, Integer.class);

    for (int i = 0; i < 50; i++)
        cache.put(i, i);

    QueryCursor<Cache.Entry<Integer, Integer>> q = cache.query(new ScanQuery<>(new IgniteBiPredicate<Integer,Integer>() {
        @Override public boolean apply(Integer k, Integer v) {
            assertNotNull(k);
            assertNotNull(v);

            return k >= 20 && v < 40;
        }
    }));

    List<Cache.Entry<Integer, Integer>> list = new ArrayList<>(q.getAll());

    Collections.sort(list, new Comparator<Cache.Entry<Integer, Integer>>() {
        @Override public int compare(Cache.Entry<Integer, Integer> e1, Cache.Entry<Integer, Integer> e2) {
            return e1.getKey().compareTo(e2.getKey());
        }
    });

    assertEquals(20, list.size());

    for (int i = 20; i < 40; i++) {
        Cache.Entry<Integer, Integer> e = list.get(i - 20);

        assertEquals(i, (int)e.getKey());
        assertEquals(i, (int)e.getValue());
    }
}
 

/**
 * @param ignite Ignite.
 * @param orgId Organization id.
 * @return Count of found Person object with specified orgId
 */
private static int getPersonsCountByScanLocalQuery(final IgniteEx ignite, final int orgId) {
    List res = ignite.cache(Person.class.getSimpleName())
        .query(new ScanQuery<>(new IgniteBiPredicate<Person.Key, Person>() {
            @Override public boolean apply(Person.Key key, Person person) {
                return person.getOrgId() == orgId;
            }
        }).setLocal(true)).getAll();

    return res.size();
}
 

/** */
@Test
public void testContinuousQuery() throws Exception {
    IgniteCache<Integer, Integer> cache = ignite0.createCache("cache-1");

    assertTrue(execute(ignite0, "SELECT * FROM SYS.CONTINUOUS_QUERIES").isEmpty());
    assertTrue(execute(ignite1, "SELECT * FROM SYS.CONTINUOUS_QUERIES").isEmpty());

    try (QueryCursor qry = cache.query(new ContinuousQuery<>()
        .setInitialQuery(new ScanQuery<>())
        .setPageSize(100)
        .setTimeInterval(1000)
        .setLocalListener(evts -> {
            // No-op.
        })
        .setRemoteFilterFactory(() -> evt -> true)
    )) {
        for (int i = 0; i < 100; i++)
            cache.put(i, i);

        checkContinuouQueryView(ignite0, true);
        checkContinuouQueryView(ignite1, false);
    }

    assertTrue(execute(ignite0, "SELECT * FROM SYS.CONTINUOUS_QUERIES").isEmpty());
    assertTrue(waitForCondition(() ->
        execute(ignite1, "SELECT * FROM SYS.CONTINUOUS_QUERIES").isEmpty(), getTestTimeout()));
}
 

/**
 * @throws Exception If failed.
 */
@Test
public void testLocalKeepBinary() throws Exception {
    IgniteCache<Integer, Value> cache = grid().createCache("test-cache");

    try {
        for (int i = 0; i < 50; i++)
            cache.put(i, new Value("str" + i, i * 100));

        Collection<List<Integer>> lists = grid().compute().broadcast(new IgniteCallable<List<Integer>>() {
            @IgniteInstanceResource
            private Ignite ignite;

            @Override public List<Integer> call() throws Exception {
                IgniteClosure<Cache.Entry<Integer, BinaryObject>, Integer> transformer =
                    new IgniteClosure<Cache.Entry<Integer, BinaryObject>, Integer>() {
                        @Override public Integer apply(Cache.Entry<Integer, BinaryObject> e) {
                            return e.getValue().field("idx");
                        }
                    };

                return ignite.cache("test-cache").withKeepBinary().query(
                    new ScanQuery<Integer, BinaryObject>().setLocal(true), transformer).getAll();
            }
        });

        List<Integer> res = new ArrayList<>(F.flatCollections(lists));

        assertEquals(50, res.size());

        Collections.sort(res);

        for (int i = 0; i < 50; i++)
            assertEquals(i * 100, res.get(i).intValue());
    }
    finally {
        cache.destroy();
    }
}
 

/**
 * Scan (with explicit {@code setLocal(true)}) should perform on the local node.
 *
 * @throws Exception If failed.
 */
@Test
public void testScanLocalExplicit() throws Exception {
    cacheMode = CacheMode.PARTITIONED;
    backups = 0;
    commSpiFactory = new TestLocalCommunicationSpiFactory();

    try {
        Ignite ignite = startGrids(GRID_CNT);

        IgniteCacheProxy<Integer, Integer> cache = fillCache(ignite);

        int part = anyLocalPartition(cache.context());

        QueryCursor<Cache.Entry<Integer, Integer>> qry =
            cache.query(new ScanQuery<Integer, Integer>().setPartition(part).setLocal(true));

        doTestScanQuery(qry, part);

        GridTestUtils.assertThrows(log, (Callable<Void>)() -> {
            int remPart = remotePartition(cache.context()).getKey();

            cache.query(new ScanQuery<Integer, Integer>().setPartition(remPart).setLocal(true));

            return null;
        }, IgniteCheckedException.class, null);
    }
    finally {
        stopAllGrids();
    }
}
 

/**
 * Test metrics for failed Scan queries.
 *
 * @throws Exception In case of error.
 */
@Test
public void testScanQueryFailedMetrics() throws Exception {
    IgniteCache<Integer, String> cache = grid(0).context().cache().jcache("A");

    ScanQuery<Integer, String> qry = new ScanQuery<>(Integer.MAX_VALUE);

    checkQueryFailedMetrics(cache, qry);
}