com.carrotsearch.randomizedtesting.generators.RandomStrings Java Examples

public void testPullOneTermsEnum() throws Exception {
  Directory dir = newDirectory();
  RandomIndexWriter w = new RandomIndexWriter(random(), dir);
  Document doc = new Document();
  doc.add(new StringField("foo", "1", Store.NO));
  w.addDocument(doc);
  DirectoryReader reader = w.getReader();
  w.close();
  final AtomicInteger counter = new AtomicInteger();
  DirectoryReader wrapped = new TermsCountingDirectoryReaderWrapper(reader, counter);

  final List<BytesRef> terms = new ArrayList<>();
  // enough terms to avoid the rewrite
  final int numTerms = TestUtil.nextInt(random(), TermInSetQuery.BOOLEAN_REWRITE_TERM_COUNT_THRESHOLD + 1, 100);
  for (int i = 0; i < numTerms; ++i) {
    final BytesRef term = new BytesRef(RandomStrings.randomUnicodeOfCodepointLength(random(), 10));
    terms.add(term);
  }

  assertEquals(0, new IndexSearcher(wrapped).count(new TermInSetQuery("bar", terms)));
  assertEquals(0, counter.get()); // missing field
  new IndexSearcher(wrapped).count(new TermInSetQuery("foo", terms));
  assertEquals(1, counter.get());
  wrapped.close();
  dir.close();
}
 

synchronized String routingKeyForShard(Index index, int shard, Random random) {
    assertThat(shard, greaterThanOrEqualTo(0));
    assertThat(shard, greaterThanOrEqualTo(0));
    for (NodeAndClient n : nodes.values()) {
        Node node = n.node;
        IndicesService indicesService = getInstanceFromNode(IndicesService.class, node);
        ClusterService clusterService = getInstanceFromNode(ClusterService.class, node);
        IndexService indexService = indicesService.indexService(index);
        if (indexService != null) {
            assertThat(indexService.getIndexSettings().getSettings().getAsInt(IndexMetaData.SETTING_NUMBER_OF_SHARDS, -1),
                    greaterThan(shard));
            OperationRouting operationRouting = clusterService.operationRouting();
            while (true) {
                String routing = RandomStrings.randomAsciiOfLength(random, 10);
                final int targetShard = operationRouting
                        .indexShards(clusterService.state(), index.getName(), null, routing)
                        .shardId().getId();
                if (shard == targetShard) {
                    return routing;
                }
            }
        }
    }
    fail("Could not find a node that holds " + index);
    return null;
}
 

public void testRamBytesUsed() {
  List<BytesRef> terms = new ArrayList<>();
  final int numTerms = 10000 + random().nextInt(1000);
  for (int i = 0; i < numTerms; ++i) {
    terms.add(new BytesRef(RandomStrings.randomUnicodeOfLength(random(), 10)));
  }
  TermInSetQuery query = new TermInSetQuery("f", terms);
  final long actualRamBytesUsed = RamUsageTester.sizeOf(query);
  final long expectedRamBytesUsed = query.ramBytesUsed();
  // error margin within 5%
  assertEquals(expectedRamBytesUsed, actualRamBytesUsed, actualRamBytesUsed / 20);
}
 

@Test
public void testIsAbsolute() throws Exception {
  try(HdfsDirectoryFactory hdfsFactory = new HdfsDirectoryFactory()) {
    String relativePath = Strings.repeat(
        RandomStrings.randomAsciiAlphanumOfLength(random(), random().nextInt(10) + 1) + '/',
        random().nextInt(5) + 1);
    assertFalse(hdfsFactory.isAbsolute(relativePath));
    assertFalse(hdfsFactory.isAbsolute("/" + relativePath));

    for(String rootPrefix : Arrays.asList("file://", "hdfs://", "s3a://", "foo://")) {
      assertTrue(hdfsFactory.isAbsolute(rootPrefix + relativePath));
    }
  }
}
 

public static String[] generateRandomStringArray(int maxArraySize, int stringSize, boolean allowNull, boolean allowEmpty) {
    if (allowNull && random().nextBoolean()) {
        return null;
    }
    int arraySize = randomIntBetween(allowEmpty ? 0 : 1, maxArraySize);
    String[] array = new String[arraySize];
    for (int i = 0; i < arraySize; i++) {
        array[i] = RandomStrings.randomAsciiOfLength(random(), stringSize);
    }
    return array;
}
 

/**
 * Randomly adds fields, objects, or arrays to the provided builder. The maximum depth is 5.
 */
private static void addFields(Random random, XContentBuilder builder, int minNumFields, int currentDepth) throws IOException {
    int numFields = randomIntBetween(random, minNumFields, 5);
    for (int i = 0; i < numFields; i++) {
        if (currentDepth < 5 && random.nextInt(100) >= 70) {
            if (random.nextBoolean()) {
                builder.startObject(RandomStrings.randomAsciiLettersOfLengthBetween(random, 6, 10));
                addFields(random, builder, minNumFields, currentDepth + 1);
                builder.endObject();
            } else {
                builder.startArray(RandomStrings.randomAsciiLettersOfLengthBetween(random, 6, 10));
                int numElements = randomIntBetween(random, 1, 5);
                boolean object = random.nextBoolean();
                int dataType = -1;
                if (object == false) {
                    dataType = randomDataType(random);
                }
                for (int j = 0; j < numElements; j++) {
                    if (object) {
                        builder.startObject();
                        addFields(random, builder, minNumFields, 5);
                        builder.endObject();
                    } else {
                        builder.value(randomFieldValue(random, dataType));
                    }
                }
                builder.endArray();
            }
        } else {
            builder.field(RandomStrings.randomAsciiLettersOfLengthBetween(random, 6, 10),
                    randomFieldValue(random, randomDataType(random)));
        }
    }
}
 

private static Object randomFieldValue(Random random, int dataType) {
    switch(dataType) {
        case 0:
            return RandomStrings.randomAsciiLettersOfLengthBetween(random, 3, 10);
        case 1:
            return RandomStrings.randomAsciiLettersOfLengthBetween(random, 3, 10);
        case 2:
            return random.nextLong();
        case 3:
            return random.nextDouble();
        default:
            throw new UnsupportedOperationException();
    }
}
 

@Test
public void testInsertStringGreaterThanDocValuesLimit() throws Exception {
    execute("create table t1 (s string index off storage with (columnstore=false))");

    String bigString = RandomStrings.randomRealisticUnicodeOfLength(random(), MAX_TERM_LENGTH + 2);
    execute("insert into t1 (s) values (?)", new Object[]{bigString});
    refresh();

    execute("select s from t1 limit 1");
    assertThat(response.rows()[0][0], is(bigString));
}
 

private static void testNGrams(int minGram, int maxGram, int length, final String nonTokenChars) throws IOException {
  final String s = RandomStrings.randomAsciiLettersOfLengthBetween(random(), length, length);
  testNGrams(minGram, maxGram, s, nonTokenChars);
}
 

private static void testNGrams(int minGram, int maxGram, int length, final String nonTokenChars) throws IOException {
  final String s = RandomStrings.randomAsciiLettersOfLengthBetween(random(), length, length);
  testNGrams(minGram, maxGram, s, nonTokenChars);
}
 

public void testAddAndUpdateRandom() throws IOException {
  AtomicInteger newCalled = new AtomicInteger(0);
  AtomicInteger addCalled = new AtomicInteger(0);
  TermsHashPerField hash = createNewHash(newCalled, addCalled);
  hash.start(null, true);
  class Posting {
    int termId = -1;
    final TreeMap<Integer, Integer> docAndFreq = new TreeMap<>();
  }
  Map<BytesRef, Posting> postingMap = new HashMap<>();
  int numStrings = 1 + random().nextInt(200);
  for (int i = 0; i < numStrings; i++) {
    String randomString = RandomStrings.randomRealisticUnicodeOfCodepointLengthBetween(random(), 1, 10);
    postingMap.putIfAbsent(new BytesRef(randomString), new Posting());
  }
  List<BytesRef> bytesRefs = Arrays.asList(postingMap.keySet().toArray(new BytesRef[0]));
  Collections.sort(bytesRefs);
  int numDocs = 1 + random().nextInt(200);
  int termOrd = 0;
  for (int i = 0; i < numDocs; i++) {
    int numTerms = 1 + random().nextInt(200);
    int doc = i;
    for (int j = 0; i < numTerms; i++) {
      BytesRef ref = RandomPicks.randomFrom(random(), bytesRefs);
      Posting posting = postingMap.get(ref);
      if (posting.termId == -1) {
        posting.termId = termOrd++;
      }
      posting.docAndFreq.putIfAbsent(doc, 0);
      posting.docAndFreq.compute(doc, (key, oldVal) -> oldVal+1);
      hash.add(ref, doc);
    }
    hash.finish();
  }
  List<Posting> values = postingMap.values().stream().filter( x -> x.termId != -1)
      .collect(Collectors.toList());
  Collections.shuffle(values, random()); // term order doesn't matter
  final ByteSliceReader reader = new ByteSliceReader();
  for (Posting p : values) {
    hash.initReader(reader, p.termId, 0);
    boolean eof = false;
    int prefDoc = 0;
    for (Map.Entry<Integer, Integer> entry : p.docAndFreq.entrySet()) {
      assertFalse("the reader must not be EOF here", eof);
      eof = assertDocAndFreq(reader, (FreqProxTermsWriterPerField.FreqProxPostingsArray) hash.postingsArray,
          prefDoc, p.termId, entry.getKey(), entry.getValue());
      prefDoc = entry.getKey();
    }
    assertTrue("the last posting must be EOF on the reader", eof);
  }
}
 

public void testMoveLargestLast() {
  SolrInputDocument inDoc = new SolrInputDocument();
  String TEXT_FLD = "text"; // not stored.  It won't be moved.  This value is the longest, however.
  inDoc.addField(TEXT_FLD,
      "NOT STORED|" + RandomStrings.randomAsciiOfLength(random(), 4 * DocumentBuilder.MIN_LENGTH_TO_MOVE_LAST));

  String CAT_FLD = "cat"; // stored, multiValued
  inDoc.addField(CAT_FLD,
      "STORED V1|");
  //  pretty long value
  inDoc.addField(CAT_FLD,
      "STORED V2|" + RandomStrings.randomAsciiOfLength(random(), 2 * DocumentBuilder.MIN_LENGTH_TO_MOVE_LAST));
  inDoc.addField(CAT_FLD,
      "STORED V3|" + RandomStrings.randomAsciiOfLength(random(), DocumentBuilder.MIN_LENGTH_TO_MOVE_LAST));

  String SUBJECT_FLD = "subject"; // stored.  This value is long, but not long enough.
  inDoc.addField(SUBJECT_FLD,
      "2ndplace|" + RandomStrings.randomAsciiOfLength(random(), DocumentBuilder.MIN_LENGTH_TO_MOVE_LAST));

  Document outDoc = DocumentBuilder.toDocument(inDoc, h.getCore().getLatestSchema());

  // filter outDoc by stored fields; convert to list.
  List<IndexableField> storedFields = StreamSupport.stream(outDoc.spliterator(), false)
      .filter(f -> f.fieldType().stored()).collect(Collectors.toList());
  // clip to last 3.  We expect these to be for CAT_FLD
  storedFields = storedFields.subList(storedFields.size() - 3, storedFields.size());

  Iterator<IndexableField> fieldIterator = storedFields.iterator();
  IndexableField field;

  // Test that we retained the particular value ordering, even though though the 2nd of three was longest

  assertTrue(fieldIterator.hasNext());
  field = fieldIterator.next();
  assertEquals(CAT_FLD, field.name());
  assertTrue(field.stringValue().startsWith("STORED V1|"));

  assertTrue(fieldIterator.hasNext());
  field = fieldIterator.next();
  assertEquals(CAT_FLD, field.name());
  assertTrue(field.stringValue().startsWith("STORED V2|"));

  assertTrue(fieldIterator.hasNext());
  field = fieldIterator.next();
  assertEquals(CAT_FLD, field.name());
  assertTrue(field.stringValue().startsWith("STORED V3|"));
}
 

private String randomStringOfLength(int min, int max) {
  return RandomStrings.randomAsciiLettersOfLengthBetween(random(), min, max).toLowerCase(Locale.ROOT);
}
 

private String randomStringOfLength(int min, int max) {
  return RandomStrings.randomAsciiLettersOfLengthBetween(random(), min, max).toLowerCase();
}
 

/**
 * Returns a tuple containing random stored field values and their corresponding expected values once printed out
 * via {@link org.elasticsearch.common.xcontent.ToXContent#toXContent(XContentBuilder, ToXContent.Params)} and parsed back via
 * {@link org.elasticsearch.common.xcontent.XContentParser#objectText()}.
 * Generates values based on what can get printed out. Stored fields values are retrieved from lucene and converted via
 * {@link org.elasticsearch.index.mapper.MappedFieldType#valueForDisplay(Object)} to either strings, numbers or booleans.
 *
 * @param random Random generator
 * @param xContentType the content type, used to determine what the expected values are for float numbers.
 */
public static Tuple<List<Object>, List<Object>> randomStoredFieldValues(Random random, XContentType xContentType) {
    int numValues = randomIntBetween(random, 1, 5);
    List<Object> originalValues = new ArrayList<>();
    List<Object> expectedParsedValues = new ArrayList<>();
    int dataType = randomIntBetween(random, 0, 8);
    for (int i = 0; i < numValues; i++) {
        switch(dataType) {
            case 0:
                long randomLong = random.nextLong();
                originalValues.add(randomLong);
                expectedParsedValues.add(randomLong);
                break;
            case 1:
                int randomInt = random.nextInt();
                originalValues.add(randomInt);
                expectedParsedValues.add(randomInt);
                break;
            case 2:
                Short randomShort = (short) random.nextInt();
                originalValues.add(randomShort);
                expectedParsedValues.add(randomShort.intValue());
                break;
            case 3:
                Byte randomByte = (byte)random.nextInt();
                originalValues.add(randomByte);
                expectedParsedValues.add(randomByte.intValue());
                break;
            case 4:
                double randomDouble = random.nextDouble();
                originalValues.add(randomDouble);
                expectedParsedValues.add(randomDouble);
                break;
            case 5:
                Float randomFloat = random.nextFloat();
                originalValues.add(randomFloat);
                if (xContentType == XContentType.SMILE) {
                    //with SMILE we get back a double (this will change in Jackson 2.9 where it will return a Float)
                    expectedParsedValues.add(randomFloat.doubleValue());
                } else {
                    //with JSON AND YAML we get back a double, but with float precision.
                    expectedParsedValues.add(Double.parseDouble(randomFloat.toString()));
                }
                break;
            case 6:
                boolean randomBoolean = random.nextBoolean();
                originalValues.add(randomBoolean);
                expectedParsedValues.add(randomBoolean);
                break;
            case 7:
                String randomString = random.nextBoolean() ? RandomStrings.randomAsciiLettersOfLengthBetween(random, 3, 10) :
                        randomUnicodeOfLengthBetween(random, 3, 10);
                originalValues.add(randomString);
                expectedParsedValues.add(randomString);
                break;
            case 8:
                byte[] randomBytes = RandomStrings.randomUnicodeOfLengthBetween(random, 10, 50).getBytes(StandardCharsets.UTF_8);
                BytesArray randomBytesArray = new BytesArray(randomBytes);
                originalValues.add(randomBytesArray);
                if (xContentType == XContentType.JSON || xContentType == XContentType.YAML) {
                    //JSON and YAML write the base64 format
                    expectedParsedValues.add(Base64.getEncoder().encodeToString(randomBytes));
                } else {
                    //SMILE and CBOR write the original bytes as they support binary format
                    expectedParsedValues.add(randomBytesArray);
                }
                break;
            default:
                throw new UnsupportedOperationException();
        }
    }
    return Tuple.tuple(originalValues, expectedParsedValues);
}