com.carrotsearch.randomizedtesting.generators.RandomStrings Java Examples

The following examples show how to use com.carrotsearch.randomizedtesting.generators.RandomStrings. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TermInSetQueryTest.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testPullOneTermsEnum() throws Exception {
  Directory dir = newDirectory();
  RandomIndexWriter w = new RandomIndexWriter(random(), dir);
  Document doc = new Document();
  doc.add(new StringField("foo", "1", Store.NO));
  w.addDocument(doc);
  DirectoryReader reader = w.getReader();
  w.close();
  final AtomicInteger counter = new AtomicInteger();
  DirectoryReader wrapped = new TermsCountingDirectoryReaderWrapper(reader, counter);

  final List<BytesRef> terms = new ArrayList<>();
  // enough terms to avoid the rewrite
  final int numTerms = TestUtil.nextInt(random(), TermInSetQuery.BOOLEAN_REWRITE_TERM_COUNT_THRESHOLD + 1, 100);
  for (int i = 0; i < numTerms; ++i) {
    final BytesRef term = new BytesRef(RandomStrings.randomUnicodeOfCodepointLength(random(), 10));
    terms.add(term);
  }

  assertEquals(0, new IndexSearcher(wrapped).count(new TermInSetQuery("bar", terms)));
  assertEquals(0, counter.get()); // missing field
  new IndexSearcher(wrapped).count(new TermInSetQuery("foo", terms));
  assertEquals(1, counter.get());
  wrapped.close();
  dir.close();
}
 
Example #2
Source File: InternalTestCluster.java    From crate with Apache License 2.0 6 votes vote down vote up
synchronized String routingKeyForShard(Index index, int shard, Random random) {
    assertThat(shard, greaterThanOrEqualTo(0));
    assertThat(shard, greaterThanOrEqualTo(0));
    for (NodeAndClient n : nodes.values()) {
        Node node = n.node;
        IndicesService indicesService = getInstanceFromNode(IndicesService.class, node);
        ClusterService clusterService = getInstanceFromNode(ClusterService.class, node);
        IndexService indexService = indicesService.indexService(index);
        if (indexService != null) {
            assertThat(indexService.getIndexSettings().getSettings().getAsInt(IndexMetaData.SETTING_NUMBER_OF_SHARDS, -1),
                    greaterThan(shard));
            OperationRouting operationRouting = clusterService.operationRouting();
            while (true) {
                String routing = RandomStrings.randomAsciiOfLength(random, 10);
                final int targetShard = operationRouting
                        .indexShards(clusterService.state(), index.getName(), null, routing)
                        .shardId().getId();
                if (shard == targetShard) {
                    return routing;
                }
            }
        }
    }
    fail("Could not find a node that holds " + index);
    return null;
}
 
Example #3
Source File: TermInSetQueryTest.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public void testRamBytesUsed() {
  List<BytesRef> terms = new ArrayList<>();
  final int numTerms = 10000 + random().nextInt(1000);
  for (int i = 0; i < numTerms; ++i) {
    terms.add(new BytesRef(RandomStrings.randomUnicodeOfLength(random(), 10)));
  }
  TermInSetQuery query = new TermInSetQuery("f", terms);
  final long actualRamBytesUsed = RamUsageTester.sizeOf(query);
  final long expectedRamBytesUsed = query.ramBytesUsed();
  // error margin within 5%
  assertEquals(expectedRamBytesUsed, actualRamBytesUsed, actualRamBytesUsed / 20);
}
 
Example #4
Source File: HdfsDirectoryFactoryTest.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Test
public void testIsAbsolute() throws Exception {
  try(HdfsDirectoryFactory hdfsFactory = new HdfsDirectoryFactory()) {
    String relativePath = Strings.repeat(
        RandomStrings.randomAsciiAlphanumOfLength(random(), random().nextInt(10) + 1) + '/',
        random().nextInt(5) + 1);
    assertFalse(hdfsFactory.isAbsolute(relativePath));
    assertFalse(hdfsFactory.isAbsolute("/" + relativePath));

    for(String rootPrefix : Arrays.asList("file://", "hdfs://", "s3a://", "foo://")) {
      assertTrue(hdfsFactory.isAbsolute(rootPrefix + relativePath));
    }
  }
}
 
Example #5
Source File: ESTestCase.java    From crate with Apache License 2.0 5 votes vote down vote up
public static String[] generateRandomStringArray(int maxArraySize, int stringSize, boolean allowNull, boolean allowEmpty) {
    if (allowNull && random().nextBoolean()) {
        return null;
    }
    int arraySize = randomIntBetween(allowEmpty ? 0 : 1, maxArraySize);
    String[] array = new String[arraySize];
    for (int i = 0; i < arraySize; i++) {
        array[i] = RandomStrings.randomAsciiOfLength(random(), stringSize);
    }
    return array;
}
 
Example #6
Source File: RandomObjects.java    From crate with Apache License 2.0 5 votes vote down vote up
/**
 * Randomly adds fields, objects, or arrays to the provided builder. The maximum depth is 5.
 */
private static void addFields(Random random, XContentBuilder builder, int minNumFields, int currentDepth) throws IOException {
    int numFields = randomIntBetween(random, minNumFields, 5);
    for (int i = 0; i < numFields; i++) {
        if (currentDepth < 5 && random.nextInt(100) >= 70) {
            if (random.nextBoolean()) {
                builder.startObject(RandomStrings.randomAsciiLettersOfLengthBetween(random, 6, 10));
                addFields(random, builder, minNumFields, currentDepth + 1);
                builder.endObject();
            } else {
                builder.startArray(RandomStrings.randomAsciiLettersOfLengthBetween(random, 6, 10));
                int numElements = randomIntBetween(random, 1, 5);
                boolean object = random.nextBoolean();
                int dataType = -1;
                if (object == false) {
                    dataType = randomDataType(random);
                }
                for (int j = 0; j < numElements; j++) {
                    if (object) {
                        builder.startObject();
                        addFields(random, builder, minNumFields, 5);
                        builder.endObject();
                    } else {
                        builder.value(randomFieldValue(random, dataType));
                    }
                }
                builder.endArray();
            }
        } else {
            builder.field(RandomStrings.randomAsciiLettersOfLengthBetween(random, 6, 10),
                    randomFieldValue(random, randomDataType(random)));
        }
    }
}
 
Example #7
Source File: RandomObjects.java    From crate with Apache License 2.0 5 votes vote down vote up
private static Object randomFieldValue(Random random, int dataType) {
    switch(dataType) {
        case 0:
            return RandomStrings.randomAsciiLettersOfLengthBetween(random, 3, 10);
        case 1:
            return RandomStrings.randomAsciiLettersOfLengthBetween(random, 3, 10);
        case 2:
            return random.nextLong();
        case 3:
            return random.nextDouble();
        default:
            throw new UnsupportedOperationException();
    }
}
 
Example #8
Source File: StorageOptionsITest.java    From crate with Apache License 2.0 5 votes vote down vote up
@Test
public void testInsertStringGreaterThanDocValuesLimit() throws Exception {
    execute("create table t1 (s string index off storage with (columnstore=false))");

    String bigString = RandomStrings.randomRealisticUnicodeOfLength(random(), MAX_TERM_LENGTH + 2);
    execute("insert into t1 (s) values (?)", new Object[]{bigString});
    refresh();

    execute("select s from t1 limit 1");
    assertThat(response.rows()[0][0], is(bigString));
}
 
Example #9
Source File: NGramTokenizerTest.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
private static void testNGrams(int minGram, int maxGram, int length, final String nonTokenChars) throws IOException {
  final String s = RandomStrings.randomAsciiLettersOfLengthBetween(random(), length, length);
  testNGrams(minGram, maxGram, s, nonTokenChars);
}
 
Example #10
Source File: EdgeNGramTokenizerTest.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
private static void testNGrams(int minGram, int maxGram, int length, final String nonTokenChars) throws IOException {
  final String s = RandomStrings.randomAsciiLettersOfLengthBetween(random(), length, length);
  testNGrams(minGram, maxGram, s, nonTokenChars);
}
 
Example #11
Source File: TestTermsHashPerField.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
public void testAddAndUpdateRandom() throws IOException {
  AtomicInteger newCalled = new AtomicInteger(0);
  AtomicInteger addCalled = new AtomicInteger(0);
  TermsHashPerField hash = createNewHash(newCalled, addCalled);
  hash.start(null, true);
  class Posting {
    int termId = -1;
    final TreeMap<Integer, Integer> docAndFreq = new TreeMap<>();
  }
  Map<BytesRef, Posting> postingMap = new HashMap<>();
  int numStrings = 1 + random().nextInt(200);
  for (int i = 0; i < numStrings; i++) {
    String randomString = RandomStrings.randomRealisticUnicodeOfCodepointLengthBetween(random(), 1, 10);
    postingMap.putIfAbsent(new BytesRef(randomString), new Posting());
  }
  List<BytesRef> bytesRefs = Arrays.asList(postingMap.keySet().toArray(new BytesRef[0]));
  Collections.sort(bytesRefs);
  int numDocs = 1 + random().nextInt(200);
  int termOrd = 0;
  for (int i = 0; i < numDocs; i++) {
    int numTerms = 1 + random().nextInt(200);
    int doc = i;
    for (int j = 0; i < numTerms; i++) {
      BytesRef ref = RandomPicks.randomFrom(random(), bytesRefs);
      Posting posting = postingMap.get(ref);
      if (posting.termId == -1) {
        posting.termId = termOrd++;
      }
      posting.docAndFreq.putIfAbsent(doc, 0);
      posting.docAndFreq.compute(doc, (key, oldVal) -> oldVal+1);
      hash.add(ref, doc);
    }
    hash.finish();
  }
  List<Posting> values = postingMap.values().stream().filter( x -> x.termId != -1)
      .collect(Collectors.toList());
  Collections.shuffle(values, random()); // term order doesn't matter
  final ByteSliceReader reader = new ByteSliceReader();
  for (Posting p : values) {
    hash.initReader(reader, p.termId, 0);
    boolean eof = false;
    int prefDoc = 0;
    for (Map.Entry<Integer, Integer> entry : p.docAndFreq.entrySet()) {
      assertFalse("the reader must not be EOF here", eof);
      eof = assertDocAndFreq(reader, (FreqProxTermsWriterPerField.FreqProxPostingsArray) hash.postingsArray,
          prefDoc, p.termId, entry.getKey(), entry.getValue());
      prefDoc = entry.getKey();
    }
    assertTrue("the last posting must be EOF on the reader", eof);
  }
}
 
Example #12
Source File: DocumentBuilderTest.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
public void testMoveLargestLast() {
  SolrInputDocument inDoc = new SolrInputDocument();
  String TEXT_FLD = "text"; // not stored.  It won't be moved.  This value is the longest, however.
  inDoc.addField(TEXT_FLD,
      "NOT STORED|" + RandomStrings.randomAsciiOfLength(random(), 4 * DocumentBuilder.MIN_LENGTH_TO_MOVE_LAST));

  String CAT_FLD = "cat"; // stored, multiValued
  inDoc.addField(CAT_FLD,
      "STORED V1|");
  //  pretty long value
  inDoc.addField(CAT_FLD,
      "STORED V2|" + RandomStrings.randomAsciiOfLength(random(), 2 * DocumentBuilder.MIN_LENGTH_TO_MOVE_LAST));
  inDoc.addField(CAT_FLD,
      "STORED V3|" + RandomStrings.randomAsciiOfLength(random(), DocumentBuilder.MIN_LENGTH_TO_MOVE_LAST));

  String SUBJECT_FLD = "subject"; // stored.  This value is long, but not long enough.
  inDoc.addField(SUBJECT_FLD,
      "2ndplace|" + RandomStrings.randomAsciiOfLength(random(), DocumentBuilder.MIN_LENGTH_TO_MOVE_LAST));

  Document outDoc = DocumentBuilder.toDocument(inDoc, h.getCore().getLatestSchema());

  // filter outDoc by stored fields; convert to list.
  List<IndexableField> storedFields = StreamSupport.stream(outDoc.spliterator(), false)
      .filter(f -> f.fieldType().stored()).collect(Collectors.toList());
  // clip to last 3.  We expect these to be for CAT_FLD
  storedFields = storedFields.subList(storedFields.size() - 3, storedFields.size());

  Iterator<IndexableField> fieldIterator = storedFields.iterator();
  IndexableField field;

  // Test that we retained the particular value ordering, even though though the 2nd of three was longest

  assertTrue(fieldIterator.hasNext());
  field = fieldIterator.next();
  assertEquals(CAT_FLD, field.name());
  assertTrue(field.stringValue().startsWith("STORED V1|"));

  assertTrue(fieldIterator.hasNext());
  field = fieldIterator.next();
  assertEquals(CAT_FLD, field.name());
  assertTrue(field.stringValue().startsWith("STORED V2|"));

  assertTrue(fieldIterator.hasNext());
  field = fieldIterator.next();
  assertEquals(CAT_FLD, field.name());
  assertTrue(field.stringValue().startsWith("STORED V3|"));
}
 
Example #13
Source File: RandomizedTaggerTest.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
private String randomStringOfLength(int min, int max) {
  return RandomStrings.randomAsciiLettersOfLengthBetween(random(), min, max).toLowerCase(Locale.ROOT);
}
 
Example #14
Source File: RandomizedTaggerTest.java    From SolrTextTagger with Apache License 2.0 4 votes vote down vote up
private String randomStringOfLength(int min, int max) {
  return RandomStrings.randomAsciiLettersOfLengthBetween(random(), min, max).toLowerCase();
}
 
Example #15
Source File: RandomObjects.java    From crate with Apache License 2.0 4 votes vote down vote up
/**
 * Returns a tuple containing random stored field values and their corresponding expected values once printed out
 * via {@link org.elasticsearch.common.xcontent.ToXContent#toXContent(XContentBuilder, ToXContent.Params)} and parsed back via
 * {@link org.elasticsearch.common.xcontent.XContentParser#objectText()}.
 * Generates values based on what can get printed out. Stored fields values are retrieved from lucene and converted via
 * {@link org.elasticsearch.index.mapper.MappedFieldType#valueForDisplay(Object)} to either strings, numbers or booleans.
 *
 * @param random Random generator
 * @param xContentType the content type, used to determine what the expected values are for float numbers.
 */
public static Tuple<List<Object>, List<Object>> randomStoredFieldValues(Random random, XContentType xContentType) {
    int numValues = randomIntBetween(random, 1, 5);
    List<Object> originalValues = new ArrayList<>();
    List<Object> expectedParsedValues = new ArrayList<>();
    int dataType = randomIntBetween(random, 0, 8);
    for (int i = 0; i < numValues; i++) {
        switch(dataType) {
            case 0:
                long randomLong = random.nextLong();
                originalValues.add(randomLong);
                expectedParsedValues.add(randomLong);
                break;
            case 1:
                int randomInt = random.nextInt();
                originalValues.add(randomInt);
                expectedParsedValues.add(randomInt);
                break;
            case 2:
                Short randomShort = (short) random.nextInt();
                originalValues.add(randomShort);
                expectedParsedValues.add(randomShort.intValue());
                break;
            case 3:
                Byte randomByte = (byte)random.nextInt();
                originalValues.add(randomByte);
                expectedParsedValues.add(randomByte.intValue());
                break;
            case 4:
                double randomDouble = random.nextDouble();
                originalValues.add(randomDouble);
                expectedParsedValues.add(randomDouble);
                break;
            case 5:
                Float randomFloat = random.nextFloat();
                originalValues.add(randomFloat);
                if (xContentType == XContentType.SMILE) {
                    //with SMILE we get back a double (this will change in Jackson 2.9 where it will return a Float)
                    expectedParsedValues.add(randomFloat.doubleValue());
                } else {
                    //with JSON AND YAML we get back a double, but with float precision.
                    expectedParsedValues.add(Double.parseDouble(randomFloat.toString()));
                }
                break;
            case 6:
                boolean randomBoolean = random.nextBoolean();
                originalValues.add(randomBoolean);
                expectedParsedValues.add(randomBoolean);
                break;
            case 7:
                String randomString = random.nextBoolean() ? RandomStrings.randomAsciiLettersOfLengthBetween(random, 3, 10) :
                        randomUnicodeOfLengthBetween(random, 3, 10);
                originalValues.add(randomString);
                expectedParsedValues.add(randomString);
                break;
            case 8:
                byte[] randomBytes = RandomStrings.randomUnicodeOfLengthBetween(random, 10, 50).getBytes(StandardCharsets.UTF_8);
                BytesArray randomBytesArray = new BytesArray(randomBytes);
                originalValues.add(randomBytesArray);
                if (xContentType == XContentType.JSON || xContentType == XContentType.YAML) {
                    //JSON and YAML write the base64 format
                    expectedParsedValues.add(Base64.getEncoder().encodeToString(randomBytes));
                } else {
                    //SMILE and CBOR write the original bytes as they support binary format
                    expectedParsedValues.add(randomBytesArray);
                }
                break;
            default:
                throw new UnsupportedOperationException();
        }
    }
    return Tuple.tuple(originalValues, expectedParsedValues);
}