Java Code Examples for org.apache.hadoop.io.file.tfile.RandomDistribution.DiscreteRNG

The following examples show how to use org.apache.hadoop.io.file.tfile.RandomDistribution.DiscreteRNG. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: TestTFileSeek.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void setUp() throws IOException {
  if (options == null) {
    options = new MyOptions(new String[0]);
  }

  conf = new Configuration();
  conf.setInt("tfile.fs.input.buffer.size", options.fsInputBufferSize);
  conf.setInt("tfile.fs.output.buffer.size", options.fsOutputBufferSize);
  path = new Path(new Path(options.rootDir), options.file);
  fs = path.getFileSystem(conf);
  timer = new NanoTimer(false);
  rng = new Random(options.seed);
  keyLenGen =
      new RandomDistribution.Zipf(new Random(rng.nextLong()),
          options.minKeyLen, options.maxKeyLen, 1.2);
  DiscreteRNG valLenGen =
      new RandomDistribution.Flat(new Random(rng.nextLong()),
          options.minValLength, options.maxValLength);
  DiscreteRNG wordLenGen =
      new RandomDistribution.Flat(new Random(rng.nextLong()),
          options.minWordLen, options.maxWordLen);
  kvGen =
      new KVGenerator(rng, true, keyLenGen, valLenGen, wordLenGen,
          options.dictSize);
}
 
Example 2
Source Project: big-c   Source File: TestTFileSeek.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void setUp() throws IOException {
  if (options == null) {
    options = new MyOptions(new String[0]);
  }

  conf = new Configuration();
  conf.setInt("tfile.fs.input.buffer.size", options.fsInputBufferSize);
  conf.setInt("tfile.fs.output.buffer.size", options.fsOutputBufferSize);
  path = new Path(new Path(options.rootDir), options.file);
  fs = path.getFileSystem(conf);
  timer = new NanoTimer(false);
  rng = new Random(options.seed);
  keyLenGen =
      new RandomDistribution.Zipf(new Random(rng.nextLong()),
          options.minKeyLen, options.maxKeyLen, 1.2);
  DiscreteRNG valLenGen =
      new RandomDistribution.Flat(new Random(rng.nextLong()),
          options.minValLength, options.maxValLength);
  DiscreteRNG wordLenGen =
      new RandomDistribution.Flat(new Random(rng.nextLong()),
          options.minWordLen, options.maxWordLen);
  kvGen =
      new KVGenerator(rng, true, keyLenGen, valLenGen, wordLenGen,
          options.dictSize);
}
 
Example 3
Source Project: attic-apex-malhar   Source File: TestTFileSeek.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void setUp() throws IOException {
  if (options == null) {
    options = new MyOptions(new String[0]);
  }

  conf = new Configuration();
  conf.setInt("tfile.fs.input.buffer.size", options.fsInputBufferSize);
  conf.setInt("tfile.fs.output.buffer.size", options.fsOutputBufferSize);
  path = new Path(new Path(options.rootDir), options.file);
  fs = path.getFileSystem(conf);
  timer = new NanoTimer(false);
  rng = new Random(options.seed);
  keyLenGen =
      new RandomDistribution.Zipf(new Random(rng.nextLong()),
          options.minKeyLen, options.maxKeyLen, 1.2);
  DiscreteRNG valLenGen =
      new RandomDistribution.Flat(new Random(rng.nextLong()),
          options.minValLength, options.maxValLength);
  DiscreteRNG wordLenGen =
      new RandomDistribution.Flat(new Random(rng.nextLong()),
          options.minWordLen, options.maxWordLen);
  kvGen =
      new KVGenerator(rng, true, keyLenGen, valLenGen, wordLenGen,
          options.dictSize);
}
 
Example 4
Source Project: RDFS   Source File: TestTFileSeek.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void setUp() throws IOException {
  if (options == null) {
    options = new MyOptions(new String[0]);
  }

  conf = new Configuration();
  conf.setInt("tfile.fs.input.buffer.size", options.fsInputBufferSize);
  conf.setInt("tfile.fs.output.buffer.size", options.fsOutputBufferSize);
  path = new Path(new Path(options.rootDir), options.file);
  fs = path.getFileSystem(conf);
  timer = new NanoTimer(false);
  rng = new Random(options.seed);
  keyLenGen =
      new RandomDistribution.Zipf(new Random(rng.nextLong()),
          options.minKeyLen, options.maxKeyLen, 1.2);
  DiscreteRNG valLenGen =
      new RandomDistribution.Flat(new Random(rng.nextLong()),
          options.minValLength, options.maxValLength);
  DiscreteRNG wordLenGen =
      new RandomDistribution.Flat(new Random(rng.nextLong()),
          options.minWordLen, options.maxWordLen);
  kvGen =
      new KVGenerator(rng, true, keyLenGen, valLenGen, wordLenGen,
          options.dictSize);
}
 
Example 5
Source Project: hadoop-gpu   Source File: TestTFileSeek.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void setUp() throws IOException {
  if (options == null) {
    options = new MyOptions(new String[0]);
  }

  conf = new Configuration();
  conf.setInt("tfile.fs.input.buffer.size", options.fsInputBufferSize);
  conf.setInt("tfile.fs.output.buffer.size", options.fsOutputBufferSize);
  path = new Path(new Path(options.rootDir), options.file);
  fs = path.getFileSystem(conf);
  timer = new NanoTimer(false);
  rng = new Random(options.seed);
  keyLenGen =
      new RandomDistribution.Zipf(new Random(rng.nextLong()),
          options.minKeyLen, options.maxKeyLen, 1.2);
  DiscreteRNG valLenGen =
      new RandomDistribution.Flat(new Random(rng.nextLong()),
          options.minValLength, options.maxValLength);
  DiscreteRNG wordLenGen =
      new RandomDistribution.Flat(new Random(rng.nextLong()),
          options.minWordLen, options.maxWordLen);
  kvGen =
      new KVGenerator(rng, true, keyLenGen, valLenGen, wordLenGen,
          options.dictSize);
}
 
Example 6
Source Project: hadoop   Source File: KVGenerator.java    License: Apache License 2.0 5 votes vote down vote up
public KVGenerator(Random random, boolean sorted, DiscreteRNG keyLenRNG,
    DiscreteRNG valLenRNG, DiscreteRNG wordLenRNG, int dictSize) {
  this.random = random;
  dict = new byte[dictSize][];
  this.sorted = sorted;
  this.keyLenRNG = keyLenRNG;
  this.valLenRNG = valLenRNG;
  for (int i = 0; i < dictSize; ++i) {
    int wordLen = wordLenRNG.nextInt();
    dict[i] = new byte[wordLen];
    random.nextBytes(dict[i]);
  }
  lastKey = new BytesWritable();
  fillKey(lastKey);
}
 
Example 7
Source Project: hadoop   Source File: KeySampler.java    License: Apache License 2.0 5 votes vote down vote up
public KeySampler(Random random, RawComparable first, RawComparable last,
    DiscreteRNG keyLenRNG) throws IOException {
  this.random = random;
  min = keyPrefixToInt(first);
  max = keyPrefixToInt(last);
  this.keyLenRNG = keyLenRNG;
}
 
Example 8
Source Project: big-c   Source File: KVGenerator.java    License: Apache License 2.0 5 votes vote down vote up
public KVGenerator(Random random, boolean sorted, DiscreteRNG keyLenRNG,
    DiscreteRNG valLenRNG, DiscreteRNG wordLenRNG, int dictSize) {
  this.random = random;
  dict = new byte[dictSize][];
  this.sorted = sorted;
  this.keyLenRNG = keyLenRNG;
  this.valLenRNG = valLenRNG;
  for (int i = 0; i < dictSize; ++i) {
    int wordLen = wordLenRNG.nextInt();
    dict[i] = new byte[wordLen];
    random.nextBytes(dict[i]);
  }
  lastKey = new BytesWritable();
  fillKey(lastKey);
}
 
Example 9
Source Project: big-c   Source File: KeySampler.java    License: Apache License 2.0 5 votes vote down vote up
public KeySampler(Random random, RawComparable first, RawComparable last,
    DiscreteRNG keyLenRNG) throws IOException {
  this.random = random;
  min = keyPrefixToInt(first);
  max = keyPrefixToInt(last);
  this.keyLenRNG = keyLenRNG;
}
 
Example 10
Source Project: RDFS   Source File: KVGenerator.java    License: Apache License 2.0 5 votes vote down vote up
public KVGenerator(Random random, boolean sorted, DiscreteRNG keyLenRNG,
    DiscreteRNG valLenRNG, DiscreteRNG wordLenRNG, int dictSize) {
  this.random = random;
  dict = new byte[dictSize][];
  this.sorted = sorted;
  this.keyLenRNG = keyLenRNG;
  this.valLenRNG = valLenRNG;
  for (int i = 0; i < dictSize; ++i) {
    int wordLen = wordLenRNG.nextInt();
    dict[i] = new byte[wordLen];
    random.nextBytes(dict[i]);
  }
  lastKey = new BytesWritable();
  fillKey(lastKey);
}
 
Example 11
Source Project: RDFS   Source File: KeySampler.java    License: Apache License 2.0 5 votes vote down vote up
public KeySampler(Random random, RawComparable first, RawComparable last,
    DiscreteRNG keyLenRNG) throws IOException {
  this.random = random;
  min = keyPrefixToInt(first);
  max = keyPrefixToInt(last);
  this.keyLenRNG = keyLenRNG;
}
 
Example 12
Source Project: hadoop-gpu   Source File: KVGenerator.java    License: Apache License 2.0 5 votes vote down vote up
public KVGenerator(Random random, boolean sorted, DiscreteRNG keyLenRNG,
    DiscreteRNG valLenRNG, DiscreteRNG wordLenRNG, int dictSize) {
  this.random = random;
  dict = new byte[dictSize][];
  this.sorted = sorted;
  this.keyLenRNG = keyLenRNG;
  this.valLenRNG = valLenRNG;
  for (int i = 0; i < dictSize; ++i) {
    int wordLen = wordLenRNG.nextInt();
    dict[i] = new byte[wordLen];
    random.nextBytes(dict[i]);
  }
  lastKey = new BytesWritable();
  fillKey(lastKey);
}
 
Example 13
Source Project: hadoop-gpu   Source File: KeySampler.java    License: Apache License 2.0 5 votes vote down vote up
public KeySampler(Random random, RawComparable first, RawComparable last,
    DiscreteRNG keyLenRNG) throws IOException {
  this.random = random;
  min = keyPrefixToInt(first);
  max = keyPrefixToInt(last);
  this.keyLenRNG = keyLenRNG;
}