org.apache.hadoop.io.file.tfile.RandomDistribution.DiscreteRNG Java Examples

The following examples show how to use org.apache.hadoop.io.file.tfile.RandomDistribution.DiscreteRNG. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestTFileSeek.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Override
public void setUp() throws IOException {
  if (options == null) {
    options = new MyOptions(new String[0]);
  }

  conf = new Configuration();
  conf.setInt("tfile.fs.input.buffer.size", options.fsInputBufferSize);
  conf.setInt("tfile.fs.output.buffer.size", options.fsOutputBufferSize);
  path = new Path(new Path(options.rootDir), options.file);
  fs = path.getFileSystem(conf);
  timer = new NanoTimer(false);
  rng = new Random(options.seed);
  keyLenGen =
      new RandomDistribution.Zipf(new Random(rng.nextLong()),
          options.minKeyLen, options.maxKeyLen, 1.2);
  DiscreteRNG valLenGen =
      new RandomDistribution.Flat(new Random(rng.nextLong()),
          options.minValLength, options.maxValLength);
  DiscreteRNG wordLenGen =
      new RandomDistribution.Flat(new Random(rng.nextLong()),
          options.minWordLen, options.maxWordLen);
  kvGen =
      new KVGenerator(rng, true, keyLenGen, valLenGen, wordLenGen,
          options.dictSize);
}
Example #2
Source File: TestTFileSeek.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Override
public void setUp() throws IOException {
  if (options == null) {
    options = new MyOptions(new String[0]);
  }

  conf = new Configuration();
  conf.setInt("tfile.fs.input.buffer.size", options.fsInputBufferSize);
  conf.setInt("tfile.fs.output.buffer.size", options.fsOutputBufferSize);
  path = new Path(new Path(options.rootDir), options.file);
  fs = path.getFileSystem(conf);
  timer = new NanoTimer(false);
  rng = new Random(options.seed);
  keyLenGen =
      new RandomDistribution.Zipf(new Random(rng.nextLong()),
          options.minKeyLen, options.maxKeyLen, 1.2);
  DiscreteRNG valLenGen =
      new RandomDistribution.Flat(new Random(rng.nextLong()),
          options.minValLength, options.maxValLength);
  DiscreteRNG wordLenGen =
      new RandomDistribution.Flat(new Random(rng.nextLong()),
          options.minWordLen, options.maxWordLen);
  kvGen =
      new KVGenerator(rng, true, keyLenGen, valLenGen, wordLenGen,
          options.dictSize);
}
Example #3
Source File: TestTFileSeek.java    From attic-apex-malhar with Apache License 2.0 6 votes vote down vote up 
@Override
public void setUp() throws IOException {
  if (options == null) {
    options = new MyOptions(new String[0]);
  }

  conf = new Configuration();
  conf.setInt("tfile.fs.input.buffer.size", options.fsInputBufferSize);
  conf.setInt("tfile.fs.output.buffer.size", options.fsOutputBufferSize);
  path = new Path(new Path(options.rootDir), options.file);
  fs = path.getFileSystem(conf);
  timer = new NanoTimer(false);
  rng = new Random(options.seed);
  keyLenGen =
      new RandomDistribution.Zipf(new Random(rng.nextLong()),
          options.minKeyLen, options.maxKeyLen, 1.2);
  DiscreteRNG valLenGen =
      new RandomDistribution.Flat(new Random(rng.nextLong()),
          options.minValLength, options.maxValLength);
  DiscreteRNG wordLenGen =
      new RandomDistribution.Flat(new Random(rng.nextLong()),
          options.minWordLen, options.maxWordLen);
  kvGen =
      new KVGenerator(rng, true, keyLenGen, valLenGen, wordLenGen,
          options.dictSize);
}
Example #4
Source File: TestTFileSeek.java    From RDFS with Apache License 2.0 6 votes vote down vote up 
@Override
public void setUp() throws IOException {
  if (options == null) {
    options = new MyOptions(new String[0]);
  }

  conf = new Configuration();
  conf.setInt("tfile.fs.input.buffer.size", options.fsInputBufferSize);
  conf.setInt("tfile.fs.output.buffer.size", options.fsOutputBufferSize);
  path = new Path(new Path(options.rootDir), options.file);
  fs = path.getFileSystem(conf);
  timer = new NanoTimer(false);
  rng = new Random(options.seed);
  keyLenGen =
      new RandomDistribution.Zipf(new Random(rng.nextLong()),
          options.minKeyLen, options.maxKeyLen, 1.2);
  DiscreteRNG valLenGen =
      new RandomDistribution.Flat(new Random(rng.nextLong()),
          options.minValLength, options.maxValLength);
  DiscreteRNG wordLenGen =
      new RandomDistribution.Flat(new Random(rng.nextLong()),
          options.minWordLen, options.maxWordLen);
  kvGen =
      new KVGenerator(rng, true, keyLenGen, valLenGen, wordLenGen,
          options.dictSize);
}
Example #5
Source File: TestTFileSeek.java    From hadoop-gpu with Apache License 2.0 6 votes vote down vote up 
@Override
public void setUp() throws IOException {
  if (options == null) {
    options = new MyOptions(new String[0]);
  }

  conf = new Configuration();
  conf.setInt("tfile.fs.input.buffer.size", options.fsInputBufferSize);
  conf.setInt("tfile.fs.output.buffer.size", options.fsOutputBufferSize);
  path = new Path(new Path(options.rootDir), options.file);
  fs = path.getFileSystem(conf);
  timer = new NanoTimer(false);
  rng = new Random(options.seed);
  keyLenGen =
      new RandomDistribution.Zipf(new Random(rng.nextLong()),
          options.minKeyLen, options.maxKeyLen, 1.2);
  DiscreteRNG valLenGen =
      new RandomDistribution.Flat(new Random(rng.nextLong()),
          options.minValLength, options.maxValLength);
  DiscreteRNG wordLenGen =
      new RandomDistribution.Flat(new Random(rng.nextLong()),
          options.minWordLen, options.maxWordLen);
  kvGen =
      new KVGenerator(rng, true, keyLenGen, valLenGen, wordLenGen,
          options.dictSize);
}
Example #6
Source File: KVGenerator.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public KVGenerator(Random random, boolean sorted, DiscreteRNG keyLenRNG,
    DiscreteRNG valLenRNG, DiscreteRNG wordLenRNG, int dictSize) {
  this.random = random;
  dict = new byte[dictSize][];
  this.sorted = sorted;
  this.keyLenRNG = keyLenRNG;
  this.valLenRNG = valLenRNG;
  for (int i = 0; i < dictSize; ++i) {
    int wordLen = wordLenRNG.nextInt();
    dict[i] = new byte[wordLen];
    random.nextBytes(dict[i]);
  }
  lastKey = new BytesWritable();
  fillKey(lastKey);
}
Example #7
Source File: KeySampler.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public KeySampler(Random random, RawComparable first, RawComparable last,
    DiscreteRNG keyLenRNG) throws IOException {
  this.random = random;
  min = keyPrefixToInt(first);
  max = keyPrefixToInt(last);
  this.keyLenRNG = keyLenRNG;
}
Example #8
Source File: KVGenerator.java    From big-c with Apache License 2.0 5 votes vote down vote up 
public KVGenerator(Random random, boolean sorted, DiscreteRNG keyLenRNG,
    DiscreteRNG valLenRNG, DiscreteRNG wordLenRNG, int dictSize) {
  this.random = random;
  dict = new byte[dictSize][];
  this.sorted = sorted;
  this.keyLenRNG = keyLenRNG;
  this.valLenRNG = valLenRNG;
  for (int i = 0; i < dictSize; ++i) {
    int wordLen = wordLenRNG.nextInt();
    dict[i] = new byte[wordLen];
    random.nextBytes(dict[i]);
  }
  lastKey = new BytesWritable();
  fillKey(lastKey);
}
Example #9
Source File: KeySampler.java    From big-c with Apache License 2.0 5 votes vote down vote up 
public KeySampler(Random random, RawComparable first, RawComparable last,
    DiscreteRNG keyLenRNG) throws IOException {
  this.random = random;
  min = keyPrefixToInt(first);
  max = keyPrefixToInt(last);
  this.keyLenRNG = keyLenRNG;
}
Example #10
Source File: KVGenerator.java    From RDFS with Apache License 2.0 5 votes vote down vote up 
public KVGenerator(Random random, boolean sorted, DiscreteRNG keyLenRNG,
    DiscreteRNG valLenRNG, DiscreteRNG wordLenRNG, int dictSize) {
  this.random = random;
  dict = new byte[dictSize][];
  this.sorted = sorted;
  this.keyLenRNG = keyLenRNG;
  this.valLenRNG = valLenRNG;
  for (int i = 0; i < dictSize; ++i) {
    int wordLen = wordLenRNG.nextInt();
    dict[i] = new byte[wordLen];
    random.nextBytes(dict[i]);
  }
  lastKey = new BytesWritable();
  fillKey(lastKey);
}
Example #11
Source File: KeySampler.java    From RDFS with Apache License 2.0 5 votes vote down vote up 
public KeySampler(Random random, RawComparable first, RawComparable last,
    DiscreteRNG keyLenRNG) throws IOException {
  this.random = random;
  min = keyPrefixToInt(first);
  max = keyPrefixToInt(last);
  this.keyLenRNG = keyLenRNG;
}
Example #12
Source File: KVGenerator.java    From hadoop-gpu with Apache License 2.0 5 votes vote down vote up 
public KVGenerator(Random random, boolean sorted, DiscreteRNG keyLenRNG,
    DiscreteRNG valLenRNG, DiscreteRNG wordLenRNG, int dictSize) {
  this.random = random;
  dict = new byte[dictSize][];
  this.sorted = sorted;
  this.keyLenRNG = keyLenRNG;
  this.valLenRNG = valLenRNG;
  for (int i = 0; i < dictSize; ++i) {
    int wordLen = wordLenRNG.nextInt();
    dict[i] = new byte[wordLen];
    random.nextBytes(dict[i]);
  }
  lastKey = new BytesWritable();
  fillKey(lastKey);
}
Example #13
Source File: KeySampler.java    From hadoop-gpu with Apache License 2.0 5 votes vote down vote up 
public KeySampler(Random random, RawComparable first, RawComparable last,
    DiscreteRNG keyLenRNG) throws IOException {
  this.random = random;
  min = keyPrefixToInt(first);
  max = keyPrefixToInt(last);
  this.keyLenRNG = keyLenRNG;
}