@Before public void setUp() throws IOException { if (options == null) { options = new MyOptions(new String[0]); } conf = new Configuration(); conf.setInt("tfile.fs.input.buffer.size", options.fsInputBufferSize); conf.setInt("tfile.fs.output.buffer.size", options.fsOutputBufferSize); path = new Path(new Path(options.rootDir), options.file); fs = path.getFileSystem(conf); timer = new NanoTimer(false); rng = new Random(options.seed); keyLenGen = new RandomDistribution.Zipf(new Random(rng.nextLong()), options.minKeyLen, options.maxKeyLen, 1.2); DiscreteRNG valLenGen = new RandomDistribution.Flat(new Random(rng.nextLong()), options.minValLength, options.maxValLength); DiscreteRNG wordLenGen = new RandomDistribution.Flat(new Random(rng.nextLong()), options.minWordLen, options.maxWordLen); kvGen = new KVGenerator(rng, true, keyLenGen, valLenGen, wordLenGen, options.dictSize); }
@Override public void setUp() throws IOException { if (options == null) { options = new MyOptions(new String[0]); } conf = new Configuration(); conf.setInt("tfile.fs.input.buffer.size", options.fsInputBufferSize); conf.setInt("tfile.fs.output.buffer.size", options.fsOutputBufferSize); path = new Path(new Path(options.rootDir), options.file); fs = path.getFileSystem(conf); timer = new NanoTimer(false); rng = new Random(options.seed); keyLenGen = new RandomDistribution.Zipf(new Random(rng.nextLong()), options.minKeyLen, options.maxKeyLen, 1.2); DiscreteRNG valLenGen = new RandomDistribution.Flat(new Random(rng.nextLong()), options.minValLength, options.maxValLength); DiscreteRNG wordLenGen = new RandomDistribution.Flat(new Random(rng.nextLong()), options.minWordLen, options.maxWordLen); kvGen = new KVGenerator(rng, true, keyLenGen, valLenGen, wordLenGen, options.dictSize); }
public KVGenerator(Random random, boolean sorted, DiscreteRNG keyLenRNG, DiscreteRNG valLenRNG, DiscreteRNG wordLenRNG, int dictSize) { this.random = random; dict = new byte[dictSize][]; this.sorted = sorted; this.keyLenRNG = keyLenRNG; this.valLenRNG = valLenRNG; for (int i = 0; i < dictSize; ++i) { int wordLen = wordLenRNG.nextInt(); dict[i] = new byte[wordLen]; random.nextBytes(dict[i]); } lastKey = new BytesWritable(); fillKey(lastKey); }
public KeySampler(Random random, RawComparable first, RawComparable last, DiscreteRNG keyLenRNG) throws IOException { this.random = random; min = keyPrefixToInt(first); max = keyPrefixToInt(last); this.keyLenRNG = keyLenRNG; }