org.roaringbitmap.buffer.ImmutableRoaringBitmap Java Examples

@Setup(Level.Trial)
public void createBitmaps() {
  random = new SplittableRandom(seed);
  RoaringBitmapWriter<RoaringBitmap> bitmapWriter = constructionStrategy.create();
  bitmaps = new RoaringBitmap[count];
  bufferBitmaps = new ImmutableRoaringBitmap[count];
  double p = Math.pow(probability, 1D/count);
  for (int i = 0; i < count; ++i) {
    for (int j = (int)(Math.log(random.nextDouble())/Math.log(1-p));
         j < size;
         j += (int)(Math.log(random.nextDouble())/Math.log(1-p)) + 1) {
        bitmapWriter.add(j);
    }
    bitmaps[i] = bitmapWriter.get();
    bufferBitmaps[i] = bitmaps[i].toMutableRoaringBitmap();
    bitmapWriter.reset();
  }
}
 

@Test
public void testNullValueVectorCreation()
    throws IOException {
  NullValueVectorCreator creator = new NullValueVectorCreator(TEMP_DIR, COLUMN_NAME);
  for (int i = 0; i < 100; i++) {
    creator.setNull(i);
  }
  ImmutableRoaringBitmap nullBitmap = creator.getNullBitmap();
  for (int i = 0; i < 100; i++) {
    Assert.assertTrue(nullBitmap.contains(i));
  }
  creator.seal();

  Assert.assertEquals(TEMP_DIR.list().length, 1);
  Assert.assertEquals(NULL_VALUE_FILE, TEMP_DIR.list()[0]);
}
 

public ImmutableState() {

      /**
       * Mutable & Immutable
       */

      for (int k = 0; k < N; ++k) {
        ewah_mutable[k] = new MutableRoaringBitmap();
        for (int x = 0; x < M; ++x) {
          ewah_mutable[k].add(x * (N - k + 2));
        }
        ewah_mutable[k].trim();
        try {
          ewah_immutable[k] = convertToMappedBitmap(ewah_mutable[k]);
        } catch (IOException e) {
          throw new RuntimeException(e);
        }
      }

      ors_immutable = new ArrayList<ImmutableRoaringBitmap[]>();
      for (int k = 1; k < N; k += 10) {
        ors_immutable.add(Arrays.copyOf(ewah_immutable, k + 1));

      }

    }
 

private EvalResult doEvalCompareEqual(CompareTupleFilter filter) {
    EvalResult result = new EvalResult();
    String column = filter.getColumn().getName();
    byte[] value = null;
    if (filter.getFirstValue() instanceof ByteArray) {
        value = ((ByteArray)filter.getFirstValue()).array();
    } else if (filter.getFirstValue() instanceof byte[]) {
        value = (byte[])filter.getFirstValue();
    } else if (filter.getFirstValue() instanceof String) {
        value = Bytes.toBytes((String) filter.getFirstValue());
    }
    ColInvertIndexSearcher colSearcher = colIndexSearchers.get(column);
    if (colSearcher == null) {
        return EvalResult.ALL_MATCH;
    }
    ImmutableRoaringBitmap bitmap = colSearcher.searchValue(value);
    if (bitmap != null) {
        result.bitmap = bitmap;
    }
    return result;
}
 

private Iterator<ImmutableRoaringBitmap> toImmutableRoaringBitmapIterator(
    final Iterable<Bitmap> bitmaps) {
  return new Iterator<ImmutableRoaringBitmap>() {
    final Iterator<Bitmap> i = bitmaps.iterator();

    @Override
    public boolean hasNext() {
      return i.hasNext();
    }

    @Override
    public ImmutableRoaringBitmap next() {
      return ((ImmutableRoaringBitmapWrapper) i.next()).bitmap;
    }

    @Override
    public void remove() {
      throw new UnsupportedOperationException();
    }
  };
}
 

@Override
public void readFields(ByteBuffer in) throws IOException {
    int size = peekLength(in);
    // make a copy of the content to be safe
    byte[] dst = new byte[size];
    in.get(dst);

    // ImmutableRoaringBitmap only maps the buffer, thus faster than constructing a MutableRoaringBitmap.
    // we'll convert to MutableRoaringBitmap later when mutate is needed
    bitmap = new ImmutableRoaringBitmap(ByteBuffer.wrap(dst));
}
 

@Setup(Level.Trial)
public void setup() throws Exception {
  ZipRealDataRetriever dataRetriever = new ZipRealDataRetriever(dataset);
  bitmaps = StreamSupport.stream(dataRetriever.fetchBitPositions().spliterator(), false)
          .map(RoaringBitmap::bitmapOf)
          .toArray(RoaringBitmap[]::new);
  immutableRoaringBitmaps = Arrays.stream(bitmaps).map(RoaringBitmap::toMutableRoaringBitmap)
          .toArray(ImmutableRoaringBitmap[]::new);
}
 

public ImmutableRoaringBitmap searchValue(byte[] value) {
    int offset = offsetDictionary.getBitMapOffset(value);
    if (offset == -1) {
        return null;
    }
    ByteBuffer usedBuffer = bitmapBuffer.asReadOnlyBuffer();
    usedBuffer.position(bitmapStartOffset + offset);
    ImmutableRoaringBitmap bitmap = new ImmutableRoaringBitmap(usedBuffer);

    return bitmap;
}
 

@Override
public BitmapAggregator naiveOrAggregator() {
  return new BitmapAggregator() {
    @Override
    public Bitmap aggregate(final Iterable<Bitmap> bitmaps) {
      Iterator<ImmutableRoaringBitmap> iterator = toImmutableRoaringBitmapIterator(bitmaps);
      return new ImmutableRoaringBitmapWrapper(BufferFastAggregation.naive_or(iterator));
    }
  };
}
 

@Override
public BitmapAggregator naiveAndAggregator() {
  return new BitmapAggregator() {
    @Override
    public Bitmap aggregate(Iterable<Bitmap> bitmaps) {
      Iterator<ImmutableRoaringBitmap> iterator = toImmutableRoaringBitmapIterator(bitmaps);
      return new ImmutableRoaringBitmapWrapper(BufferFastAggregation.naive_and(iterator));
    }
  };
}
 

@Override
public ImmutableBitmap complement(ImmutableBitmap b)
{
  return new WrappedImmutableRoaringBitmap(
      ImmutableRoaringBitmap.flip(
          ((WrappedImmutableRoaringBitmap) b).getBitmap(),
          0,
          b.size()
      )
  );
}
 

@Override
public void readFields(ByteBuffer in) throws IOException {
    int size = peekLength(in);
    // make a copy of the content to be safe
    byte[] dst = new byte[size];
    in.get(dst);

    // ImmutableRoaringBitmap only maps the buffer, thus faster than constructing a MutableRoaringBitmap.
    // we'll convert to MutableRoaringBitmap later when mutate is needed
    bitmap = new ImmutableRoaringBitmap(ByteBuffer.wrap(dst));
}
 

private EvalResult doEvalCompareIn(CompareTupleFilter filter) {
    EvalResult result = new EvalResult();
    String column = filter.getColumn().getName();
    ColInvertIndexSearcher colSearcher = colIndexSearchers.get(column);
    if (colSearcher == null) {
        return EvalResult.ALL_MATCH;
    }
    List<ImmutableRoaringBitmap> bitmaps = Lists.newArrayList();
    for (Object value : filter.getValues()) {
        byte[] bytes = null;
        if (value instanceof ByteArray) {
            bytes = ((ByteArray)value).array();
        } else if (value instanceof byte[]) {
            bytes = (byte[])value;
        } else if (value instanceof String) {
            bytes = Bytes.toBytes((String)value);
        }
        ImmutableRoaringBitmap bitmap = colSearcher.searchValue(bytes);
        if (bitmap != null) {
            bitmaps.add(bitmap);
        }
    }
    if (bitmaps.isEmpty()) {
        return result;
    }

    result.bitmap = ImmutableRoaringBitmap.or(bitmaps.toArray(new ImmutableRoaringBitmap[bitmaps.size()]));
    return result;
}
 

@Override
public MutableRoaringBitmap applyAnd(ImmutableRoaringBitmap docIds) {
  ProjectionOperator projectionOperator = new ProjectionOperator(_dataSourceMap, new BitmapDocIdSetOperator(docIds));
  MutableRoaringBitmap matchingDocIds = new MutableRoaringBitmap();
  ProjectionBlock projectionBlock;
  while ((projectionBlock = projectionOperator.nextBlock()) != null) {
    processProjectionBlock(projectionBlock, matchingDocIds);
  }
  return matchingDocIds;
}
 

protected static ImmutableRoaringBitmap makeOffheapRoaring(MutableRoaringBitmap r) throws IOException
{
  final int size = r.serializedSizeInBytes();
  final ByteBuffer buf = ByteBuffer.allocateDirect(size);
  totalRoaringBytes += size;
  roaringCount++;
  return writeImmutable(r, buf);
}
 

private ImmutableRoaringBitmap convertToMappedBitmap(MutableRoaringBitmap orig)
    throws IOException {
  File tmpfile = File.createTempFile("roaring", ".bin");
  tmpfile.deleteOnExit();
  final FileOutputStream fos = new FileOutputStream(tmpfile);
  orig.serialize(new DataOutputStream(fos));
  long totalcount = fos.getChannel().position();
  fos.close();
  RandomAccessFile memoryMappedFile = new RandomAccessFile(tmpfile, "r");
  ByteBuffer bb =
      memoryMappedFile.getChannel().map(FileChannel.MapMode.READ_ONLY, 0, totalcount);
  memoryMappedFile.close();
  return new ImmutableRoaringBitmap(bb);
}
 

@Override
public MutableRoaringBitmap applyAnd(ImmutableRoaringBitmap docIds) {
  MutableRoaringBitmap result = new MutableRoaringBitmap();
  IntIterator docIdIterator = docIds.getIntIterator();
  int nextDocId;
  while (docIdIterator.hasNext() && (nextDocId = docIdIterator.next()) < _numDocs) {
    _numEntriesScanned++;
    if (_valueMatcher.doesValueMatch(nextDocId)) {
      result.add(nextDocId);
    }
  }
  return result;
}
 

@Benchmark
public void viaImmutable(BenchmarkState state, Blackhole bh) {
  byte[][] buffers = state.buffers;
  for (int i = 0; i < buffers.length; ++i) {
    RoaringBitmap bitmap = new ImmutableRoaringBitmap(ByteBuffer.wrap(state.buffers[i])).toRoaringBitmap();
    bh.consume(bitmap);
  }
}
 

@Benchmark
public ImmutableRoaringBitmap immutablePairwiseACAndNotRC(RealDataRoaringOnlyBenchmarkState bs) {
  ImmutableRoaringBitmap last = null;
  for (int k = 0; k + 1 < bs.immutableBitmaps.size(); ++k) {
    last = ImmutableRoaringBitmap.andNot(bs.immutableOnlyArrayContainers.get(k), bs.immutableOnlyRunContainers.get(k + 1));
  }
  return last;
}
 

public List<ImmutableRoaringBitmap> convertToImmutableRoaring(List<MutableRoaringBitmap> source)
    throws IOException {
  File tmpfile = File.createTempFile("roaring", "bin");
  tmpfile.deleteOnExit();
  final FileOutputStream fos = new FileOutputStream(tmpfile);
  final DataOutputStream dos = new DataOutputStream(fos);

  for (MutableRoaringBitmap rb1 : source)
    rb1.serialize(dos);

  final long totalcount = fos.getChannel().position();
  dos.close();
  final RandomAccessFile memoryMappedFile = new RandomAccessFile(tmpfile, "r");
  ByteBuffer out =
      memoryMappedFile.getChannel().map(FileChannel.MapMode.READ_ONLY, 0, totalcount);
  ArrayList<ImmutableRoaringBitmap> answer =
      new ArrayList<ImmutableRoaringBitmap>(source.size());
  while (out.position() < out.limit()) {
    final ByteBuffer bb = out.slice();
    MutableRoaringBitmap equiv = source.get(answer.size());
    ImmutableRoaringBitmap newbitmap = new ImmutableRoaringBitmap(bb);
    if (!equiv.equals(newbitmap))
      throw new RuntimeException("bitmaps do not match");
    answer.add(newbitmap);
    out.position(out.position() + newbitmap.serializedSizeInBytes());
  }
  memoryMappedFile.close();
  return answer;
}
 

public List<ImmutableRoaringBitmap> convertToImmutableRoaring(List<MutableRoaringBitmap> source)
    throws IOException {
  File tmpfile = File.createTempFile("roaring", "bin");
  tmpfile.deleteOnExit();
  final FileOutputStream fos = new FileOutputStream(tmpfile);
  final DataOutputStream dos = new DataOutputStream(fos);

  for (MutableRoaringBitmap rb1 : source)
    rb1.serialize(dos);

  final long totalcount = fos.getChannel().position();
  dos.close();
  final RandomAccessFile memoryMappedFile = new RandomAccessFile(tmpfile, "r");
  ByteBuffer out =
      memoryMappedFile.getChannel().map(FileChannel.MapMode.READ_ONLY, 0, totalcount);
  ArrayList<ImmutableRoaringBitmap> answer =
      new ArrayList<ImmutableRoaringBitmap>(source.size());
  while (out.position() < out.limit()) {
    final ByteBuffer bb = out.slice();
    MutableRoaringBitmap equiv = source.get(answer.size());
    ImmutableRoaringBitmap newbitmap = new ImmutableRoaringBitmap(bb);
    if (!equiv.equals(newbitmap))
      throw new RuntimeException("bitmaps do not match");
    answer.add(newbitmap);
    out.position(out.position() + newbitmap.serializedSizeInBytes());
  }
  memoryMappedFile.close();
  return answer;
}
 

public BitmapBasedFilterOperator(ImmutableRoaringBitmap docIds, boolean exclusive, int numDocs) {
  _predicateEvaluator = null;
  _invertedIndexReader = null;
  _docIds = docIds;
  _exclusive = exclusive;
  _numDocs = numDocs;
}
 

private synchronized ImmutableRoaringBitmap buildRoaringBitmapForIndex(final int rangeId) {
  final long currentOffset = getOffset(rangeId);
  final long nextOffset = getOffset(rangeId + 1);
  final int bufferLength = (int) (nextOffset - currentOffset);

  // Slice the buffer appropriately for Roaring Bitmap
  ByteBuffer bb = _dataBuffer.toDirectByteBuffer(currentOffset, bufferLength);
  return new ImmutableRoaringBitmap(bb);
}
 

@Test
public void flipInvalidRange3() {
    assertThrows(IllegalArgumentException.class, () -> {
        ImmutableRoaringBitmap a = ImmutableRoaringBitmap.bitmapOf(1, 5, 7, 13);
        ImmutableRoaringBitmap.flip(a, 1L, 1L << 33);
    });
}
 

@Test
public void flipInvalidRange2() {
    assertThrows(IllegalArgumentException.class, () -> {
        ImmutableRoaringBitmap a = ImmutableRoaringBitmap.bitmapOf(1, 5, 7, 13);
        ImmutableRoaringBitmap.flip(a, 1L << 32, 1L << 33);
    });
}
 

@Test
public void flipInvalidRange() {
    assertThrows(RuntimeException.class, () -> {
        ImmutableRoaringBitmap a = ImmutableRoaringBitmap.bitmapOf(1, 5, 7, 13);
        ImmutableRoaringBitmap.flip(a, 7L, 5L);
    });
}
 

@Test
public void or() {
    ImmutableRoaringBitmap a = ImmutableRoaringBitmap.bitmapOf(1, 73647, 83469);
    ImmutableRoaringBitmap b = ImmutableRoaringBitmap.bitmapOf(1, 5, 10<<16);
    ImmutableRoaringBitmap expected = ImmutableRoaringBitmap.bitmapOf(1, 5, 73647, 83469, 10<<16);
    assertEquals(expected, ImmutableRoaringBitmap.or(a, b));
    assertEquals(expected, ImmutableRoaringBitmap.or(b, a));
}
 

/**
 * Create a RoaringBitmap from a MutableRoaringBitmap or ImmutableRoaringBitmap. The source is not
 * modified.
 *
 * @param rb the original bitmap
 */
public RoaringBitmap(ImmutableRoaringBitmap rb) {
  highLowContainer = new RoaringArray();
  MappeableContainerPointer cp = rb.getContainerPointer();
  while (cp.getContainer() != null) {
    highLowContainer.append(cp.key(), cp.getContainer().toContainer());
    cp.advance();
  }
}
 

private void validate(File invertedIndex, Set<Integer>[] postingLists)
    throws IOException {
  try (BitmapInvertedIndexReader reader = new BitmapInvertedIndexReader(
      PinotDataBuffer.mapReadOnlyBigEndianFile(invertedIndex), CARDINALITY)) {
    for (int dictId = 0; dictId < CARDINALITY; dictId++) {
      ImmutableRoaringBitmap bitmap = reader.getDocIds(dictId);
      Set<Integer> expected = postingLists[dictId];
      Assert.assertEquals(bitmap.getCardinality(), expected.size());
      IntIterator intIterator = bitmap.getIntIterator();
      while (intIterator.hasNext()) {
        Assert.assertTrue(expected.contains(intIterator.next()));
      }
    }
  }
}
 

private static Iterable<ImmutableRoaringBitmap> unwrap(
    final Iterable<ImmutableBitmap> b
)
{
  return new Iterable<ImmutableRoaringBitmap>()
  {
    @Override
    public Iterator<ImmutableRoaringBitmap> iterator()
    {
      final Iterator<ImmutableBitmap> i = b.iterator();
      return new Iterator<ImmutableRoaringBitmap>()
      {
        @Override
        public void remove()
        {
          throw new UnsupportedOperationException();
        }

        @Override
        public boolean hasNext()
        {
          return i.hasNext();
        }

        @Override
        public ImmutableRoaringBitmap next()
        {
          WrappedImmutableRoaringBitmap wrappedBitmap = (WrappedImmutableRoaringBitmap) i.next();

          if (wrappedBitmap == null) {
            return EMPTY_IMMUTABLE_BITMAP;
          }

          return wrappedBitmap.getBitmap();
        }
      };
    }
  };
}