org.apache.lucene.util.BitUtil Java Examples

The following examples show how to use org.apache.lucene.util.BitUtil. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: GeoHashUtils.java    From crate with Apache License 2.0 6 votes vote down vote up
/**
 * Encode to a morton long value from a given geohash string
 */
public static final long mortonEncode(final String hash) {
    if (hash.isEmpty()) {
        throw new IllegalArgumentException("empty geohash");
    }
    int level = 11;
    long b;
    long l = 0L;
    for (char c : hash.toCharArray()) {
        b = (long) (BASE_32_STRING.indexOf(c));
        if (b < 0) {
            throw new IllegalArgumentException("unsupported symbol [" + c + "] in geohash [" + hash + "]");
        }
        l |= (b << ((level-- * 5) + MORTON_OFFSET));
        if (level < 0) {
            // We cannot handle more than 12 levels
            break;
        }
    }
    return BitUtil.flipFlop(l);
}
 
Example #2
Source File: VersionInfo.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public VersionInfo(UpdateLog ulog, int nBuckets) {
  this.ulog = ulog;
  IndexSchema schema = ulog.uhandler.core.getLatestSchema(); 
  versionField = getAndCheckVersionField(schema);
  versionBucketLockTimeoutMs = ulog.uhandler.core.getSolrConfig().getInt("updateHandler/versionBucketLockTimeoutMs",
      Integer.parseInt(System.getProperty(SYS_PROP_BUCKET_VERSION_LOCK_TIMEOUT_MS, "0")));
  buckets = new VersionBucket[ BitUtil.nextHighestPowerOfTwo(nBuckets) ];
  for (int i=0; i<buckets.length; i++) {
    if (versionBucketLockTimeoutMs > 0) {
      buckets[i] = new TimedVersionBucket();
    } else {
      buckets[i] = new VersionBucket();
    }
  }
}
 
Example #3
Source File: StreamInput.java    From crate with Apache License 2.0 5 votes vote down vote up
public long readZLong() throws IOException {
    long accumulator = 0L;
    int i = 0;
    long currentByte;
    while (((currentByte = readByte()) & 0x80L) != 0) {
        accumulator |= (currentByte & 0x7F) << i;
        i += 7;
        if (i > 63) {
            throw new IOException("variable-length stream is too long");
        }
    }
    return BitUtil.zigZagDecode(accumulator | (currentByte << i));
}
 
Example #4
Source File: StreamOutput.java    From crate with Apache License 2.0 5 votes vote down vote up
/**
 * Writes a long in a variable-length format. Writes between one and ten bytes.
 * Values are remapped by sliding the sign bit into the lsb and then encoded as an unsigned number
 * e.g., 0 -;&gt; 0, -1 -;&gt; 1, 1 -;&gt; 2, ..., Long.MIN_VALUE -;&gt; -1, Long.MAX_VALUE -;&gt; -2
 * Numbers with small absolute value will have a small encoding
 * If the numbers are known to be non-negative, use {@link #writeVLong(long)}
 */
public void writeZLong(long i) throws IOException {
    // zig-zag encoding cf. https://developers.google.com/protocol-buffers/docs/encoding?hl=en
    long value = BitUtil.zigZagEncode(i);
    while ((value & 0xFFFFFFFFFFFFFF80L) != 0L) {
        writeByte((byte)((value & 0x7F) | 0x80));
        value >>>= 7;
    }
    writeByte((byte) (value & 0x7F));
}
 
Example #5
Source File: GeoHashUtils.java    From crate with Apache License 2.0 5 votes vote down vote up
/**
 * Encode to a morton long value from a given geohash long value
 */
public static final long mortonEncode(final long geoHashLong) {
    final int level = (int) (geoHashLong & 15);
    final short odd = (short) (level & 1);

    return BitUtil.flipFlop(((geoHashLong >>> 4) << odd) << (((12 - level) * 5) + (MORTON_OFFSET - odd)));
}
 
Example #6
Source File: FacetFieldProcessorByHashDV.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private SimpleOrderedMap<Object> calcFacets() throws IOException {

    if (sf.getType().getNumberType() != null) {
      calc = FacetRangeProcessor.getNumericCalc(sf);
    } else {
      calc = new TermOrdCalc(); // kind of a hack
    }

    // TODO: Use the number of indexed terms, if present, as an estimate!
    //    Even for NumericDocValues, we could check for a terms index for an estimate.
    //    Our estimation should aim high to avoid expensive rehashes.

    int possibleValues = fcontext.base.size();
    // size smaller tables so that no resize will be necessary
    int currHashSize = BitUtil.nextHighestPowerOfTwo((int) (possibleValues * (1 / LongCounts.LOAD_FACTOR) + 1));
    currHashSize = Math.min(currHashSize, MAXIMUM_STARTING_TABLE_SIZE);
    table = new LongCounts(currHashSize) {
      @Override
      protected void rehash() {
        super.rehash();
        doRehash(this);
        oldToNewMapping = null; // allow for gc
      }
    };

    // note: these methods/phases align with FacetFieldProcessorByArray's

    createCollectAcc();

    collectDocs();

    return super.findTopSlots(table.numSlots(), table.cardinality(),
        slotNum -> calc.bitsToValue(table.vals[slotNum]), // getBucketValFromSlotNum
        val -> calc.formatValue(val)); // getFieldQueryVal
  }
 
Example #7
Source File: ByteBuffersDataOutput.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private static int computeBlockSizeBitsFor(long bytes) {
  long powerOfTwo = BitUtil.nextHighestPowerOfTwo(bytes / MAX_BLOCKS_BEFORE_BLOCK_EXPANSION);
  if (powerOfTwo == 0) {
    return DEFAULT_MIN_BITS_PER_BLOCK;
  }
  
  int blockBits = Long.numberOfTrailingZeros(powerOfTwo);
  blockBits = Math.min(blockBits, DEFAULT_MAX_BITS_PER_BLOCK);
  blockBits = Math.max(blockBits, DEFAULT_MIN_BITS_PER_BLOCK);
  return blockBits;
}
 
Example #8
Source File: CompressingStoredFieldsWriter.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/** 
 * Writes a long in a variable-length format.  Writes between one and 
 * ten bytes. Small values or values representing timestamps with day,
 * hour or second precision typically require fewer bytes.
 * <p>
 * ZLong --&gt; Header, Bytes*?
 * <ul>
 *    <li>Header --&gt; The first two bits indicate the compression scheme:
 *       <ul>
 *          <li>00 - uncompressed
 *          <li>01 - multiple of 1000 (second)
 *          <li>10 - multiple of 3600000 (hour)
 *          <li>11 - multiple of 86400000 (day)
 *       </ul>
 *       Then the next bit is a continuation bit, indicating whether more
 *       bytes need to be read, and the last 5 bits are the lower bits of
 *       the encoded value. In order to reconstruct the value, you need to
 *       combine the 5 lower bits of the header with a vLong in the next
 *       bytes (if the continuation bit is set to 1). Then
 *       {@link BitUtil#zigZagDecode(int) zigzag-decode} it and finally
 *       multiply by the multiple corresponding to the compression scheme.
 *    <li>Bytes --&gt; Potential additional bytes to read depending on the
 *       header.
 * </ul>
 */
// T for "timestamp"
static void writeTLong(DataOutput out, long l) throws IOException {
  int header; 
  if (l % SECOND != 0) {
    header = 0;
  } else if (l % DAY == 0) {
    // timestamp with day precision
    header = DAY_ENCODING;
    l /= DAY;
  } else if (l % HOUR == 0) {
    // timestamp with hour precision, or day precision with a timezone
    header = HOUR_ENCODING;
    l /= HOUR;
  } else {
    // timestamp with second precision
    header = SECOND_ENCODING;
    l /= SECOND;
  }

  final long zigZagL = BitUtil.zigZagEncode(l);
  header |= (zigZagL & 0x1F); // last 5 bits
  final long upperBits = zigZagL >>> 5;
  if (upperBits != 0) {
    header |= 0x20;
  }
  out.writeByte((byte) header);
  if (upperBits != 0) {
    out.writeVLong(upperBits);
  }
}
 
Example #9
Source File: CompressingStoredFieldsReader.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/**
 * Reads a long in a variable-length format.  Reads between one andCorePropLo
 * nine bytes. Small values typically take fewer bytes.
 */
static long readTLong(DataInput in) throws IOException {
  int header = in.readByte() & 0xFF;

  long bits = header & 0x1F;
  if ((header & 0x20) != 0) {
    // continuation bit
    bits |= in.readVLong() << 5;
  }

  long l = BitUtil.zigZagDecode(bits);

  switch (header & DAY_ENCODING) {
    case SECOND_ENCODING:
      l *= SECOND;
      break;
    case HOUR_ENCODING:
      l *= HOUR;
      break;
    case DAY_ENCODING:
      l *= DAY;
      break;
    case 0:
      // uncompressed
      break;
    default:
      throw new AssertionError();
  }

  return l;
}
 
Example #10
Source File: Tessellator.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
protected Node(final double[] x, final double[] y, final int index, final int vertexIndex, final boolean isGeo) {
  this.idx = index;
  this.vrtxIdx = vertexIndex;
  this.polyX = x;
  this.polyY = y;
  // casting to float is safe as original values for non-geo are represented as floats
  this.y = isGeo ? encodeLatitude(polyY[vrtxIdx]) : XYEncodingUtils.encode((float) polyY[vrtxIdx]);
  this.x = isGeo ? encodeLongitude(polyX[vrtxIdx]) : XYEncodingUtils.encode((float) polyX[vrtxIdx]);
  this.morton = BitUtil.interleave(this.x ^ 0x80000000, this.y ^ 0x80000000);
  this.previous = null;
  this.next = null;
  this.previousZ = null;
  this.nextZ = null;
  this.isNextEdgeFromPolygon = true;
}
 
Example #11
Source File: GeoPoint.java    From Elasticsearch with Apache License 2.0 4 votes vote down vote up
public GeoPoint resetFromGeoHash(long geohashLong) {
    final int level = (int)(12 - (geohashLong&15));
    return this.resetFromIndexHash(BitUtil.flipFlop((geohashLong >>> 4) << ((level * 5) + 2)));
}
 
Example #12
Source File: GeoPoint.java    From crate with Apache License 2.0 4 votes vote down vote up
public GeoPoint resetFromGeoHash(long geohashLong) {
    final int level = (int) (12 - (geohashLong & 15));
    return this.resetFromIndexHash(BitUtil.flipFlop((geohashLong >>> 4) << ((level * 5) + 2)));
}
 
Example #13
Source File: GeoHashUtils.java    From crate with Apache License 2.0 4 votes vote down vote up
/**
 * Convert from a morton encoded long from a geohash encoded long
 */
public static long fromMorton(long morton, int level) {
    long mFlipped = BitUtil.flipFlop(morton);
    mFlipped >>>= (((GeoHashUtils.PRECISION - level) * 5) + MORTON_OFFSET);
    return (mFlipped << 4) | level;
}
 
Example #14
Source File: GeoHashUtils.java    From crate with Apache License 2.0 4 votes vote down vote up
/** decode longitude value from morton encoded geo point */
public static final double decodeLongitude(final long hash) {
    return unscaleLon(BitUtil.deinterleave(hash));
}
 
Example #15
Source File: GeoHashUtils.java    From crate with Apache License 2.0 4 votes vote down vote up
/** decode latitude value from morton encoded geo point */
public static final double decodeLatitude(final long hash) {
    return unscaleLat(BitUtil.deinterleave(hash >>> 1));
}
 
Example #16
Source File: Tessellator.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
/** Uses morton code for speed to determine whether or not and edge defined by a and b overlaps with a polygon edge */
private static final boolean isMortonEdgeFromPolygon(final Node a, final Node b) {
  // edge bbox (flip the bits so negative encoded values are < positive encoded values)
  final int minTX = StrictMath.min(a.x, b.x) ^ 0x80000000;
  final int minTY = StrictMath.min(a.y, b.y) ^ 0x80000000;
  final int maxTX = StrictMath.max(a.x, b.x) ^ 0x80000000;
  final int maxTY = StrictMath.max(a.y, b.y) ^ 0x80000000;

  // z-order range for the current edge;
  final long minZ = BitUtil.interleave(minTX, minTY);
  final long maxZ = BitUtil.interleave(maxTX, maxTY);

  // now make sure we don't have other points inside the potential ear;

  // look for points inside edge in both directions
  Node p = a.previousZ;
  Node n = a.nextZ;
  while (p != null && Long.compareUnsigned(p.morton, minZ) >= 0
      && n != null && Long.compareUnsigned(n.morton, maxZ) <= 0) {
    if (isPointInLine(p, p.next, a) && isPointInLine(p, p.next, b)) {
      return p.isNextEdgeFromPolygon;
    }
    if (isPointInLine(p, p.previous, a) && isPointInLine(p, p.previous, b)) {
      return p.previous.isNextEdgeFromPolygon;
    }

    p = p.previousZ;

    if (isPointInLine(n, n.next, a) && isPointInLine(n, n.next, b)) {
      return n.isNextEdgeFromPolygon;
    }
    if (isPointInLine(n, n.previous, a) && isPointInLine(n, n.previous, b)) {
      return n.previous.isNextEdgeFromPolygon;
    }

    n = n.nextZ;
  }

  // first look for points inside the edge in decreasing z-order
  while (p != null && Long.compareUnsigned(p.morton, minZ) >= 0) {
    if (isPointInLine(p, p.next, a) && isPointInLine(p, p.next, b)) {
      return p.isNextEdgeFromPolygon;
    }
    if (isPointInLine(p, p.previous, a) && isPointInLine(p, p.previous, b)) {
      return p.previous.isNextEdgeFromPolygon;
    }
    p = p.previousZ;
  }
  // then look for points in increasing z-order
  while (n != null &&
      Long.compareUnsigned(n.morton, maxZ) <= 0) {
    if (isPointInLine(n, n.next, a) && isPointInLine(n, n.next, b)) {
      return n.isNextEdgeFromPolygon;
    }
    if (isPointInLine(n, n.previous, a) && isPointInLine(n, n.previous, b)) {
      return n.previous.isNextEdgeFromPolygon;
    }
    n = n.nextZ;
  }
  return false;
}
 
Example #17
Source File: Tessellator.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
/** Uses morton code for speed to determine whether or a polygon node forms a valid ear w/ adjacent nodes */
private static final boolean mortonIsEar(final Node ear) {
  // triangle bbox (flip the bits so negative encoded values are < positive encoded values)
  int minTX = StrictMath.min(StrictMath.min(ear.previous.x, ear.x), ear.next.x) ^ 0x80000000;
  int minTY = StrictMath.min(StrictMath.min(ear.previous.y, ear.y), ear.next.y) ^ 0x80000000;
  int maxTX = StrictMath.max(StrictMath.max(ear.previous.x, ear.x), ear.next.x) ^ 0x80000000;
  int maxTY = StrictMath.max(StrictMath.max(ear.previous.y, ear.y), ear.next.y) ^ 0x80000000;

  // z-order range for the current triangle bbox;
  long minZ = BitUtil.interleave(minTX, minTY);
  long maxZ = BitUtil.interleave(maxTX, maxTY);

  // now make sure we don't have other points inside the potential ear;

  // look for points inside the triangle in both directions
  Node p = ear.previousZ;
  Node n = ear.nextZ;
  while (p != null && Long.compareUnsigned(p.morton, minZ) >= 0
      && n != null && Long.compareUnsigned(n.morton, maxZ) <= 0) {
    if (p.idx != ear.previous.idx && p.idx != ear.next.idx &&
        pointInEar(p.getX(), p.getY(), ear.previous.getX(), ear.previous.getY(), ear.getX(), ear.getY(), ear.next.getX(), ear.next.getY()) &&
        area(p.previous.getX(), p.previous.getY(), p.getX(), p.getY(), p.next.getX(), p.next.getY()) >= 0) return false;
    p = p.previousZ;

    if (n.idx != ear.previous.idx && n.idx != ear.next.idx &&
        pointInEar(n.getX(), n.getY(), ear.previous.getX(), ear.previous.getY(), ear.getX(), ear.getY(), ear.next.getX(), ear.next.getY()) &&
        area(n.previous.getX(), n.previous.getY(), n.getX(), n.getY(), n.next.getX(), n.next.getY()) >= 0) return false;
    n = n.nextZ;
  }

  // first look for points inside the triangle in decreasing z-order
  while (p != null && Long.compareUnsigned(p.morton, minZ) >= 0) {
    if (p.idx != ear.previous.idx && p.idx != ear.next.idx
          && pointInEar(p.getX(), p.getY(), ear.previous.getX(), ear.previous.getY(), ear.getX(), ear.getY(), ear.next.getX(), ear.next.getY())
          && area(p.previous.getX(), p.previous.getY(), p.getX(), p.getY(), p.next.getX(), p.next.getY()) >= 0) {
        return false;
      }
    p = p.previousZ;
  }
  // then look for points in increasing z-order
  while (n != null &&
      Long.compareUnsigned(n.morton, maxZ) <= 0) {
      if (n.idx != ear.previous.idx && n.idx != ear.next.idx
          && pointInEar(n.getX(), n.getY(), ear.previous.getX(), ear.previous.getY(), ear.getX(), ear.getY(), ear.next.getX(), ear.next.getY())
          && area(n.previous.getX(), n.previous.getY(), n.getX(), n.getY(), n.next.getX(), n.next.getY()) >= 0) {
        return false;
      }
    n = n.nextZ;
  }
  return true;
}
 
Example #18
Source File: MortonEncoder.java    From crate with Apache License 2.0 3 votes vote down vote up
/**
 * Main encoding method to quantize lat/lon points and bit interleave them into a binary morton code
 * in the range of 0x00000000... : 0xFFFFFFFF...
 *
 * @param latitude latitude value: must be within standard +/-90 coordinate bounds.
 * @param longitude longitude value: must be within standard +/-180 coordinate bounds.
 * @return bit interleaved encoded values as a 64-bit {@code long}
 * @throws IllegalArgumentException if latitude or longitude is out of bounds
 */
public static long encode(double latitude, double longitude) {
    checkLatitude(latitude);
    checkLongitude(longitude);
    // encode lat/lon flipping the sign bit so negative ints sort before positive ints
    final int latEnc = encodeLatitude(latitude) ^ 0x80000000;
    final int lonEnc = encodeLongitude(longitude) ^ 0x80000000;
    return BitUtil.interleave(lonEnc, latEnc);
}
 
Example #19
Source File: DataInput.java    From lucene-solr with Apache License 2.0 2 votes vote down vote up
/**
 * Read a {@link BitUtil#zigZagDecode(long) zig-zag}-encoded
 * {@link #readVLong() variable-length} integer. Reads between one and ten
 * bytes.
 * @see DataOutput#writeZLong(long)
 */
public long readZLong() throws IOException {
  return BitUtil.zigZagDecode(readVLong(true));
}
 
Example #20
Source File: DataOutput.java    From lucene-solr with Apache License 2.0 2 votes vote down vote up
/**
 * Write a {@link BitUtil#zigZagEncode(int) zig-zag}-encoded
 * {@link #writeVInt(int) variable-length} integer. This is typically useful
 * to write small signed ints and is equivalent to calling
 * <code>writeVInt(BitUtil.zigZagEncode(i))</code>.
 * @see DataInput#readZInt()
 */
public final void writeZInt(int i) throws IOException {
  writeVInt(BitUtil.zigZagEncode(i));
}
 
Example #21
Source File: DataOutput.java    From lucene-solr with Apache License 2.0 2 votes vote down vote up
/**
 * Write a {@link BitUtil#zigZagEncode(long) zig-zag}-encoded
 * {@link #writeVLong(long) variable-length} long. Writes between one and ten
 * bytes. This is typically useful to write small signed ints.
 * @see DataInput#readZLong()
 */
public final void writeZLong(long i) throws IOException {
  writeSignedVLong(BitUtil.zigZagEncode(i));
}
 
Example #22
Source File: DataInput.java    From lucene-solr with Apache License 2.0 2 votes vote down vote up
/**
 * Read a {@link BitUtil#zigZagDecode(int) zig-zag}-encoded
 * {@link #readVInt() variable-length} integer.
 * @see DataOutput#writeZInt(int)
 */
public int readZInt() throws IOException {
  return BitUtil.zigZagDecode(readVInt());
}