Java Code Examples for java.nio.ByteBuffer#getFloat()

@Test
public void shouldGenerateSaneBottlenecks() throws IOException {
  LiteBottleneckModel model =
      new LiteBottleneckModel(
          new AssetModelLoader(InstrumentationRegistry.getInstrumentation().getContext(), "model")
              .loadBaseModel());

  ByteBuffer image =
      ByteBuffer.allocateDirect(IMAGE_SIZE * IMAGE_SIZE * NUM_IMAGE_CHANNELS * FLOAT_BYTES);

  for (int idx = 0; idx < IMAGE_SIZE * IMAGE_SIZE * NUM_IMAGE_CHANNELS; idx++) {
    image.putFloat(IMAGE_FILL);
  }
  image.rewind();

  ByteBuffer bottleneck = model.generateBottleneck(image, null);
  int nonZeroCount = 0;
  for (int idx = 0; idx < NUM_BOTTLENECK_FEATURES; idx++) {
    float feature = bottleneck.getFloat();
    if (Math.abs(feature) > EPS) {
      nonZeroCount++;
    }
  }
  assertTrue(nonZeroCount > 0);
}
 

/**
 * Decodes a byte array into a float value.
 */
public static Number decodeFloat(byte[] value) throws TlvException {
    ByteBuffer floatBb = ByteBuffer.wrap(value);
    if (value.length == 4) {
        return floatBb.getFloat();
    } else if (value.length == 8) {
        return floatBb.getDouble();
    } else {
        throw new TlvException("Invalid length for a float value: " + value.length);
    }
}
 

public static MatrixBlock convertPy4JArrayToMB(byte[] data, int rlen, int clen, boolean isSparse,
	Types.ValueType valueType) {
	MatrixBlock mb = new MatrixBlock(rlen, clen, isSparse, -1);
	if(isSparse) {
		throw new DMLRuntimeException("Convertion to sparse format not supported");
	}
	else {
		long limit = (long) rlen * clen;
		if(limit > Integer.MAX_VALUE)
			throw new DMLRuntimeException(
				"Dense NumPy array of size " + limit + " cannot be converted to MatrixBlock");
		double[] denseBlock = new double[(int) limit];
		ByteBuffer buf = ByteBuffer.wrap(data);
		buf.order(ByteOrder.nativeOrder());
		switch(valueType) {
			case INT32:
				for(int i = 0; i < rlen * clen; i++)
					denseBlock[i] = buf.getInt();
				break;
			case FP32:
				for(int i = 0; i < rlen * clen; i++)
					denseBlock[i] = buf.getFloat();
				break;
			case FP64:
				for(int i = 0; i < rlen * clen; i++)
					denseBlock[i] = buf.getDouble();
				break;
			default:
				throw new DMLRuntimeException("Unsupported value type: " + valueType.name());
		}
		mb.init(denseBlock, rlen, clen);
	}
	mb.recomputeNonZeros();
	mb.examSparsity();
	return mb;
}
 

@Override
public void parse(ByteBuffer buf)
{
    if(this.getMode() == DataPacketMode.FORGING)
        throw new IllegalStateException("Can not parse on a Forging mode Data Packet Tree element");
    _float = buf.getFloat();
    // sets the raw bytes
    _bytes = new byte[4];
    buf.position(buf.position()-4);
    buf.get(_bytes);
}
 

@Override
public void decode(final PVATypeRegistry types, final ByteBuffer buffer) throws Exception
{
    final int size = PVASize.decodeSize(buffer);
    final float[] new_value = new float[size];
    for (int i=0; i<size; ++i)
        new_value[i] = buffer.getFloat();
    value = new_value;
}
 

private static Object internalFromByteBuffer(Type type, ByteBuffer buffer) {
  if (buffer == null) {
    return null;
  }

  ByteBuffer tmp = buffer.duplicate();
  if (type == Types.UUIDType.get() || type instanceof Types.DecimalType) {
    tmp.order(ByteOrder.BIG_ENDIAN);
  } else {
    tmp.order(ByteOrder.LITTLE_ENDIAN);
  }
  switch (type.typeId()) {
    case BOOLEAN:
      return tmp.get() != 0x00;
    case INTEGER:
    case DATE:
      return tmp.getInt();
    case LONG:
    case TIME:
    case TIMESTAMP:
      if (tmp.remaining() < 8) {
        // type was later promoted to long
        return (long) tmp.getInt();
      }
      return tmp.getLong();
    case FLOAT:
      return tmp.getFloat();
    case DOUBLE:
      if (tmp.remaining() < 8) {
        // type was later promoted to long
        return (double) tmp.getFloat();
      }
      return tmp.getDouble();
    case STRING:
      try {
        return DECODER.get().decode(tmp);
      } catch (CharacterCodingException e) {
        throw new RuntimeIOException(e, "Failed to decode value as UTF-8: " + buffer);
      }
    case UUID:
      long mostSigBits = tmp.getLong();
      long leastSigBits = tmp.getLong();
      return new UUID(mostSigBits, leastSigBits);
    case FIXED:
    case BINARY:
      return tmp;
    case DECIMAL:
      Types.DecimalType decimal = (Types.DecimalType) type;
      byte[] unscaledBytes = new byte[buffer.remaining()];
      tmp.get(unscaledBytes);
      return new BigDecimal(new BigInteger(unscaledBytes), decimal.scale());
    default:
      throw new UnsupportedOperationException("Cannot deserialize type: " + type);
  }
}
 

public static float readFloat(ByteBuffer buf) {
	return buf.getFloat();
}
 

public static float byteArrayToFloat(byte[] value, int index) {
	ByteBuffer byteBuffer = ByteBuffer.wrap(value);
	return byteBuffer.getFloat(index);
}
 

@Override
public IColumn toColumn( final ColumnBinary columnBinary ) throws IOException {
  ICellManager cellManager;

  ByteBuffer wrapBuffer = ByteBuffer.wrap(
      columnBinary.binary , columnBinary.binaryStart , columnBinary.binaryLength );
  switch ( columnBinary.columnType ) {
    case BOOLEAN:
      if ( wrapBuffer.get() == 1 ) {
        cellManager = new ConstantCellManager(
            columnBinary.columnType , new BooleanObj( true ) , columnBinary.rowCount );
      } else {
        cellManager = new ConstantCellManager(
            columnBinary.columnType , new BooleanObj( false ) , columnBinary.rowCount );
      }
      break;
    case BYTE:
      byte byteValue = wrapBuffer.get();
      cellManager = new ConstantCellManager(
          columnBinary.columnType , new ByteObj( byteValue ) , columnBinary.rowCount );
      cellManager.setIndex( new RangeByteIndex( byteValue , byteValue ) );
      break;
    case SHORT:
      short shortValue = wrapBuffer.getShort();
      cellManager = new ConstantCellManager(
          columnBinary.columnType , new ShortObj( shortValue ) , columnBinary.rowCount );
      cellManager.setIndex( new RangeShortIndex( shortValue , shortValue ) );
      break;
    case INTEGER:
      int intValue = wrapBuffer.getInt();
      cellManager = new ConstantCellManager(
          columnBinary.columnType , new IntegerObj( intValue ) , columnBinary.rowCount );
      cellManager.setIndex( new RangeIntegerIndex( intValue , intValue ) );
      break;
    case LONG:
      long longValue = wrapBuffer.getLong();
      cellManager = new ConstantCellManager(
          columnBinary.columnType , new LongObj( longValue ) , columnBinary.rowCount );
      cellManager.setIndex( new RangeLongIndex( longValue , longValue ) );
      break;
    case FLOAT:
      float floatValue = wrapBuffer.getFloat();
      cellManager = new ConstantCellManager(
          columnBinary.columnType ,
          new FloatObj( floatValue ) ,
          columnBinary.rowCount );
      cellManager.setIndex( new RangeFloatIndex( floatValue , floatValue ) );
      break;
    case DOUBLE:
      double doubleValue = wrapBuffer.getDouble();
      cellManager = new ConstantCellManager(
          columnBinary.columnType ,
          new DoubleObj( doubleValue ) ,
          columnBinary.rowCount );
      cellManager.setIndex( new RangeDoubleIndex( doubleValue , doubleValue ) );
      break;
    case STRING:
      int stringLength = wrapBuffer.getInt();
      byte[] stringBytes = new byte[stringLength];
      wrapBuffer.get( stringBytes );
      cellManager = new ConstantCellManager(
          columnBinary.columnType ,
          new Utf8BytesLinkObj( stringBytes , 0 , stringBytes.length ) ,
          columnBinary.rowCount );
      String string = new String( stringBytes , 0 , stringBytes.length , "UTF-8" );
      cellManager.setIndex( new RangeStringIndex( string , string ) );
      break;
    case BYTES:
      int byteLength = wrapBuffer.getInt();
      byte[] byteBytes = new byte[byteLength];
      wrapBuffer.get( byteBytes );
      cellManager = new ConstantCellManager(
          columnBinary.columnType ,
          new BytesObj( byteBytes , 0 , byteBytes.length ) ,
          columnBinary.rowCount );
      break;
    default:
      throw new IOException( "Unknown primitive type." );
  }
  IColumn result = new PrimitiveColumn( columnBinary.columnType , columnBinary.columnName );
  result.setCellManager( cellManager );
  return result;
}
 

/**
 * Reads a binary STL file.
 *
 * @param is InputStream to read from.
 * @param skip Number of bytes to skip before reading the STL file.
 * @return Mesh
 * @throws IOException If an error occurs during reading.
 */
private Mesh readBinary(InputStream is, int skip) throws IOException {
    /* Prepare a ByteBuffer to read the rest of the STL file. */
    byte[] bytes = new byte[50];
    ByteBuffer buffer = ByteBuffer.wrap(bytes);
    buffer.order(ByteOrder.LITTLE_ENDIAN);

    /* Skip the STL header! */
    is.skip(skip);

    /* Read the bytes for the size (unsigned 32 bit int, little-endian). */
    byte[] sizeBytes = new byte[4];
    is.read(sizeBytes, 0, 4);
    long triangles = ((sizeBytes[0] & 0xFF)) | ((sizeBytes[1] & 0xFF) << 8) | ((sizeBytes[2] & 0xFF) << 16) | ((sizeBytes[3] & 0xFF) << 24);

    /* TODO: Properly handle models whose triangles > MAX_TRIANGLES. */
    if (triangles <= 0) {
        LOGGER.error("The number of triangles in the Mesh seems to be smaller than zero. This STL file is probably corrupt!");
        return null;
    } else if (triangles > MAX_TRIANGLES) {
        LOGGER.error("The number of triangles in the Mesh exceeds the limit that can currently be processed by STLMeshDecoder. The Mesh will be downsampled!");
        return null;
    }

    /* Prepare Mesh. */
    Mesh mesh = new Mesh((int)triangles, (int)triangles);

    /* Prepare helper structures. */
    TObjectIntHashMap<Vector3f> vertexBuffer = new TObjectIntHashMap<>();
    int index = 0;
    int[] vertexindices = new int[3];

    /* Now add all triangles. */
    for (int i=0; i<triangles; i++) {
        /* Ready 48 bytes from the stream. */
        buffer.rewind();
        is.read(bytes);

        /* Read and ignore three floats. */
        buffer.getFloat();
        buffer.getFloat();
        buffer.getFloat();

        /* Add the vertices and the vertex-normal to the mesh. */
        for (int vidx = 0; vidx < 3; vidx++) {
            Vector3f vertex = new Vector3f(buffer.getFloat(), buffer.getFloat(), buffer.getFloat());
            if (!vertexBuffer.containsKey(vertex)) {
                mesh.addVertex(vertex);
                vertexBuffer.put(vertex, index);
                index++;
            }
            vertexindices[vidx] = vertexBuffer.get(vertex);
        }

        /* Add a new face to the Mesh. */
        if (!mesh.addFace(new Vector3i(vertexindices[0], vertexindices[1], vertexindices[2]))) {
            LOGGER.warn("Could not add face {}/{}/{} because index points to non-existing vertex.", vertexindices[0], vertexindices[1], vertexindices[2]);
        }
    }

    /* Closes the InputStream. */
    is.close();
    return mesh;
}
 

public synchronized float[] getZScores(int snp) throws IOException {
        long snpBytePos = snpBytes[snp];

        long snpByteNextPos = 0;
        if (snp == snpBytes.length - 1) {
            snpByteNextPos = raf.length();
        } else {
            snpByteNextPos = snpBytes[snp + 1];
        }

        int readlen = (int) (snpByteNextPos - snpBytePos);

//		System.out.println("snp pos: " + snpBytePos);

        if (mappedzscorehandle == null || snpBytePos > currentmapend || snpByteNextPos > currentmapend || snpBytePos < currentmapstart) {

            long nrBytesPerBuffer = 1048576 * 10; // 256 mb

            if (snpBytePos + nrBytesPerBuffer > raf.length()) {
                nrBytesPerBuffer = raf.length() - snpBytePos;
            }
            mappedzscorehandle = raf.getChannel().map(FileChannel.MapMode.READ_ONLY, snpBytePos, nrBytesPerBuffer);
            mappedzscorehandle.load();
            bZs = new byte[(int) nrBytesPerBuffer];
            mappedzscorehandle.get(bZs);

            currentmapstart = snpBytePos;
            currentmapend = currentmapstart + nrBytesPerBuffer;
            System.out.println("New buffer: " + bZs.length + "\tsta: " + currentmapstart + "\tsto: " + currentmapend +
                    "\tlen: " + (currentmapend - currentmapstart) + "\tnrbytes: " + nrBytesPerBuffer);
        }

        int offset = (int) (snpBytePos - currentmapstart);
//		System.out.println("byte pos: " + snpBytePos + "\toffset " + offset + "\tlen " + readlen);
        byte[] bytesToRead = new byte[readlen];

        System.arraycopy(bZs, offset, bytesToRead, 0, readlen);
//
////
////        if (snpCisProbeMap != null) {
////            nrZ = snpCisProbeMap[snp].length;
////        }
////        System.out.println(snp + "\t" + snpBytePos + "\t" + snpByteNextPos);
//		raf.seek(snpBytePos);
//

//		raf.read(bytesToRead);
        ByteBuffer bytebuffer = ByteBuffer.wrap(bytesToRead);
        float[] output = new float[readlen / 4];
        for (int i = 0; i < output.length; i++) {
            output[i] = bytebuffer.getFloat();
        }
        
        
        return output;
    }
 

@Override
StatValue build(ByteBuffer value) {
  return new Value(value.getFloat(0));
}
 

@Override
public void decode(final PVATypeRegistry types, final ByteBuffer buffer) throws Exception
{
    value = buffer.getFloat();
}
 

@Test
public void shouldCalculateGradientsCorrectly() throws IOException {
  LiteTrainHeadModel model =
      new LiteTrainHeadModel(
          new AssetModelLoader(InstrumentationRegistry.getInstrumentation().getContext(), "model")
              .loadTrainModel());

  ByteBuffer bottlenecks = generateRandomByteBuffer(
      BATCH_SIZE * BOTTLENECK_SIZE, random::nextFloat);
  ByteBuffer classes = generateRandomByteBuffer(BATCH_SIZE * NUM_CLASSES, () -> 0.f);

  int[][] parameterShapes = model.getParameterShapes();
  ByteBuffer[] parameters = new ByteBuffer[parameterShapes.length];
  for (int parameterIdx = 0; parameterIdx < parameterShapes.length; parameterIdx++) {
    parameters[parameterIdx] = generateRandomByteBuffer(parameterShapes[parameterIdx]);
  }

  // Fill with one-hot.
  for (int sampleIdx = 0; sampleIdx < BATCH_SIZE; sampleIdx++) {
    int sampleClass = random.nextInt(NUM_CLASSES);
    classes.putFloat((sampleIdx * NUM_CLASSES + sampleClass) * FLOAT_BYTES, 1);
  }

  ByteBuffer[] gradients = new ByteBuffer[parameterShapes.length];
  for (int parameterIdx = 0; parameterIdx < parameterShapes.length; parameterIdx++) {
    gradients[parameterIdx] = generateRandomByteBuffer(parameterShapes[parameterIdx]);
  }

  float loss = model.calculateGradients(bottlenecks, classes, parameters, gradients);

  for (int parameterIdx = 0; parameterIdx < parameters.length; parameterIdx++) {
    ByteBuffer parameter = parameters[parameterIdx];
    ByteBuffer analyticalGrads = gradients[parameterIdx];
    int numElementsToCheck =
        Math.min(product(parameterShapes[parameterIdx]), MAX_ELEMENTS_TO_CHECK);

    for (int elementIdx = 0; elementIdx < numElementsToCheck; elementIdx++) {
      float analyticalGrad = analyticalGrads.getFloat(elementIdx * FLOAT_BYTES);

      float originalParam = parameter.getFloat(elementIdx * FLOAT_BYTES);
      parameter.putFloat(elementIdx * FLOAT_BYTES, originalParam + DELTA_PARAM);
      float newLoss = model.calculateGradients(bottlenecks, classes, parameters, gradients);

      float numericalGrad = (newLoss - loss) / DELTA_PARAM;
      assertTrue(
          String.format("Numerical gradient %.5f is different from analytical %.5f "
              + "(loss = %.5f -> %.5f)",
              numericalGrad, analyticalGrad, loss, newLoss),
          Math.abs(numericalGrad - analyticalGrad) < EPS);

      parameter.putFloat(elementIdx * FLOAT_BYTES, originalParam);
    }
  }
}
 

@Override
public void loadInMemoryStorage( final ColumnBinary columnBinary , final IMemoryAllocator allocator ) throws IOException{
  ByteBuffer wrapBuffer = ByteBuffer.wrap( columnBinary.binary , columnBinary.binaryStart , columnBinary.binaryLength );
  switch( columnBinary.columnType ){
    case BOOLEAN:
      boolean booleanObj = wrapBuffer.get() == 1;
      for( int i = 0 ; i < columnBinary.rowCount ; i++ ){
        allocator.setBoolean( i , booleanObj );
      }
      break;
    case BYTE:
      byte byteObj = wrapBuffer.get();
      for( int i = 0 ; i < columnBinary.rowCount ; i++ ){
        allocator.setByte( i , byteObj );
      }
      break;
    case SHORT:
      short shortObj = wrapBuffer.getShort();
      for( int i = 0 ; i < columnBinary.rowCount ; i++ ){
        allocator.setShort( i , shortObj );
      }
      break;
    case INTEGER:
      int intObj = wrapBuffer.getInt();
      for( int i = 0 ; i < columnBinary.rowCount ; i++ ){
        allocator.setInteger( i , intObj );
      }
      break;
    case LONG:
      long longObj = wrapBuffer.getLong();
      for( int i = 0 ; i < columnBinary.rowCount ; i++ ){
        allocator.setLong( i , longObj );
      }
      break;
    case FLOAT:
      float floatObj = wrapBuffer.getFloat();
      for( int i = 0 ; i < columnBinary.rowCount ; i++ ){
        allocator.setFloat( i , floatObj );
      }
      break;
    case DOUBLE:
      double doubleObj = wrapBuffer.getDouble();
      for( int i = 0 ; i < columnBinary.rowCount ; i++ ){
        allocator.setDouble( i , doubleObj );
      }
      break;
    case STRING:
      int stringLength = wrapBuffer.getInt();
      byte[] stringBytes = new byte[stringLength];
      wrapBuffer.get( stringBytes );
      String utf8 = new String( stringBytes , "UTF-8" );
      for( int i = 0 ; i < columnBinary.rowCount ; i++ ){
        allocator.setString( i , utf8 );
      }
      break;
    case BYTES:
      int byteLength = wrapBuffer.getInt();
      byte[] byteBytes = new byte[byteLength];
      wrapBuffer.get( byteBytes );
      for( int i = 0 ; i < columnBinary.rowCount ; i++ ){
        allocator.setBytes( i , byteBytes );
      }
      break;
    default:
      throw new IOException( "Unknown primitive type." );
  }
  allocator.setValueCount( columnBinary.rowCount );
}
 

@Override
protected void parsePayload(ByteBuffer buffer) {
  this.offsetFromRoadCenter = buffer.getFloat();
  this.laneChangeId = Byte.toUnsignedInt(buffer.get());
}
 

public static void fromBytes(byte[] data, float[] preAllocatedValues) {
    final ByteBuffer byteBuffer = ByteBuffer.wrap(data);
    for (int i = 0; i<preAllocatedValues.length; i++) { preAllocatedValues[i] = byteBuffer.getFloat(); }
}
 

@Override
public void readFromBufferAndSet(ByteBuffer byteBuffer, float[] array, int index) {
  array[index] = byteBuffer.getFloat();
}
 

@Override
public void setFromBinary( final byte[] buffer , final int start , final int length ){
  ByteBuffer wrapBuffer = ByteBuffer.wrap( buffer );
  min = wrapBuffer.getFloat();
  max = wrapBuffer.getFloat();
}
 

private static Object internalFromByteBuffer(Type type, ByteBuffer buffer) {
  ByteBuffer tmp = buffer.duplicate().order(ByteOrder.LITTLE_ENDIAN);
  switch (type.typeId()) {
    case BOOLEAN:
      return (tmp.get() != 0x00);
    case INTEGER:
    case DATE:
      return tmp.getInt();
    case LONG:
    case TIME:
    case TIMESTAMP:
      if (tmp.remaining() < 8) {
        // type was later promoted to long
        return (long) tmp.getInt();
      }
      return tmp.getLong();
    case FLOAT:
      return tmp.getFloat();
    case DOUBLE:
      if (tmp.remaining() < 8) {
        // type was later promoted to long
        return (double) tmp.getFloat();
      }
      return tmp.getDouble();
    case STRING:
      try {
        return DECODER.get().decode(tmp);
      } catch (CharacterCodingException e) {
        throw new RuntimeIOException(e, "Failed to decode value as UTF-8: " + buffer);
      }
    case UUID:
      long mostSigBits = tmp.getLong();
      long leastSigBits = tmp.getLong();
      return new UUID(mostSigBits, leastSigBits);
    case FIXED:
    case BINARY:
      return tmp;
    case DECIMAL:
      Types.DecimalType decimal = (Types.DecimalType) type;
      byte[] unscaledBytes = new byte[buffer.remaining()];
      tmp.get(unscaledBytes);
      return new BigDecimal(new BigInteger(unscaledBytes), decimal.scale());
    default:
      throw new UnsupportedOperationException("Cannot deserialize type: " + type);
  }
}