Java Code Examples for org.apache.arrow.vector.complex.StructVector#allocateNew()

The following examples show how to use org.apache.arrow.vector.complex.StructVector#allocateNew() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: ArrowFixedSchemaStructMemoryAllocator.java    From yosegi with Apache License 2.0 6 votes vote down vote up 
/**
 * Set the vector of Struct and initialize it.
 */
public ArrowFixedSchemaStructMemoryAllocator(
    final StructContainerField schema ,
    final BufferAllocator allocator ,
    final StructVector vector ,
    final int rowCount ) throws IOException {
  this.vector = vector;
  vector.allocateNew();

  loaderMap = new HashMap<String,IMemoryAllocator>();
  for ( String key : schema.getKeys() ) {
    IField childSchema = schema.get( key );
    loaderMap.put( key , ArrowFixedSchemaMemoryAllocatorFactory.getFromStructVector(
        childSchema , key , allocator , vector , rowCount ) );
  }
}
Example 2
Source File: TestArrowBytesMemoryAllocator.java    From multiple-dimension-spread with Apache License 2.0 5 votes vote down vote up 
@Test
public void T_setBytes_1() throws IOException{
  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.BYTES , "target" , allocator , parent , 4 );

  memoryAllocator.setBytes( 0 , "a".getBytes() );
  memoryAllocator.setBytes( 1 , "b".getBytes() );
  memoryAllocator.setBytes( 5 , "c".getBytes() );
  memoryAllocator.setBytes( 1000 , "a b c".getBytes() );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( new String( reader.readByteArray() ) , "a" );
  reader.setPosition( 1 );
  assertEquals( new String( reader.readByteArray() ) , "b" );
  reader.setPosition( 5 );
  assertEquals( new String( reader.readByteArray() ) , "c" );
  for( int i = 6 ; i < 1000 ; i++ ){
    reader.setPosition( i );
    assertEquals( reader.readByteArray() , null );
  }
  reader.setPosition( 1000 );
  assertEquals( new String( reader.readByteArray() ) , "a b c" );
}
Example 3
Source File: ArrowFixedSchemaMapMemoryAllocator.java    From yosegi with Apache License 2.0 5 votes vote down vote up 
/**
 * Set the vector of Map and initialize it.
 */
public ArrowFixedSchemaMapMemoryAllocator(
    final MapContainerField schema ,
    final BufferAllocator allocator ,
    final StructVector vector , final int rowCount ) {
  this.allocator = allocator;
  this.vector = vector;
  this.rowCount = rowCount;
  vector.allocateNew();
  childSchema = schema.getField();
}
Example 4
Source File: TestArrowIntegerMemoryAllocator.java    From multiple-dimension-spread with Apache License 2.0 5 votes vote down vote up 
@Test
public void T_setInteger_1() throws IOException{
  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.INTEGER , "target" , allocator , parent , 1001 );

  memoryAllocator.setInteger( 0 , 100 );
  memoryAllocator.setInteger( 1 , 200 );
  memoryAllocator.setInteger( 5 , 255 );
  memoryAllocator.setInteger( 1000 , 10 );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readInteger().intValue() , 100 );
  reader.setPosition( 1 );
  assertEquals( reader.readInteger().intValue() , 200 );
  reader.setPosition( 5 );
  assertEquals( reader.readInteger().intValue() , 255 );
  for( int i = 6 ; i < 1000 ; i++ ){
    reader.setPosition( i );
    assertEquals( reader.readInteger() , null );
  }
  reader.setPosition( 1000 );
  assertEquals( reader.readInteger().intValue() , 10 );
}
Example 5
Source File: TestArrowArrayMemoryAllocator.java    From multiple-dimension-spread with Apache License 2.0 5 votes vote down vote up 
@Test
public void T_setArray_1() throws IOException{
  IColumn column = new ArrayColumn( "array" );
  List<Object> value = new ArrayList<Object>();
  value.add( new StringObj( "a" ) );
  value.add( new StringObj( "b" ) );
  value.add( new StringObj( "c" ) );
  column.add( ColumnType.ARRAY , value , 0 );
  column.add( ColumnType.ARRAY , value , 1 );
  column.add( ColumnType.ARRAY , value , 2 );
  column.add( ColumnType.ARRAY , value , 3 );
  column.add( ColumnType.ARRAY , value , 6 );

  ColumnBinaryMakerConfig defaultConfig = new ColumnBinaryMakerConfig();
  ColumnBinaryMakerCustomConfigNode configNode = new ColumnBinaryMakerCustomConfigNode( "root" , defaultConfig );

  IColumnBinaryMaker maker = new DumpArrayColumnBinaryMaker();
  ColumnBinary columnBinary = maker.toBinary( defaultConfig , null , column );

  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();

  ListVector listVector = parent.addOrGetList( "target" );
  IMemoryAllocator memoryAllocator = new ArrowArrayMemoryAllocator( allocator , listVector , 7 );
  maker.loadInMemoryStorage( columnBinary , memoryAllocator );

  for( int i = 0 ; i < 7 ; i++ ){
    System.out.println( "count:" + listVector.getInnerValueCountAt(i) );
    System.out.println( "obj:" + listVector.getObject(i) );
  }
}
Example 6
Source File: TestArrowIntegerMemoryAllocator.java    From yosegi with Apache License 2.0 5 votes vote down vote up 
@Test
public void T_setInteger_2() throws IOException{
  IColumn column = new PrimitiveColumn( ColumnType.INTEGER , "boolean" );
  column.add( ColumnType.INTEGER , new IntegerObj( 100 ) , 0 );
  column.add( ColumnType.INTEGER , new IntegerObj( 200 ) , 1 );
  column.add( ColumnType.INTEGER , new IntegerObj( 255 ) , 5 );

  ColumnBinaryMakerConfig defaultConfig = new ColumnBinaryMakerConfig();
  ColumnBinaryMakerCustomConfigNode configNode = new ColumnBinaryMakerCustomConfigNode( "root" , defaultConfig );

  IColumnBinaryMaker maker = new UnsafeOptimizeLongColumnBinaryMaker();
  ColumnBinary columnBinary = maker.toBinary( defaultConfig , null , new CompressResultNode() , column );

  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.INTEGER , "target" , allocator , parent , 6 );

  maker.loadInMemoryStorage( columnBinary , memoryAllocator );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readInteger().intValue() , 100 );
  reader.setPosition( 1 );
  assertEquals( reader.readInteger().intValue() , 200 );
  reader.setPosition( 5 );
  assertEquals( reader.readInteger().intValue() , 255 );
  reader.setPosition( 2 );
  assertEquals( reader.readInteger() , null );
  reader.setPosition( 3 );
  assertEquals( reader.readInteger() , null );
  reader.setPosition( 4 );
  assertEquals( reader.readInteger() , null );
}
Example 7
Source File: TestArrowIntegerMemoryAllocator.java    From yosegi with Apache License 2.0 5 votes vote down vote up 
@Test
public void T_setInteger_1() throws IOException{
  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.INTEGER , "target" , allocator , parent , 1001 );

  memoryAllocator.setInteger( 0 , 100 );
  memoryAllocator.setInteger( 1 , 200 );
  memoryAllocator.setInteger( 5 , 255 );
  memoryAllocator.setInteger( 1000 , 10 );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readInteger().intValue() , 100 );
  reader.setPosition( 1 );
  assertEquals( reader.readInteger().intValue() , 200 );
  reader.setPosition( 5 );
  assertEquals( reader.readInteger().intValue() , 255 );
  for( int i = 6 ; i < 1000 ; i++ ){
    reader.setPosition( i );
    assertEquals( reader.readInteger() , null );
  }
  reader.setPosition( 1000 );
  assertEquals( reader.readInteger().intValue() , 10 );
}
Example 8
Source File: TestArrowLongMemoryAllocator.java    From multiple-dimension-spread with Apache License 2.0 5 votes vote down vote up 
@Test
public void T_setLong_2() throws IOException{
  IColumn column = new PrimitiveColumn( ColumnType.LONG , "boolean" );
  column.add( ColumnType.LONG , new LongObj( (long)100 ) , 0 );
  column.add( ColumnType.LONG , new LongObj( (long)200 ) , 1 );
  column.add( ColumnType.LONG , new LongObj( (long)255 ) , 5 );

  ColumnBinaryMakerConfig defaultConfig = new ColumnBinaryMakerConfig();
  ColumnBinaryMakerCustomConfigNode configNode = new ColumnBinaryMakerCustomConfigNode( "root" , defaultConfig );

  IColumnBinaryMaker maker = new OptimizeLongColumnBinaryMaker();
  ColumnBinary columnBinary = maker.toBinary( defaultConfig , null , column );

  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.LONG , "target" , allocator , parent , 3 );

  maker.loadInMemoryStorage( columnBinary , memoryAllocator );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readLong().longValue() , (long)100 );
  reader.setPosition( 1 );
  assertEquals( reader.readLong().longValue() , (long)200 );
  reader.setPosition( 5 );
  assertEquals( reader.readLong().longValue() , (long)255 );
  reader.setPosition( 2 );
  assertEquals( reader.readLong() , null );
  reader.setPosition( 3 );
  assertEquals( reader.readLong() , null );
  reader.setPosition( 4 );
  assertEquals( reader.readLong() , null );
}
Example 9
Source File: TestArrowStringMemoryAllocator.java    From multiple-dimension-spread with Apache License 2.0 5 votes vote down vote up 
@Test
public void T_setString_1() throws IOException{
  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.STRING , "target" , allocator , parent , 4 );

  memoryAllocator.setString( 0 , "a" );
  memoryAllocator.setString( 1 , "b" );
  memoryAllocator.setString( 5 , "c" );
  memoryAllocator.setString( 1000 , "a b c" );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readText().toString() , "a" );
  reader.setPosition( 1 );
  assertEquals( reader.readText().toString() , "b" );
  reader.setPosition( 5 );
  assertEquals( reader.readText().toString() , "c" );
  for( int i = 6 ; i < 1000 ; i++ ){
    reader.setPosition( i );
    assertEquals( reader.readText() , null );
  }
  reader.setPosition( 1000 );
  assertEquals( reader.readText().toString() , "a b c" );
}
Example 10
Source File: TestArrowBytesMemoryAllocator.java    From yosegi with Apache License 2.0 5 votes vote down vote up 
@Test
public void T_setBytes_2() throws IOException{
  IColumn column = new PrimitiveColumn( ColumnType.STRING , "boolean" );
  column.add( ColumnType.STRING , new BytesObj( "a".getBytes() ) , 0 );
  column.add( ColumnType.STRING , new BytesObj( "b".getBytes() ) , 1 );
  column.add( ColumnType.STRING , new BytesObj( "c".getBytes() ) , 5 );

  ColumnBinaryMakerConfig defaultConfig = new ColumnBinaryMakerConfig();
  ColumnBinaryMakerCustomConfigNode configNode = new ColumnBinaryMakerCustomConfigNode( "root" , defaultConfig );

  IColumnBinaryMaker maker = new DumpBytesColumnBinaryMaker();
  ColumnBinary columnBinary = maker.toBinary( defaultConfig , null , new CompressResultNode() , column );

  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.BYTES , "target" , allocator , parent , 3 );
  maker.loadInMemoryStorage( columnBinary , memoryAllocator );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( new String( reader.readByteArray() ) , "a" );
  reader.setPosition( 1 );
  assertEquals( new String( reader.readByteArray() ) , "b" );
  reader.setPosition( 5 );
  assertEquals( new String( reader.readByteArray() ) , "c" );
  reader.setPosition( 2 );
  assertEquals( reader.readByteArray() , null );
  reader.setPosition( 3 );
  assertEquals( reader.readByteArray() , null );
  reader.setPosition( 4 );
  assertEquals( reader.readByteArray() , null );
}
Example 11
Source File: TestArrowByteMemoryAllocator.java    From multiple-dimension-spread with Apache License 2.0 5 votes vote down vote up 
@Test
public void T_setByte_2() throws IOException{
  IColumn column = new PrimitiveColumn( ColumnType.BYTE , "boolean" );
  column.add( ColumnType.BYTE , new ByteObj( (byte)100 ) , 0 );
  column.add( ColumnType.BYTE , new ByteObj( (byte)200 ) , 1 );
  column.add( ColumnType.BYTE , new ByteObj( (byte)255 ) , 5 );

  ColumnBinaryMakerConfig defaultConfig = new ColumnBinaryMakerConfig();
  ColumnBinaryMakerCustomConfigNode configNode = new ColumnBinaryMakerCustomConfigNode( "root" , defaultConfig );

  IColumnBinaryMaker maker = new OptimizeLongColumnBinaryMaker();
  ColumnBinary columnBinary = maker.toBinary( defaultConfig , null , column );

  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.BYTE , "target" , allocator , parent , 6 );

  maker.loadInMemoryStorage( columnBinary , memoryAllocator );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readByte().byteValue() , (byte)100 );
  reader.setPosition( 1 );
  assertEquals( reader.readByte().byteValue() , (byte)200 );
  reader.setPosition( 5 );
  assertEquals( reader.readByte().byteValue() , (byte)255 );
  reader.setPosition( 2 );
  assertEquals( reader.readByte() , null );
  reader.setPosition( 3 );
  assertEquals( reader.readByte() , null );
  reader.setPosition( 4 );
  assertEquals( reader.readByte() , null );
}
Example 12
Source File: TestArrowStringMemoryAllocator.java    From yosegi with Apache License 2.0 5 votes vote down vote up 
@Test
public void T_setString_2() throws IOException{
  IColumn column = new PrimitiveColumn( ColumnType.STRING , "boolean" );
  column.add( ColumnType.STRING , new StringObj( "a" ) , 0 );
  column.add( ColumnType.STRING , new StringObj( "b" ) , 1 );
  column.add( ColumnType.STRING , new StringObj( "c" ) , 5 );

  ColumnBinaryMakerConfig defaultConfig = new ColumnBinaryMakerConfig();
  ColumnBinaryMakerCustomConfigNode configNode = new ColumnBinaryMakerCustomConfigNode( "root" , defaultConfig );

  IColumnBinaryMaker maker = new UnsafeOptimizeDumpStringColumnBinaryMaker();
  ColumnBinary columnBinary = maker.toBinary( defaultConfig , null , new CompressResultNode() , column );

  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.STRING , "target" , allocator , parent , 3 );

  maker.loadInMemoryStorage( columnBinary , memoryAllocator );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readText().toString() , "a" );
  reader.setPosition( 1 );
  assertEquals( reader.readText().toString() , "b" );
  reader.setPosition( 5 );
  assertEquals( reader.readText().toString() , "c" );
  reader.setPosition( 2 );
  assertEquals( reader.readText() , null );
  reader.setPosition( 3 );
  assertEquals( reader.readText() , null );
  reader.setPosition( 4 );
  assertEquals( reader.readText() , null );
}
Example 13
Source File: TestArrowByteMemoryAllocator.java    From yosegi with Apache License 2.0 5 votes vote down vote up 
@Test
public void T_setByte_2() throws IOException{
  IColumn column = new PrimitiveColumn( ColumnType.BYTE , "boolean" );
  column.add( ColumnType.BYTE , new ByteObj( (byte)100 ) , 0 );
  column.add( ColumnType.BYTE , new ByteObj( (byte)200 ) , 1 );
  column.add( ColumnType.BYTE , new ByteObj( (byte)255 ) , 5 );

  ColumnBinaryMakerConfig defaultConfig = new ColumnBinaryMakerConfig();
  ColumnBinaryMakerCustomConfigNode configNode = new ColumnBinaryMakerCustomConfigNode( "root" , defaultConfig );

  IColumnBinaryMaker maker = new UnsafeOptimizeLongColumnBinaryMaker();
  ColumnBinary columnBinary = maker.toBinary( defaultConfig , null , new CompressResultNode() , column );

  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.BYTE , "target" , allocator , parent , 6 );

  maker.loadInMemoryStorage( columnBinary , memoryAllocator );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readByte().byteValue() , (byte)100 );
  reader.setPosition( 1 );
  assertEquals( reader.readByte().byteValue() , (byte)200 );
  reader.setPosition( 5 );
  assertEquals( reader.readByte().byteValue() , (byte)255 );
  reader.setPosition( 2 );
  assertEquals( reader.readByte() , null );
  reader.setPosition( 3 );
  assertEquals( reader.readByte() , null );
  reader.setPosition( 4 );
  assertEquals( reader.readByte() , null );
}
Example 14
Source File: TestArrowBooleanMemoryAllocator.java    From yosegi with Apache License 2.0 5 votes vote down vote up 
@Test
public void T_setBoolean_2() throws IOException{
  IColumn column = new PrimitiveColumn( ColumnType.BOOLEAN , "boolean" );
  column.add( ColumnType.BOOLEAN , new BooleanObj( true ) , 0 );
  column.add( ColumnType.BOOLEAN , new BooleanObj( false ) , 1 );
  column.add( ColumnType.BOOLEAN , new BooleanObj( true ) , 5 );

  ColumnBinaryMakerConfig defaultConfig = new ColumnBinaryMakerConfig();
  ColumnBinaryMakerCustomConfigNode configNode = new ColumnBinaryMakerCustomConfigNode( "root" , defaultConfig );

  IColumnBinaryMaker maker = new DumpBooleanColumnBinaryMaker();
  ColumnBinary columnBinary = maker.toBinary( defaultConfig , null , new CompressResultNode() , column );

  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.BOOLEAN , "target" , allocator , parent , 3 );
  maker.loadInMemoryStorage( columnBinary , memoryAllocator );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readBoolean().booleanValue() , true );
  reader.setPosition( 1 );
  assertEquals( reader.readBoolean().booleanValue() , false );
  reader.setPosition( 5 );
  assertEquals( reader.readBoolean().booleanValue() , true );
  reader.setPosition( 2 );
  assertEquals( reader.readBoolean() , null );
  reader.setPosition( 3 );
  assertEquals( reader.readBoolean() , null );
  reader.setPosition( 4 );
  assertEquals( reader.readBoolean() , null );
}
Example 15
Source File: TestArrowBooleanMemoryAllocator.java    From yosegi with Apache License 2.0 5 votes vote down vote up 
@Test
public void T_setBoolean_1() throws IOException{
  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.BOOLEAN , "target" , allocator , parent , 4 );

  memoryAllocator.setBoolean( 0 , true );
  memoryAllocator.setBoolean( 1 , false );
  memoryAllocator.setBoolean( 5 , true );
  memoryAllocator.setBoolean( 1000 , true );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readBoolean().booleanValue() , true );
  reader.setPosition( 1 );
  assertEquals( reader.readBoolean().booleanValue() , false );
  reader.setPosition( 5 );
  assertEquals( reader.readBoolean().booleanValue() , true );
  for( int i = 6 ; i < 1000 ; i++ ){
    reader.setPosition( i );
    assertEquals( reader.readBoolean() , null );
  }
  reader.setPosition( 1000 );
  assertEquals( reader.readBoolean().booleanValue() , true );
}
Example 16
Source File: TestArrowDoubleMemoryAllocator.java    From yosegi with Apache License 2.0 5 votes vote down vote up 
@Test
public void T_setDouble_2() throws IOException{
  IColumn column = new PrimitiveColumn( ColumnType.DOUBLE , "boolean" );
  column.add( ColumnType.DOUBLE , new DoubleObj( (double)100 ) , 0 );
  column.add( ColumnType.DOUBLE , new DoubleObj( (double)200 ) , 1 );
  column.add( ColumnType.DOUBLE , new DoubleObj( (double)255 ) , 5 );

  ColumnBinaryMakerConfig defaultConfig = new ColumnBinaryMakerConfig();
  ColumnBinaryMakerCustomConfigNode configNode = new ColumnBinaryMakerCustomConfigNode( "root" , defaultConfig );

  IColumnBinaryMaker maker = new UnsafeOptimizeDoubleColumnBinaryMaker();
  ColumnBinary columnBinary = maker.toBinary( defaultConfig , null , new CompressResultNode() , column );

  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.DOUBLE , "target" , allocator , parent , 3 );

  maker.loadInMemoryStorage( columnBinary , memoryAllocator );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readDouble().doubleValue() , (double)100 );
  reader.setPosition( 1 );
  assertEquals( reader.readDouble().doubleValue() , (double)200 );
  reader.setPosition( 5 );
  assertEquals( reader.readDouble().doubleValue() , (double)255 );
  reader.setPosition( 2 );
  assertEquals( reader.readDouble() , null );
  reader.setPosition( 3 );
  assertEquals( reader.readDouble() , null );
  reader.setPosition( 4 );
  assertEquals( reader.readDouble() , null );
}
Example 17
Source File: TestArrowDoubleMemoryAllocator.java    From yosegi with Apache License 2.0 5 votes vote down vote up 
@Test
public void T_setDouble_1() throws IOException{
  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.DOUBLE , "target" , allocator , parent , 4 );

  memoryAllocator.setDouble( 0 , (double)0.1 );
  memoryAllocator.setDouble( 1 , (double)0.2 );
  memoryAllocator.setDouble( 5 , (double)0.255 );
  memoryAllocator.setDouble( 1000 , (double)0.1 );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readDouble().doubleValue() , (double)0,1 );
  reader.setPosition( 1 );
  assertEquals( reader.readDouble().doubleValue() , (double)0.2 );
  reader.setPosition( 5 );
  assertEquals( reader.readDouble().doubleValue() , (double)0.255 );
  for( int i = 6 ; i < 1000 ; i++ ){
    reader.setPosition( i );
    assertEquals( reader.readDouble() , null );
  }
  reader.setPosition( 1000 );
  assertEquals( reader.readDouble().doubleValue() , (double)0.1 );
}
Example 18
Source File: TestArrowLongMemoryAllocator.java    From yosegi with Apache License 2.0 5 votes vote down vote up 
@Test
public void T_setLong_2() throws IOException{
  IColumn column = new PrimitiveColumn( ColumnType.LONG , "boolean" );
  column.add( ColumnType.LONG , new LongObj( (long)100 ) , 0 );
  column.add( ColumnType.LONG , new LongObj( (long)200 ) , 1 );
  column.add( ColumnType.LONG , new LongObj( (long)255 ) , 5 );

  ColumnBinaryMakerConfig defaultConfig = new ColumnBinaryMakerConfig();
  ColumnBinaryMakerCustomConfigNode configNode = new ColumnBinaryMakerCustomConfigNode( "root" , defaultConfig );

  IColumnBinaryMaker maker = new UnsafeOptimizeLongColumnBinaryMaker();
  ColumnBinary columnBinary = maker.toBinary( defaultConfig , null , new CompressResultNode() , column );

  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.LONG , "target" , allocator , parent , 3 );

  maker.loadInMemoryStorage( columnBinary , memoryAllocator );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readLong().longValue() , (long)100 );
  reader.setPosition( 1 );
  assertEquals( reader.readLong().longValue() , (long)200 );
  reader.setPosition( 5 );
  assertEquals( reader.readLong().longValue() , (long)255 );
  reader.setPosition( 2 );
  assertEquals( reader.readLong() , null );
  reader.setPosition( 3 );
  assertEquals( reader.readLong() , null );
  reader.setPosition( 4 );
  assertEquals( reader.readLong() , null );
}
Example 19
Source File: ArrowFixedSchemaMapMemoryAllocator.java    From multiple-dimension-spread with Apache License 2.0 5 votes vote down vote up 
public ArrowFixedSchemaMapMemoryAllocator( final MapContainerField schema , final BufferAllocator allocator , final StructVector vector , final int rowCount ){
  this.allocator = allocator;
  this.vector = vector;
  this.rowCount = rowCount;
  vector.allocateNew();
  childSchema = schema.getField();
}
Example 20
Source File: TestArrowDoubleMemoryAllocator.java    From multiple-dimension-spread with Apache License 2.0 5 votes vote down vote up 
@Test
public void T_setDouble_1() throws IOException{
  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.DOUBLE , "target" , allocator , parent , 4 );

  memoryAllocator.setDouble( 0 , (double)0.1 );
  memoryAllocator.setDouble( 1 , (double)0.2 );
  memoryAllocator.setDouble( 5 , (double)0.255 );
  memoryAllocator.setDouble( 1000 , (double)0.1 );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readDouble().doubleValue() , (double)0,1 );
  reader.setPosition( 1 );
  assertEquals( reader.readDouble().doubleValue() , (double)0.2 );
  reader.setPosition( 5 );
  assertEquals( reader.readDouble().doubleValue() , (double)0.255 );
  for( int i = 6 ; i < 1000 ; i++ ){
    reader.setPosition( i );
    assertEquals( reader.readDouble() , null );
  }
  reader.setPosition( 1000 );
  assertEquals( reader.readDouble().doubleValue() , (double)0.1 );
}