org.apache.arrow.vector.types.pojo.ArrowType.Struct Java Examples

The following examples show how to use org.apache.arrow.vector.types.pojo.ArrowType.Struct. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: Describer.java    From dremio-oss with Apache License 2.0 6 votes vote down vote up 
@Override
public String visit(Struct type) {
  StringBuilder sb = new StringBuilder();
  if(includeName){
    sb.append(field.getName());
    sb.append("::");
  }

  sb.append("struct<");
  boolean first = true;
  for (Field f : field.getChildren()) {
    if (first) {
      first = false;
    } else {
      sb.append(", ");
    }
    sb.append(describe(f, true));
  }
  sb.append(">");
  return sb.toString();
}
Example #2
Source File: TestArrowIntegerMemoryAllocator.java    From multiple-dimension-spread with Apache License 2.0 5 votes vote down vote up 
@Test
public void T_setInteger_1() throws IOException{
  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.INTEGER , "target" , allocator , parent , 1001 );

  memoryAllocator.setInteger( 0 , 100 );
  memoryAllocator.setInteger( 1 , 200 );
  memoryAllocator.setInteger( 5 , 255 );
  memoryAllocator.setInteger( 1000 , 10 );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readInteger().intValue() , 100 );
  reader.setPosition( 1 );
  assertEquals( reader.readInteger().intValue() , 200 );
  reader.setPosition( 5 );
  assertEquals( reader.readInteger().intValue() , 255 );
  for( int i = 6 ; i < 1000 ; i++ ){
    reader.setPosition( i );
    assertEquals( reader.readInteger() , null );
  }
  reader.setPosition( 1000 );
  assertEquals( reader.readInteger().intValue() , 10 );
}
Example #3
Source File: TestArrowLongMemoryAllocator.java    From multiple-dimension-spread with Apache License 2.0 5 votes vote down vote up 
@Test
public void T_setLong_2() throws IOException{
  IColumn column = new PrimitiveColumn( ColumnType.LONG , "boolean" );
  column.add( ColumnType.LONG , new LongObj( (long)100 ) , 0 );
  column.add( ColumnType.LONG , new LongObj( (long)200 ) , 1 );
  column.add( ColumnType.LONG , new LongObj( (long)255 ) , 5 );

  ColumnBinaryMakerConfig defaultConfig = new ColumnBinaryMakerConfig();
  ColumnBinaryMakerCustomConfigNode configNode = new ColumnBinaryMakerCustomConfigNode( "root" , defaultConfig );

  IColumnBinaryMaker maker = new OptimizeLongColumnBinaryMaker();
  ColumnBinary columnBinary = maker.toBinary( defaultConfig , null , column );

  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.LONG , "target" , allocator , parent , 3 );

  maker.loadInMemoryStorage( columnBinary , memoryAllocator );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readLong().longValue() , (long)100 );
  reader.setPosition( 1 );
  assertEquals( reader.readLong().longValue() , (long)200 );
  reader.setPosition( 5 );
  assertEquals( reader.readLong().longValue() , (long)255 );
  reader.setPosition( 2 );
  assertEquals( reader.readLong() , null );
  reader.setPosition( 3 );
  assertEquals( reader.readLong() , null );
  reader.setPosition( 4 );
  assertEquals( reader.readLong() , null );
}
Example #4
Source File: TestArrowDoubleMemoryAllocator.java    From multiple-dimension-spread with Apache License 2.0 5 votes vote down vote up 
@Test
public void T_setDouble_1() throws IOException{
  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.DOUBLE , "target" , allocator , parent , 4 );

  memoryAllocator.setDouble( 0 , (double)0.1 );
  memoryAllocator.setDouble( 1 , (double)0.2 );
  memoryAllocator.setDouble( 5 , (double)0.255 );
  memoryAllocator.setDouble( 1000 , (double)0.1 );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readDouble().doubleValue() , (double)0,1 );
  reader.setPosition( 1 );
  assertEquals( reader.readDouble().doubleValue() , (double)0.2 );
  reader.setPosition( 5 );
  assertEquals( reader.readDouble().doubleValue() , (double)0.255 );
  for( int i = 6 ; i < 1000 ; i++ ){
    reader.setPosition( i );
    assertEquals( reader.readDouble() , null );
  }
  reader.setPosition( 1000 );
  assertEquals( reader.readDouble().doubleValue() , (double)0.1 );
}
Example #5
Source File: TestArrowDoubleMemoryAllocator.java    From multiple-dimension-spread with Apache License 2.0 5 votes vote down vote up 
@Test
public void T_setDouble_2() throws IOException{
  IColumn column = new PrimitiveColumn( ColumnType.DOUBLE , "boolean" );
  column.add( ColumnType.DOUBLE , new DoubleObj( (double)100 ) , 0 );
  column.add( ColumnType.DOUBLE , new DoubleObj( (double)200 ) , 1 );
  column.add( ColumnType.DOUBLE , new DoubleObj( (double)255 ) , 5 );

  ColumnBinaryMakerConfig defaultConfig = new ColumnBinaryMakerConfig();
  ColumnBinaryMakerCustomConfigNode configNode = new ColumnBinaryMakerCustomConfigNode( "root" , defaultConfig );

  IColumnBinaryMaker maker = new OptimizeDoubleColumnBinaryMaker();
  ColumnBinary columnBinary = maker.toBinary( defaultConfig , null , column );

  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.DOUBLE , "target" , allocator , parent , 3 );

  maker.loadInMemoryStorage( columnBinary , memoryAllocator );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readDouble().doubleValue() , (double)100 );
  reader.setPosition( 1 );
  assertEquals( reader.readDouble().doubleValue() , (double)200 );
  reader.setPosition( 5 );
  assertEquals( reader.readDouble().doubleValue() , (double)255 );
  reader.setPosition( 2 );
  assertEquals( reader.readDouble() , null );
  reader.setPosition( 3 );
  assertEquals( reader.readDouble() , null );
  reader.setPosition( 4 );
  assertEquals( reader.readDouble() , null );
}
Example #6
Source File: TestArrowBooleanMemoryAllocator.java    From multiple-dimension-spread with Apache License 2.0 5 votes vote down vote up 
@Test
public void T_setBoolean_1() throws IOException{
  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.BOOLEAN , "target" , allocator , parent , 4 );

  memoryAllocator.setBoolean( 0 , true );
  memoryAllocator.setBoolean( 1 , false );
  memoryAllocator.setBoolean( 5 , true );
  memoryAllocator.setBoolean( 1000 , true );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readBoolean().booleanValue() , true );
  reader.setPosition( 1 );
  assertEquals( reader.readBoolean().booleanValue() , false );
  reader.setPosition( 5 );
  assertEquals( reader.readBoolean().booleanValue() , true );
  for( int i = 6 ; i < 1000 ; i++ ){
    reader.setPosition( i );
    assertEquals( reader.readBoolean() , null );
  }
  reader.setPosition( 1000 );
  assertEquals( reader.readBoolean().booleanValue() , true );
}
Example #7
Source File: DirectArrowLoader.java    From yosegi with Apache License 2.0 5 votes vote down vote up 
/**
 * FileReader and Arrow memory allocators are set and initialized.
 */
public DirectArrowLoader(
    final IRootMemoryAllocator rootMemoryAllocator ,
    final YosegiReader reader ,
    final BufferAllocator allocator ) {
  this.reader = reader;
  this.allocator = allocator;
  this.rootMemoryAllocator = rootMemoryAllocator;
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  rootVector = new StructVector(
      "root" , allocator , new FieldType( true , Struct.INSTANCE , null , null ) , callBack );
}
Example #8
Source File: TestArrowStringMemoryAllocator.java    From multiple-dimension-spread with Apache License 2.0 5 votes vote down vote up 
@Test
public void T_setString_1() throws IOException{
  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.STRING , "target" , allocator , parent , 4 );

  memoryAllocator.setString( 0 , "a" );
  memoryAllocator.setString( 1 , "b" );
  memoryAllocator.setString( 5 , "c" );
  memoryAllocator.setString( 1000 , "a b c" );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readText().toString() , "a" );
  reader.setPosition( 1 );
  assertEquals( reader.readText().toString() , "b" );
  reader.setPosition( 5 );
  assertEquals( reader.readText().toString() , "c" );
  for( int i = 6 ; i < 1000 ; i++ ){
    reader.setPosition( i );
    assertEquals( reader.readText() , null );
  }
  reader.setPosition( 1000 );
  assertEquals( reader.readText().toString() , "a b c" );
}
Example #9
Source File: TestArrowStringMemoryAllocator.java    From multiple-dimension-spread with Apache License 2.0 5 votes vote down vote up 
@Test
public void T_setString_2() throws IOException{
  IColumn column = new PrimitiveColumn( ColumnType.STRING , "boolean" );
  column.add( ColumnType.STRING , new StringObj( "a" ) , 0 );
  column.add( ColumnType.STRING , new StringObj( "b" ) , 1 );
  column.add( ColumnType.STRING , new StringObj( "c" ) , 5 );

  ColumnBinaryMakerConfig defaultConfig = new ColumnBinaryMakerConfig();
  ColumnBinaryMakerCustomConfigNode configNode = new ColumnBinaryMakerCustomConfigNode( "root" , defaultConfig );

  IColumnBinaryMaker maker = new OptimizeDumpStringColumnBinaryMaker();
  ColumnBinary columnBinary = maker.toBinary( defaultConfig , null , column );

  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.STRING , "target" , allocator , parent , 3 );

  maker.loadInMemoryStorage( columnBinary , memoryAllocator );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readText().toString() , "a" );
  reader.setPosition( 1 );
  assertEquals( reader.readText().toString() , "b" );
  reader.setPosition( 5 );
  assertEquals( reader.readText().toString() , "c" );
  reader.setPosition( 2 );
  assertEquals( reader.readText() , null );
  reader.setPosition( 3 );
  assertEquals( reader.readText() , null );
  reader.setPosition( 4 );
  assertEquals( reader.readText() , null );
}
Example #10
Source File: TestArrowBytesMemoryAllocator.java    From multiple-dimension-spread with Apache License 2.0 5 votes vote down vote up 
@Test
public void T_setBytes_1() throws IOException{
  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.BYTES , "target" , allocator , parent , 4 );

  memoryAllocator.setBytes( 0 , "a".getBytes() );
  memoryAllocator.setBytes( 1 , "b".getBytes() );
  memoryAllocator.setBytes( 5 , "c".getBytes() );
  memoryAllocator.setBytes( 1000 , "a b c".getBytes() );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( new String( reader.readByteArray() ) , "a" );
  reader.setPosition( 1 );
  assertEquals( new String( reader.readByteArray() ) , "b" );
  reader.setPosition( 5 );
  assertEquals( new String( reader.readByteArray() ) , "c" );
  for( int i = 6 ; i < 1000 ; i++ ){
    reader.setPosition( i );
    assertEquals( reader.readByteArray() , null );
  }
  reader.setPosition( 1000 );
  assertEquals( new String( reader.readByteArray() ) , "a b c" );
}
Example #11
Source File: TestArrowBytesMemoryAllocator.java    From multiple-dimension-spread with Apache License 2.0 5 votes vote down vote up 
@Test
public void T_setBytes_2() throws IOException{
  IColumn column = new PrimitiveColumn( ColumnType.STRING , "boolean" );
  column.add( ColumnType.STRING , new BytesObj( "a".getBytes() ) , 0 );
  column.add( ColumnType.STRING , new BytesObj( "b".getBytes() ) , 1 );
  column.add( ColumnType.STRING , new BytesObj( "c".getBytes() ) , 5 );

  ColumnBinaryMakerConfig defaultConfig = new ColumnBinaryMakerConfig();
  ColumnBinaryMakerCustomConfigNode configNode = new ColumnBinaryMakerCustomConfigNode( "root" , defaultConfig );

  IColumnBinaryMaker maker = new DumpBytesColumnBinaryMaker();
  ColumnBinary columnBinary = maker.toBinary( defaultConfig , null , column );

  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.BYTES , "target" , allocator , parent , 3 );
  maker.loadInMemoryStorage( columnBinary , memoryAllocator );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( new String( reader.readByteArray() ) , "a" );
  reader.setPosition( 1 );
  assertEquals( new String( reader.readByteArray() ) , "b" );
  reader.setPosition( 5 );
  assertEquals( new String( reader.readByteArray() ) , "c" );
  reader.setPosition( 2 );
  assertEquals( reader.readByteArray() , null );
  reader.setPosition( 3 );
  assertEquals( reader.readByteArray() , null );
  reader.setPosition( 4 );
  assertEquals( reader.readByteArray() , null );
}
Example #12
Source File: TestArrowFloatMemoryAllocator.java    From multiple-dimension-spread with Apache License 2.0 5 votes vote down vote up 
@Test
public void T_setFloat_1() throws IOException{
  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.FLOAT , "target" , allocator , parent , 4 );

  memoryAllocator.setFloat( 0 , (float)0.1 );
  memoryAllocator.setFloat( 1 , (float)0.2 );
  memoryAllocator.setFloat( 5 , (float)0.255 );
  memoryAllocator.setFloat( 1000 , (float)0.1 );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readFloat().floatValue() , (float)0,1 );
  reader.setPosition( 1 );
  assertEquals( reader.readFloat().floatValue() , (float)0.2 );
  reader.setPosition( 5 );
  assertEquals( reader.readFloat().floatValue() , (float)0.255 );
  for( int i = 6 ; i < 1000 ; i++ ){
    reader.setPosition( i );
    assertEquals( reader.readFloat() , null );
  }
  reader.setPosition( 1000 );
  assertEquals( reader.readFloat().floatValue() , (float)0.1 );
}
Example #13
Source File: TestArrowFloatMemoryAllocator.java    From multiple-dimension-spread with Apache License 2.0 5 votes vote down vote up 
@Test
public void T_setFloat_2() throws IOException{
  IColumn column = new PrimitiveColumn( ColumnType.FLOAT , "boolean" );
  column.add( ColumnType.FLOAT , new FloatObj( (float)100 ) , 0 );
  column.add( ColumnType.FLOAT , new FloatObj( (float)200 ) , 1 );
  column.add( ColumnType.FLOAT , new FloatObj( (float)255 ) , 5 );

  ColumnBinaryMakerConfig defaultConfig = new ColumnBinaryMakerConfig();
  ColumnBinaryMakerCustomConfigNode configNode = new ColumnBinaryMakerCustomConfigNode( "root" , defaultConfig );

  IColumnBinaryMaker maker = new OptimizeFloatColumnBinaryMaker();
  ColumnBinary columnBinary = maker.toBinary( defaultConfig , null , column );

  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.FLOAT , "target" , allocator , parent , 3 );

  maker.loadInMemoryStorage( columnBinary , memoryAllocator );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readFloat().floatValue() , (float)100 );
  reader.setPosition( 1 );
  assertEquals( reader.readFloat().floatValue() , (float)200 );
  reader.setPosition( 5 );
  assertEquals( reader.readFloat().floatValue() , (float)255 );
  reader.setPosition( 2 );
  assertEquals( reader.readFloat() , null );
  reader.setPosition( 3 );
  assertEquals( reader.readFloat() , null );
  reader.setPosition( 4 );
  assertEquals( reader.readFloat() , null );
}
Example #14
Source File: TestArrowBooleanMemoryAllocator.java    From multiple-dimension-spread with Apache License 2.0 5 votes vote down vote up 
@Test
public void T_setBoolean_2() throws IOException{
  IColumn column = new PrimitiveColumn( ColumnType.BOOLEAN , "boolean" );
  column.add( ColumnType.BOOLEAN , new BooleanObj( true ) , 0 );
  column.add( ColumnType.BOOLEAN , new BooleanObj( false ) , 1 );
  column.add( ColumnType.BOOLEAN , new BooleanObj( true ) , 5 );

  ColumnBinaryMakerConfig defaultConfig = new ColumnBinaryMakerConfig();
  ColumnBinaryMakerCustomConfigNode configNode = new ColumnBinaryMakerCustomConfigNode( "root" , defaultConfig );

  IColumnBinaryMaker maker = new DumpBooleanColumnBinaryMaker();
  ColumnBinary columnBinary = maker.toBinary( defaultConfig , null , column );

  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.BOOLEAN , "target" , allocator , parent , 3 );
  maker.loadInMemoryStorage( columnBinary , memoryAllocator );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readBoolean().booleanValue() , true );
  reader.setPosition( 1 );
  assertEquals( reader.readBoolean().booleanValue() , false );
  reader.setPosition( 5 );
  assertEquals( reader.readBoolean().booleanValue() , true );
  reader.setPosition( 2 );
  assertEquals( reader.readBoolean() , null );
  reader.setPosition( 3 );
  assertEquals( reader.readBoolean() , null );
  reader.setPosition( 4 );
  assertEquals( reader.readBoolean() , null );
}
Example #15
Source File: TestArrowIntegerMemoryAllocator.java    From multiple-dimension-spread with Apache License 2.0 5 votes vote down vote up 
@Test
public void T_setInteger_2() throws IOException{
  IColumn column = new PrimitiveColumn( ColumnType.INTEGER , "boolean" );
  column.add( ColumnType.INTEGER , new IntegerObj( 100 ) , 0 );
  column.add( ColumnType.INTEGER , new IntegerObj( 200 ) , 1 );
  column.add( ColumnType.INTEGER , new IntegerObj( 255 ) , 5 );

  ColumnBinaryMakerConfig defaultConfig = new ColumnBinaryMakerConfig();
  ColumnBinaryMakerCustomConfigNode configNode = new ColumnBinaryMakerCustomConfigNode( "root" , defaultConfig );

  IColumnBinaryMaker maker = new OptimizeLongColumnBinaryMaker();
  ColumnBinary columnBinary = maker.toBinary( defaultConfig , null , column );

  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.INTEGER , "target" , allocator , parent , 6 );

  maker.loadInMemoryStorage( columnBinary , memoryAllocator );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readInteger().intValue() , 100 );
  reader.setPosition( 1 );
  assertEquals( reader.readInteger().intValue() , 200 );
  reader.setPosition( 5 );
  assertEquals( reader.readInteger().intValue() , 255 );
  reader.setPosition( 2 );
  assertEquals( reader.readInteger() , null );
  reader.setPosition( 3 );
  assertEquals( reader.readInteger() , null );
  reader.setPosition( 4 );
  assertEquals( reader.readInteger() , null );
}
Example #16
Source File: TestArrowShortMemoryAllocator.java    From multiple-dimension-spread with Apache License 2.0 5 votes vote down vote up 
@Test
public void T_setShort_1() throws IOException{
  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.SHORT , "target" , allocator , parent , 1001 );

  memoryAllocator.setShort( 0 , (short)100 );
  memoryAllocator.setShort( 1 , (short)200 );
  memoryAllocator.setShort( 5 , (short)255 );
  memoryAllocator.setShort( 1000 , (short)10 );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( (short)( reader.readShort() ) , (short)100 );
  reader.setPosition( 1 );
  assertEquals( (short)( reader.readShort() ) , (short)200 );
  reader.setPosition( 5 );
  assertEquals( (short)( reader.readShort() ) , (short)255 );
  for( int i = 6 ; i < 1000 ; i++ ){
    reader.setPosition( i );
    assertEquals( reader.readShort() , null );
  }
  reader.setPosition( 1000 );
  assertEquals( (short)( reader.readShort() ) , (short)10 );
}
Example #17
Source File: TestArrowShortMemoryAllocator.java    From multiple-dimension-spread with Apache License 2.0 5 votes vote down vote up 
@Test
public void T_setShort_2() throws IOException{
  IColumn column = new PrimitiveColumn( ColumnType.SHORT , "boolean" );
  column.add( ColumnType.SHORT , new ShortObj( (short)100 ) , 0 );
  column.add( ColumnType.SHORT , new ShortObj( (short)200 ) , 1 );
  column.add( ColumnType.SHORT , new ShortObj( (short)255 ) , 5 );

  ColumnBinaryMakerConfig defaultConfig = new ColumnBinaryMakerConfig();
  ColumnBinaryMakerCustomConfigNode configNode = new ColumnBinaryMakerCustomConfigNode( "root" , defaultConfig );

  IColumnBinaryMaker maker = new OptimizeLongColumnBinaryMaker();
  ColumnBinary columnBinary = maker.toBinary( defaultConfig , null , column );

  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.SHORT , "target" , allocator , parent , 3 );
  maker.loadInMemoryStorage( columnBinary , memoryAllocator );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( (short)( reader.readShort() ) , (short)100 );
  reader.setPosition( 1 );
  assertEquals( (short)( reader.readShort() ) , (short)200 );
  reader.setPosition( 5 );
  assertEquals( (short)( reader.readShort() ) , (short)255 );
  reader.setPosition( 2 );
  assertEquals( reader.readShort() , null );
  reader.setPosition( 3 );
  assertEquals( reader.readShort() , null );
  reader.setPosition( 4 );
  assertEquals( reader.readShort() , null );
}
Example #18
Source File: TestArrowByteMemoryAllocator.java    From multiple-dimension-spread with Apache License 2.0 5 votes vote down vote up 
@Test
public void T_setByte_1() throws IOException{
  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.BYTE , "target" , allocator , parent , 1001 );

  memoryAllocator.setByte( 0 , (byte)100 );
  memoryAllocator.setByte( 1 , (byte)200 );
  memoryAllocator.setByte( 5 , (byte)255 );
  memoryAllocator.setByte( 1000 , (byte)10 );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readByte().byteValue() , (byte)100 );
  reader.setPosition( 1 );
  assertEquals( reader.readByte().byteValue() , (byte)200 );
  reader.setPosition( 5 );
  assertEquals( reader.readByte().byteValue() , (byte)255 );
  for( int i = 6 ; i < 1000 ; i++ ){
    reader.setPosition( i );
    assertEquals( reader.readByte() , null );
  }
  reader.setPosition( 1000 );
  assertEquals( reader.readByte().byteValue() , (byte)10 );
}
Example #19
Source File: TestArrowByteMemoryAllocator.java    From multiple-dimension-spread with Apache License 2.0 5 votes vote down vote up 
@Test
public void T_setByte_2() throws IOException{
  IColumn column = new PrimitiveColumn( ColumnType.BYTE , "boolean" );
  column.add( ColumnType.BYTE , new ByteObj( (byte)100 ) , 0 );
  column.add( ColumnType.BYTE , new ByteObj( (byte)200 ) , 1 );
  column.add( ColumnType.BYTE , new ByteObj( (byte)255 ) , 5 );

  ColumnBinaryMakerConfig defaultConfig = new ColumnBinaryMakerConfig();
  ColumnBinaryMakerCustomConfigNode configNode = new ColumnBinaryMakerCustomConfigNode( "root" , defaultConfig );

  IColumnBinaryMaker maker = new OptimizeLongColumnBinaryMaker();
  ColumnBinary columnBinary = maker.toBinary( defaultConfig , null , column );

  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.BYTE , "target" , allocator , parent , 6 );

  maker.loadInMemoryStorage( columnBinary , memoryAllocator );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readByte().byteValue() , (byte)100 );
  reader.setPosition( 1 );
  assertEquals( reader.readByte().byteValue() , (byte)200 );
  reader.setPosition( 5 );
  assertEquals( reader.readByte().byteValue() , (byte)255 );
  reader.setPosition( 2 );
  assertEquals( reader.readByte() , null );
  reader.setPosition( 3 );
  assertEquals( reader.readByte() , null );
  reader.setPosition( 4 );
  assertEquals( reader.readByte() , null );
}
Example #20
Source File: TestArrowArrayMemoryAllocator.java    From multiple-dimension-spread with Apache License 2.0 5 votes vote down vote up 
@Test
public void T_setArray_1() throws IOException{
  IColumn column = new ArrayColumn( "array" );
  List<Object> value = new ArrayList<Object>();
  value.add( new StringObj( "a" ) );
  value.add( new StringObj( "b" ) );
  value.add( new StringObj( "c" ) );
  column.add( ColumnType.ARRAY , value , 0 );
  column.add( ColumnType.ARRAY , value , 1 );
  column.add( ColumnType.ARRAY , value , 2 );
  column.add( ColumnType.ARRAY , value , 3 );
  column.add( ColumnType.ARRAY , value , 6 );

  ColumnBinaryMakerConfig defaultConfig = new ColumnBinaryMakerConfig();
  ColumnBinaryMakerCustomConfigNode configNode = new ColumnBinaryMakerCustomConfigNode( "root" , defaultConfig );

  IColumnBinaryMaker maker = new DumpArrayColumnBinaryMaker();
  ColumnBinary columnBinary = maker.toBinary( defaultConfig , null , column );

  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();

  ListVector listVector = parent.addOrGetList( "target" );
  IMemoryAllocator memoryAllocator = new ArrowArrayMemoryAllocator( allocator , listVector , 7 );
  maker.loadInMemoryStorage( columnBinary , memoryAllocator );

  for( int i = 0 ; i < 7 ; i++ ){
    System.out.println( "count:" + listVector.getInnerValueCountAt(i) );
    System.out.println( "obj:" + listVector.getObject(i) );
  }
}
Example #21
Source File: CompleteType.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up 
public boolean isComparable() {
  switch(type.getTypeID()) {
    case Struct:
    case List:
      return false;
    default:
      return true;
  }
}
Example #22
Source File: CompleteType.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up 
public boolean isScalar() {
  switch(type.getTypeID()){
  case List:
  case Struct:
  case Union:
    return false;
  default:
    return true;
  }
}
Example #23
Source File: CompleteType.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up 
public boolean isFixedWidthScalar() {
  switch(type.getTypeID()){
  case List:
  case Struct:
  case Union:
  case Binary:
  case Utf8:
    return false;
  default:
    return true;
  }
}
Example #24
Source File: AbstractTestNamespaceService.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up 
@Test
public void testDataSetSchema() throws Exception {
  Field field1 = new Field("a", true, new Int(32, true), null);
  Field child1 = new Field("c", true, Utf8.INSTANCE, null);
  Field field2 = new Field("b", true, Struct.INSTANCE, ImmutableList.of(child1));
  Schema schema = new Schema(ImmutableList.of(field1, field2));
  FlatBufferBuilder builder = new FlatBufferBuilder();
  schema.getSchema(builder);
  builder.finish(schema.getSchema(builder));
  NamespaceTestUtils.addSource(namespaceService, "s");
  NamespaceTestUtils.addPhysicalDS(namespaceService, "s.foo", builder.sizedByteArray());
  ByteBuffer bb = ByteBuffer.wrap(DatasetHelper.getSchemaBytes(namespaceService.getDataset(new NamespaceKey(PathUtils.parseFullPath("s.foo")))).toByteArray());
  Schema returnedSchema = Schema.convertSchema(org.apache.arrow.flatbuf.Schema.getRootAsSchema(bb));
  assertEquals(schema, returnedSchema);
}
Example #25
Source File: TestArrowBooleanMemoryAllocator.java    From yosegi with Apache License 2.0 5 votes vote down vote up 
@Test
public void T_setBoolean_2() throws IOException{
  IColumn column = new PrimitiveColumn( ColumnType.BOOLEAN , "boolean" );
  column.add( ColumnType.BOOLEAN , new BooleanObj( true ) , 0 );
  column.add( ColumnType.BOOLEAN , new BooleanObj( false ) , 1 );
  column.add( ColumnType.BOOLEAN , new BooleanObj( true ) , 5 );

  ColumnBinaryMakerConfig defaultConfig = new ColumnBinaryMakerConfig();
  ColumnBinaryMakerCustomConfigNode configNode = new ColumnBinaryMakerCustomConfigNode( "root" , defaultConfig );

  IColumnBinaryMaker maker = new DumpBooleanColumnBinaryMaker();
  ColumnBinary columnBinary = maker.toBinary( defaultConfig , null , new CompressResultNode() , column );

  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.BOOLEAN , "target" , allocator , parent , 3 );
  maker.loadInMemoryStorage( columnBinary , memoryAllocator );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readBoolean().booleanValue() , true );
  reader.setPosition( 1 );
  assertEquals( reader.readBoolean().booleanValue() , false );
  reader.setPosition( 5 );
  assertEquals( reader.readBoolean().booleanValue() , true );
  reader.setPosition( 2 );
  assertEquals( reader.readBoolean() , null );
  reader.setPosition( 3 );
  assertEquals( reader.readBoolean() , null );
  reader.setPosition( 4 );
  assertEquals( reader.readBoolean() , null );
}
Example #26
Source File: DynamicArrowLoader.java    From yosegi with Apache License 2.0 5 votes vote down vote up 
/**
 * FileReader and Arrow memory allocators are set and initialized.
 */
public DynamicArrowLoader(
    final IRootMemoryAllocator rootMemoryAllocator ,
    final YosegiReader reader ,
    final BufferAllocator allocator ) {
  this.reader = reader;
  this.allocator = allocator;
  this.rootMemoryAllocator = rootMemoryAllocator;
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  rootVector = new StructVector(
      "root" , allocator , new FieldType( true , Struct.INSTANCE , null , null ) , callBack );
}
Example #27
Source File: TestArrowLongMemoryAllocator.java    From yosegi with Apache License 2.0 5 votes vote down vote up 
@Test
public void T_setLong_1() throws IOException{
  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.LONG , "target" , allocator , parent , 1001 );

  memoryAllocator.setLong( 0 , (long)100 );
  memoryAllocator.setLong( 1 , (long)200 );
  memoryAllocator.setLong( 5 , (long)255 );
  memoryAllocator.setLong( 1000 , (long)10 );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readLong().longValue() , (long)100 );
  reader.setPosition( 1 );
  assertEquals( reader.readLong().longValue() , (long)200 );
  reader.setPosition( 5 );
  assertEquals( reader.readLong().longValue() , (long)255 );
  for( int i = 6 ; i < 1000 ; i++ ){
    reader.setPosition( i );
    assertEquals( reader.readLong() , null );
  }
  reader.setPosition( 1000 );
  assertEquals( reader.readLong().longValue() , (long)10 );
}
Example #28
Source File: TestArrowLongMemoryAllocator.java    From yosegi with Apache License 2.0 5 votes vote down vote up 
@Test
public void T_setLong_2() throws IOException{
  IColumn column = new PrimitiveColumn( ColumnType.LONG , "boolean" );
  column.add( ColumnType.LONG , new LongObj( (long)100 ) , 0 );
  column.add( ColumnType.LONG , new LongObj( (long)200 ) , 1 );
  column.add( ColumnType.LONG , new LongObj( (long)255 ) , 5 );

  ColumnBinaryMakerConfig defaultConfig = new ColumnBinaryMakerConfig();
  ColumnBinaryMakerCustomConfigNode configNode = new ColumnBinaryMakerCustomConfigNode( "root" , defaultConfig );

  IColumnBinaryMaker maker = new UnsafeOptimizeLongColumnBinaryMaker();
  ColumnBinary columnBinary = maker.toBinary( defaultConfig , null , new CompressResultNode() , column );

  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.LONG , "target" , allocator , parent , 3 );

  maker.loadInMemoryStorage( columnBinary , memoryAllocator );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readLong().longValue() , (long)100 );
  reader.setPosition( 1 );
  assertEquals( reader.readLong().longValue() , (long)200 );
  reader.setPosition( 5 );
  assertEquals( reader.readLong().longValue() , (long)255 );
  reader.setPosition( 2 );
  assertEquals( reader.readLong() , null );
  reader.setPosition( 3 );
  assertEquals( reader.readLong() , null );
  reader.setPosition( 4 );
  assertEquals( reader.readLong() , null );
}
Example #29
Source File: TestArrowDoubleMemoryAllocator.java    From yosegi with Apache License 2.0 5 votes vote down vote up 
@Test
public void T_setDouble_1() throws IOException{
  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.DOUBLE , "target" , allocator , parent , 4 );

  memoryAllocator.setDouble( 0 , (double)0.1 );
  memoryAllocator.setDouble( 1 , (double)0.2 );
  memoryAllocator.setDouble( 5 , (double)0.255 );
  memoryAllocator.setDouble( 1000 , (double)0.1 );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readDouble().doubleValue() , (double)0,1 );
  reader.setPosition( 1 );
  assertEquals( reader.readDouble().doubleValue() , (double)0.2 );
  reader.setPosition( 5 );
  assertEquals( reader.readDouble().doubleValue() , (double)0.255 );
  for( int i = 6 ; i < 1000 ; i++ ){
    reader.setPosition( i );
    assertEquals( reader.readDouble() , null );
  }
  reader.setPosition( 1000 );
  assertEquals( reader.readDouble().doubleValue() , (double)0.1 );
}
Example #30
Source File: TestArrowDoubleMemoryAllocator.java    From yosegi with Apache License 2.0 5 votes vote down vote up 
@Test
public void T_setDouble_2() throws IOException{
  IColumn column = new PrimitiveColumn( ColumnType.DOUBLE , "boolean" );
  column.add( ColumnType.DOUBLE , new DoubleObj( (double)100 ) , 0 );
  column.add( ColumnType.DOUBLE , new DoubleObj( (double)200 ) , 1 );
  column.add( ColumnType.DOUBLE , new DoubleObj( (double)255 ) , 5 );

  ColumnBinaryMakerConfig defaultConfig = new ColumnBinaryMakerConfig();
  ColumnBinaryMakerCustomConfigNode configNode = new ColumnBinaryMakerCustomConfigNode( "root" , defaultConfig );

  IColumnBinaryMaker maker = new UnsafeOptimizeDoubleColumnBinaryMaker();
  ColumnBinary columnBinary = maker.toBinary( defaultConfig , null , new CompressResultNode() , column );

  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.DOUBLE , "target" , allocator , parent , 3 );

  maker.loadInMemoryStorage( columnBinary , memoryAllocator );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readDouble().doubleValue() , (double)100 );
  reader.setPosition( 1 );
  assertEquals( reader.readDouble().doubleValue() , (double)200 );
  reader.setPosition( 5 );
  assertEquals( reader.readDouble().doubleValue() , (double)255 );
  reader.setPosition( 2 );
  assertEquals( reader.readDouble() , null );
  reader.setPosition( 3 );
  assertEquals( reader.readDouble() , null );
  reader.setPosition( 4 );
  assertEquals( reader.readDouble() , null );
}