Java Code Examples for org.apache.arrow.vector.ValueVector#getObject()

The following examples show how to use org.apache.arrow.vector.ValueVector#getObject() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestTextColumn.java    From dremio-oss with Apache License 2.0 6 votes vote down vote up 
private List<List<String>> getOutput(List<QueryDataBatch> batches) throws SchemaChangeException {
  List<List<String>> output = new ArrayList<>();
  RecordBatchLoader loader = new RecordBatchLoader(getAllocator());
  int last = 0;
  for(QueryDataBatch batch : batches) {
    int rows = batch.getHeader().getRowCount();
    if(batch.getData() != null) {
      loader.load(batch.getHeader().getDef(), batch.getData());
      // TODO:  Clean:  DRILL-2933:  That load(...) no longer throws
      // SchemaChangeException, so check/clean throws clause above.
      for (int i = 0; i < rows; ++i) {
        output.add(new ArrayList<String>());
        for (VectorWrapper<?> vw: loader) {
          ValueVector vv = vw.getValueVector();
          Object o = vv.getObject(i);
          output.get(last).add(o == null ? null: o.toString());
        }
        ++last;
      }
    }
    loader.clear();
    batch.release();
  }
  return output;
}
Example 2
Source File: ParquetResultListener.java    From dremio-oss with Apache License 2.0 6 votes vote down vote up 
@SuppressWarnings("unchecked")
private <T> void assertField(ValueVector valueVector, int index,
    TypeProtos.MinorType expectedMinorType, T value, String name, int parentFieldId) {

  if (expectedMinorType == TypeProtos.MinorType.STRUCT) {
    return;
  }

  final T val;
  try {
    val = (T) valueVector.getObject(index);
  } catch (Throwable ex) {
    throw ex;
  }

  if (val instanceof byte[]) {
    assertTrue(Arrays.equals((byte[]) value, (byte[]) val));
  } else {
    assertEquals(value, val);
  }
}
Example 3
Source File: ParquetResultListener.java    From dremio-oss with Apache License 2.0 6 votes vote down vote up 
public void printColumnMajor(ValueVector vv) {
  if (ParquetRecordReaderTest.VERBOSE_DEBUG){
    System.out.println("\n" + vv.getField().getName());
  }
  for (int j = 0; j < vv.getValueCount(); j++) {
    if (ParquetRecordReaderTest.VERBOSE_DEBUG){
      Object o = vv.getObject(j);
      if (o instanceof byte[]) {
        try {
          o = new String((byte[])o, "UTF-8");
        } catch (UnsupportedEncodingException e) {
          throw new RuntimeException(e);
        }
      }
      System.out.print(Strings.padStart(o + "", 20, ' ') + " ");
      System.out.print(", " + (j % 25 == 0 ? "\n batch:" + batchCounter + " v:" + j + " - " : ""));
    }
  }
  if (ParquetRecordReaderTest.VERBOSE_DEBUG) {
    System.out.println("\n" + vv.getValueCount());
  }
}
Example 4
Source File: BatchPrinter.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up 
private static void printBatchNoSV(VectorAccessible batch) {
  List<String> columns = Lists.newArrayList();
  List<ValueVector> vectors = Lists.newArrayList();
  for (VectorWrapper<?> vw : batch) {
    columns.add(vw.getValueVector().getField().getName());
    vectors.add(vw.getValueVector());
  }
  int width = columns.size();
  int rows = batch.getRecordCount();
  rows = Math.min(ROWS_TO_PRINT, rows);
  for (int row = 0; row < rows; row++) {
    if (row%50 == 0) {
      System.out.println(StringUtils.repeat("-", width * 52 + 1));
      for (String column : columns) {
        System.out.printf("| %-50s", width <= 50 ? column : column.substring(0, Math.min(column.length(), 49)));
      }
      System.out.printf("|\n");
      System.out.println(StringUtils.repeat("-", width*52 + 1));
    }
    for (ValueVector vv : vectors) {
      Object o = vv.getObject(row);
      String value;
      if (o == null) {
        value = "null";
      } else
      if (o instanceof byte[]) {
        value = new String((byte[]) o);
      } else {
        value = o.toString();
      }
      System.out.printf("| %-50s",value.length() <= 50 ? value : value.substring(0, 49));
    }
    System.out.printf("|\n");
  }
}
Example 5
Source File: BatchPrinter.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up 
private static void printBatchSV2(VectorAccessible batch, SelectionVector2 sv2) {
  List<String> columns = Lists.newArrayList();
  List<ValueVector> vectors = Lists.newArrayList();
  for (VectorWrapper<?> vw : batch) {
    columns.add(vw.getValueVector().getField().getName());
    vectors.add(vw.getValueVector());
  }
  int width = columns.size();
  int rows = batch.getRecordCount();
  rows = Math.min(ROWS_TO_PRINT, rows);
  for (int i = 0; i < rows; i++) {
    if (i%50 == 0) {
      System.out.println(StringUtils.repeat("-", width * 32 + 1));
      for (String column : columns) {
        System.out.printf("| %-30s", width <= 30 ? column : column.substring(0, 29));
      }
      System.out.printf("|\n");
      System.out.println(StringUtils.repeat("-", width*32 + 1));
    }
    int row = sv2.getIndex(i);
    for (ValueVector vv : vectors) {
      Object o = vv.getObject(row);
      String value;
      if (o == null) {
        value = "null";
      } else
      if (o instanceof byte[]) {
        value = new String((byte[]) o);
      } else {
        value = o.toString();
      }
      System.out.printf("| %-30s",value.length() <= 30 ? value : value.substring(0, 29));
    }
    System.out.printf("|\n");
  }
}
Example 6
Source File: DremioSeparatePlanningTest.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up 
@Override
public void dataArrived(QueryDataBatch result, ConnectionThrottle throttle) {
  QueryData queryHeader = result.getHeader();
  int rows = queryHeader.getRowCount();
  try {
    if ( result.hasData() ) {
      ArrowBuf data = result.getData();
      loader.load(queryHeader.getDef(), data);
      for (int i = 0; i < rows; i++) {
         Map<String,String> record = Maps.newHashMap();
        for (VectorWrapper<?> vw : loader) {
          final String field = vw.getValueVector().getField().getName();
          final ValueVector vv = vw.getValueVector();
          final Object value = i < vv.getValueCount() ? vv.getObject(i) : null;
          final String display = value == null ? null : value.toString();
          record.put(field, display);
        }
        records.add(record);
      }
      loader.clear();
    }
    result.release();
  } catch (SchemaChangeException e) {
    fail(e.getMessage());
  }

}
Example 7
Source File: TestParquetPhysicalPlan.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up 
@Test
  @Ignore
  public void testParseParquetPhysicalPlan() throws Exception {
    try (ClusterCoordinator clusterCoordinator = LocalClusterCoordinator.newRunningCoordinator();
         SabotNode bit1 = new SabotNode(DEFAULT_SABOT_CONFIG, clusterCoordinator, CLASSPATH_SCAN_RESULT, true); DremioClient client = new DremioClient(DEFAULT_SABOT_CONFIG, clusterCoordinator)) {
      bit1.run();
      client.connect();
      List<QueryDataBatch> results = client.runQuery(com.dremio.exec.proto.UserBitShared.QueryType.PHYSICAL, Resources.toString(Resources.getResource(fileName),Charsets.UTF_8));
      RecordBatchLoader loader = new RecordBatchLoader(bit1.getContext().getAllocator());
      int count = 0;
      for (QueryDataBatch b : results) {
        System.out.println(String.format("Got %d results", b.getHeader().getRowCount()));
        count += b.getHeader().getRowCount();
        loader.load(b.getHeader().getDef(), b.getData());
        for (VectorWrapper vw : loader) {
          System.out.print(vw.getValueVector().getField().getName() + ": ");
          ValueVector vv = vw.getValueVector();
          for (int i = 0; i < vv.getValueCount(); i++) {
            Object o = vv.getObject(i);
            if (o instanceof byte[]) {
              System.out.print(" [" + new String((byte[]) o) + "]");
            } else {
              System.out.print(" [" + vv.getObject(i) + "]");
            }
//            break;
          }
          System.out.println();
        }
        loader.clear();
        b.release();
      }
      client.close();
      System.out.println(String.format("Got %d total results", count));
    }
  }
Example 8
Source File: BaseTestQuery.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up 
protected static String getValueInFirstRecord(String sql, String columnName)
    throws Exception {
  final List<QueryDataBatch> results = testSqlWithResults(sql);
  final RecordBatchLoader loader = new RecordBatchLoader(getSabotContext().getAllocator());
  final StringBuilder builder = new StringBuilder();
  final boolean silent = config != null && config.getBoolean(QueryTestUtil.TEST_QUERY_PRINTING_SILENT);

  for (final QueryDataBatch b : results) {
    if (!b.hasData()) {
      continue;
    }

    loader.load(b.getHeader().getDef(), b.getData());

    final VectorWrapper<?> vw;
    try {
        vw = loader.getValueAccessorById(
            VarCharVector.class,
            loader.getValueVectorId(SchemaPath.getSimplePath(columnName)).getFieldIds());
    } catch (Throwable t) {
      throw new Exception("Looks like you did not provide an explain plan query, please add EXPLAIN PLAN FOR to the beginning of your query.");
    }

    if (!silent) {
      System.out.println(vw.getValueVector().getField().getName());
    }
    final ValueVector vv = vw.getValueVector();
    for (int i = 0; i < vv.getValueCount(); i++) {
      final Object o = vv.getObject(i);
      builder.append(o);
      if (!silent) {
        System.out.println(o);
      }
    }
    loader.clear();
    b.release();
  }

  return builder.toString();
}
Example 9
Source File: TestMergingReceiver.java    From dremio-oss with Apache License 2.0 4 votes vote down vote up 
@Test
public void testMultipleProvidersMixedSizes() throws Exception {
  try (final ClusterCoordinator clusterCoordinator = LocalClusterCoordinator.newRunningCoordinator();
       final SabotNode bit1 = new SabotNode(DEFAULT_SABOT_CONFIG, clusterCoordinator, CLASSPATH_SCAN_RESULT, true);
       final SabotNode bit2 = new SabotNode(DEFAULT_SABOT_CONFIG, clusterCoordinator, CLASSPATH_SCAN_RESULT, false);
       final DremioClient client = new DremioClient(DEFAULT_SABOT_CONFIG, clusterCoordinator);) {

    bit1.run();
    bit2.run();
    client.connect();
    final List<QueryDataBatch> results =
        client.runQuery(com.dremio.exec.proto.UserBitShared.QueryType.PHYSICAL,
            Files.toString(FileUtils.getResourceAsFile("/mergerecv/multiple_providers.json"),
                Charsets.UTF_8));
    int count = 0;
    final RecordBatchLoader batchLoader = new RecordBatchLoader(client.getRecordAllocator());
    // print the results
    Long lastBlueValue = null;
    for (final QueryDataBatch b : results) {
      final QueryData queryData = b.getHeader();
      final int batchRowCount = queryData.getRowCount();
      count += batchRowCount;
      batchLoader.load(queryData.getDef(), b.getData());
      for (final VectorWrapper vw : batchLoader) {
        final ValueVector vv = vw.getValueVector();
        final Field materializedField = vv.getField();
        final int numValues = vv.getValueCount();
        for(int valueIdx = 0; valueIdx < numValues; ++valueIdx) {
          if (materializedField.getName().equals("blue")) {
            Long val = (Long) vv.getObject(valueIdx);
            if (val != null) {
              final long longValue = ((Long) vv.getObject(valueIdx)).longValue();
              // check that order is ascending
              if (lastBlueValue != null) {
                assertTrue(longValue >= lastBlueValue);
              }
              lastBlueValue = longValue;
            }
          }
        }
      }
      b.release();
      batchLoader.clear();
    }
    assertEquals(400000, count);
  }
}