Java Code Examples for io.protostuff.ByteString

The following examples show how to use io.protostuff.ByteString. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: dremio-oss   Source File: WriterOptions.java    License: Apache License 2.0 6 votes vote down vote up
@JsonCreator
public WriterOptions(
  @JsonProperty("ringCount") Integer ringCount,
  @JsonProperty("partitionColumns") List<String> partitionColumns,
  @JsonProperty("sortColumns") List<String> sortColumns,
  @JsonProperty("distributionColumns") List<String> distributionColumns,
  @JsonProperty("partitionDistributionStrategy") PartitionDistributionStrategy partitionDistributionStrategy,
  @JsonProperty("singleWriter") boolean singleWriter,
  @JsonProperty("recordLimit") long recordLimit,
  @JsonProperty("icebergWriterOperation") IcebergWriterOperation icebergWriterOperation,
  @JsonProperty("extendedProperty") ByteString extendedProperty
  ) {
  this.ringCount = ringCount;
  this.partitionColumns = partitionColumns;
  this.sortColumns = sortColumns;
  this.distributionColumns = distributionColumns;
  this.partitionDistributionStrategy = partitionDistributionStrategy;
  this.singleWriter = singleWriter;
  this.recordLimit = recordLimit;
  this.icebergWriterOperation = icebergWriterOperation;
  this.extendedProperty = extendedProperty;
}
 
Example 2
Source Project: dremio-oss   Source File: DatasetRetrievalOptions.java    License: Apache License 2.0 6 votes vote down vote up
private <T extends MetadataOption> void addDatasetOptions(Class<T> clazz, DatasetConfig datasetConfig, List<T> outOptions) {
  if(datasetConfig == null) {
    return;
  }

  List<MetadataOption> options = new ArrayList<>();
  if(datasetConfig.getPhysicalDataset() != null && datasetConfig.getPhysicalDataset().getFormatSettings() != null) {
    options.add(new FileConfigOption(datasetConfig.getPhysicalDataset().getFormatSettings()));
  }

  if(datasetConfig.getReadDefinition() != null && datasetConfig.getReadDefinition().getSortColumnsList() != null) {
    options.add(new SortColumnsOption(datasetConfig.getReadDefinition().getSortColumnsList()));
  }

  BatchSchema schema = DatasetHelper.getSchemaBytes(datasetConfig) != null ? CalciteArrowHelper.fromDataset(datasetConfig) : null;
  if(schema != null) {
    options.add(new CurrentSchemaOption(schema));
  }

  if (datasetConfig.getReadDefinition() != null && datasetConfig.getReadDefinition().getExtendedProperty() != null) {
    options.add(new ExtendedPropertyOption(os -> ByteString.writeTo(os, datasetConfig.getReadDefinition().getExtendedProperty())));
  }

  List<T> addOptions = options.stream().filter(o -> clazz.isAssignableFrom(o.getClass())).map(o -> clazz.cast(o)).collect(Collectors.toList());
  outOptions.addAll(addOptions);
}
 
Example 3
Source Project: dremio-oss   Source File: ParquetSubScan.java    License: Apache License 2.0 6 votes vote down vote up
public ParquetSubScan(
  OpProps props,
  FileConfig formatSettings,
  List<SplitAndPartitionInfo> splits,
  BatchSchema fullSchema,
  List<List<String>> tablePath,
  List<ParquetFilterCondition> conditions,
  StoragePluginId pluginId,
  List<SchemaPath> columns,
  List<String> partitionColumns,
  List<GlobalDictionaryFieldInfo> globalDictionaryEncodedColumns,
  ByteString extendedProperty
) {
  super(props, fullSchema, tablePath, columns);
  this.formatSettings = formatSettings;
  this.splits = splits;
  this.tablePath = tablePath;
  this.conditions = conditions == null ? null : ImmutableList.copyOf(conditions);
  this.pluginId = pluginId;
  this.partitionColumns = partitionColumns;
  this.globalDictionaryEncodedColumns = globalDictionaryEncodedColumns;
  this.extendedProperty = extendedProperty;
}
 
Example 4
Source Project: dremio-oss   Source File: ParquetSubScan.java    License: Apache License 2.0 6 votes vote down vote up
@JsonCreator
public ParquetSubScan(
  @JsonProperty("props") OpProps props,
  @JsonProperty("formatSettings") FileConfig formatSettings,
  @JsonProperty("schema") BatchSchema fullSchema,
  @JsonProperty("referencedTables") List<List<String>> tablePath,
  @JsonProperty("conditions") List<ParquetFilterCondition> conditions,
  @JsonProperty("pluginId") StoragePluginId pluginId,
  @JsonProperty("columns") List<SchemaPath> columns,
  @JsonProperty("partitionColumns") List<String> partitionColumns,
  @JsonProperty("globalDictionaryEncodedColumns") List<GlobalDictionaryFieldInfo> globalDictionaryEncodedColumns,
  @JsonProperty("extendedProperty") ByteString extendedProperty) {

  this(props, formatSettings, null, fullSchema, tablePath, conditions, pluginId, columns, partitionColumns,
    globalDictionaryEncodedColumns, extendedProperty);
}
 
Example 5
Source Project: dremio-oss   Source File: TestConfigurationStore.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testStore() throws Exception {
  try(final LegacyKVStoreProvider kvstore =
    new LocalKVStoreProvider(DremioTest.CLASSPATH_SCAN_RESULT, null, true, false).asLegacy()) {
    kvstore.start();
    ConfigurationStore store = new ConfigurationStore(kvstore);

    ConfigurationEntry supportEntry = new ConfigurationEntry();
    supportEntry.setType("mytype");
    supportEntry.setValue(ByteString.copyFrom("test string", "UTF8"));
    store.put("key", supportEntry);

    ConfigurationEntry retrieved = store.get("key");
    Assert.assertEquals(retrieved.getType(), supportEntry.getType());
    Assert.assertEquals(retrieved.getValue(), supportEntry.getValue());

    store.delete("key");
    retrieved = store.get("key");
    Assert.assertEquals(retrieved, null);
  }
}
 
Example 6
Source Project: dremio-oss   Source File: FileFormat.java    License: Apache License 2.0 6 votes vote down vote up
@JsonIgnore
@SuppressWarnings({ "rawtypes", "unchecked" })
public FileConfig asFileConfig() {
  buffer.clear();
  FileConfig fc = new FileConfig();
  fc.setType(getFileType());
  fc.setName(name);
  fc.setOwner(owner);
  fc.setCtime(ctime);
  fc.setType(getFileType());
  fc.setTag(getVersion());
  fc.setLocation(location);
  byte[] bytes = ProtobufIOUtil.toByteArray(this, (Schema) getPrivateSchema(), buffer);
  fc.setExtendedConfig(ByteString.copyFrom(bytes));
  fc.setFullPathList(fullPath);
  return fc;
}
 
Example 7
Source Project: dremio-oss   Source File: NamespaceConverter.java    License: Apache License 2.0 6 votes vote down vote up
public static String[] getColumnsLowerCase(DatasetConfig datasetConfig) {
  final ByteString schemaBytes = DatasetHelper.getSchemaBytes(datasetConfig);
  if (schemaBytes != null) {
    Schema schema = Schema.getRootAsSchema(schemaBytes.asReadOnlyByteBuffer());
    org.apache.arrow.vector.types.pojo.Schema s = org.apache.arrow.vector.types.pojo.Schema.convertSchema(schema);
    return s.getFields().stream().map(input -> input.getName().toLowerCase()).toArray(String[]::new);
  } else {
    // If virtual dataset was created with view fields
    if (datasetConfig.getType() == DatasetType.VIRTUAL_DATASET) {
      final List<ViewFieldType> viewFieldTypes = datasetConfig.getVirtualDataset().getSqlFieldsList();
      if (notEmpty(viewFieldTypes)) {
        return viewFieldTypes.stream().map(input -> input.getName().toLowerCase()).toArray(String[]::new);
      }
    }
  }

  return new String[0];
}
 
Example 8
Source Project: dremio-oss   Source File: DatasetsUtil.java    License: Apache License 2.0 6 votes vote down vote up
public static List<org.apache.arrow.vector.types.pojo.Field> getArrowFieldsFromDatasetConfig(DatasetConfig datasetConfig) {
  List<org.apache.arrow.vector.types.pojo.Field> fields = Lists.newArrayList();
  final ByteString schemaBytes = DatasetHelper.getSchemaBytes(datasetConfig);
  if (schemaBytes == null) {
    return null;
  }

  final BatchSchema batchSchema = BatchSchema.deserialize(schemaBytes);
  for (int i = 0; i < batchSchema.getFieldCount(); i++) {
    final org.apache.arrow.vector.types.pojo.Field field = batchSchema.getColumn(i);
    if (!NamespaceTable.SYSTEM_COLUMNS.contains(field.getName())) {
      fields.add(field);
    }
  }

  return fields;
}
 
Example 9
Source Project: dremio-oss   Source File: TestHomeFiles.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testNASFileStore() throws Exception {

  final CatalogServiceImpl catalog = (CatalogServiceImpl) l(CatalogService.class);
  final SourceConfig config = catalog.getManagedSource(HomeFileSystemStoragePlugin.HOME_PLUGIN_NAME).getId().getClonedConfig();
  final ByteString oldConfig = config.getConfig();
  final HomeFileConf nasHomeFileStore = new HomeFileConf(Path.of("file:///" + BaseTestServer.folder1.getRoot().toString() + "/" + "testNASFileStore/").toString(), "localhost");
  nasHomeFileStore.getFilesystemAndCreatePaths("localhost");
  config.setConnectionConf(nasHomeFileStore);
  catalog.getSystemUserCatalog().updateSource(config);
  HomeFileTool tool = l(HomeFileTool.class);

  try {
    runTests(nasHomeFileStore);
  } finally {
    tool.clear();
    // reset plugin
    SourceConfig backConfig = catalog.getManagedSource(HomeFileSystemStoragePlugin.HOME_PLUGIN_NAME).getId().getClonedConfig();
    backConfig.setConfig(oldConfig);
    catalog.getSystemUserCatalog().updateSource(backConfig);
  }
}
 
Example 10
Source Project: sofa-jraft   Source File: UnsafeNioBufInput.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("all")
@Override
public ByteString readBytes() throws IOException {
    try {
        return (ByteString) byteStringWrapMethod.invoke(null, readByteArray());
    } catch (Exception e) {
        ThrowUtil.throwException(e);
    }
    return null; // never get here
}
 
Example 11
Source Project: protostuff   Source File: Foo.java    License: Apache License 2.0 5 votes vote down vote up
public Foo(List<Integer> someInt, List<String> someString,
        List<Bar> someBar, List<EnumSample> someEnum,
        List<ByteString> someBytes, List<Boolean> someBoolean,
        List<Float> someFloat, List<Double> someDouble, List<Long> someLong)
{
    this.someInt = someInt;
    this.someString = someString;
    this.someBar = someBar;
    this.someEnum = someEnum;
    this.someBytes = someBytes;
    this.someBoolean = someBoolean;
    this.someFloat = someFloat;
    this.someDouble = someDouble;
    this.someLong = someLong;
}
 
Example 12
Source Project: protostuff   Source File: NullArrayElementTest.java    License: Apache License 2.0 5 votes vote down vote up
public void testNullLast() throws IOException
{
    Schema<PojoWithNonPrimitiveArrays> schema =
            getSchema(PojoWithNonPrimitiveArrays.class);

    Pipe.Schema<PojoWithNonPrimitiveArrays> pipeSchema =
            ((RuntimeSchema<PojoWithNonPrimitiveArrays>) schema).getPipeSchema();

    PojoWithNonPrimitiveArrays p = new PojoWithNonPrimitiveArrays(
            new Boolean[] { true, false, null },
            new Character[] { 'a', 'b', null },
            new Short[] { 1, 2, null },
            new Integer[] { 1, 2, null },
            new Long[] { 1l, 2l, null },
            new Float[] { 1.1f, 2.2f, null },
            new Double[] { 1.1d, 2.2d, null },
            new String[] { "a", "b", null },
            new ByteString[] { ByteString.copyFromUtf8("a"), ByteString.copyFromUtf8("b"), null },
            new byte[][] { new byte[] { 'a' }, new byte[] { 'b' }, null },
            new BigDecimal[] { new BigDecimal(1.1d), new BigDecimal(2.2d), null },
            new BigInteger[] { new BigInteger("1"), new BigInteger("2"), null },
            new Date[] { new Date(), new Date(), null },
            new Size[] { Size.MEDIUM, Size.LARGE, null },
            new SomePojo[] { new SomePojo("a"), new SomePojo("b"), null }
            );

    byte[] data = toByteArray(p, schema);

    PojoWithNonPrimitiveArrays pFromByteArray = schema.newMessage();
    mergeFrom(data, 0, data.length, pFromByteArray, schema);
    assertEquals(p, pFromByteArray);

    PojoWithNonPrimitiveArrays pFromStream = schema.newMessage();
    ByteArrayInputStream in = new ByteArrayInputStream(data);
    mergeFrom(in, pFromStream, schema);
    assertEquals(p, pFromStream);

    roundTrip(p, schema, pipeSchema);
}
 
Example 13
Source Project: dremio-oss   Source File: CalciteArrowHelper.java    License: Apache License 2.0 5 votes vote down vote up
public static BatchSchema fromDataset(DatasetConfig config){
  try{
    ByteString bytes = DatasetHelper.getSchemaBytes(config);
    if(bytes == null){
      throw new IllegalStateException(String.format("Schema is currently unavailable for dataset %s.", PathUtils.constructFullPath(config.getFullPathList())));
    }
    return BatchSchema.deserialize(bytes);
  }catch(Exception ex){
    throw new IllegalStateException(String.format("Schema for dataset %s is corrupt.", PathUtils.constructFullPath(config.getFullPathList())), ex);
  }
}
 
Example 14
Source Project: dremio-oss   Source File: ViewFieldsHelper.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Get view fields from dataset config as it is in the batch schema (if not present, defaults to reltree's row type)
 * @param config dataset config
 * @return List of view fields
 */
public static List<ViewFieldType> getViewFields(DatasetConfig config) {
  final ByteString schemaBytes = DatasetHelper.getSchemaBytes(config);
  final boolean virtualDataset = config.getType() == DatasetType.VIRTUAL_DATASET;
  if (schemaBytes == null && virtualDataset) {
    return config.getVirtualDataset().getSqlFieldsList();
  }
  if (schemaBytes != null) {
    return getBatchSchemaFields(BatchSchema.deserialize(schemaBytes.toByteArray()));
  }
  return null;
}
 
Example 15
Source Project: dremio-oss   Source File: EasySubScan.java    License: Apache License 2.0 5 votes vote down vote up
@JsonCreator
public EasySubScan(
  @JsonProperty("props") OpProps props,
  @JsonProperty("settings") FileConfig config,
  @JsonProperty("fullSchema") BatchSchema fullSchema,
  @JsonProperty("tableSchemaPath") List<String> tablePath,
  @JsonProperty("pluginId") StoragePluginId pluginId,
  @JsonProperty("columns") List<SchemaPath> columns,
  @JsonProperty("partitionColumns") List<String> partitionColumns,
  @JsonProperty("extendedProperty") ByteString extendedProperty) {

  this(props, config, null, fullSchema, tablePath, pluginId, columns, partitionColumns, extendedProperty);
}
 
Example 16
Source Project: dremio-oss   Source File: ParquetRecordWriter.java    License: Apache License 2.0 5 votes vote down vote up
private void initIcebergColumnIDList(ByteString extendedProperty) {
  try {
    IcebergProtobuf.IcebergDatasetXAttr icebergDatasetXAttr = LegacyProtobufSerializer.parseFrom(IcebergProtobuf.IcebergDatasetXAttr.PARSER,
      extendedProperty.toByteArray());
    List<IcebergProtobuf.IcebergSchemaField> icebergColumnIDs = icebergDatasetXAttr.getColumnIdsList();
    Map<String, Integer> icebergColumns = new HashMap<>();
    icebergColumnIDs.forEach(field -> icebergColumns.put(field.getSchemaPath(), field.getId()));
    this.icebergColumnIDMap = CaseInsensitiveImmutableBiMap.newImmutableMap(icebergColumns);
  } catch (InvalidProtocolBufferException e) {
    throw new RuntimeException("Could not deserialize Parquet dataset info", e);
  }
}
 
Example 17
Source Project: protostuff   Source File: Bar.java    License: Apache License 2.0 5 votes vote down vote up
public Bar(int someInt, String someString, Baz baz, Status someEnum,
        ByteString someBytes, boolean someBoolean, float someFloat,
        double someDouble, long someLong)
{
    this.someInt = someInt;
    this.someString = someString;
    this.someBaz = baz;
    this.someEnum = someEnum;
    this.someBytes = someBytes;
    this.someBoolean = someBoolean;
    this.someFloat = someFloat;
    this.someDouble = someDouble;
    this.someLong = someLong;
}
 
Example 18
Source Project: dremio-oss   Source File: MaterializationCache.java    License: Apache License 2.0 5 votes vote down vote up
private boolean schemaChanged(MaterializationDescriptor old, Materialization materialization) {
  if (namespaceService == null) {
    return false;
  }
  try {
    //TODO is this enough ? shouldn't we use the dataset hash instead ??
    final NamespaceKey matKey = new NamespaceKey(ReflectionUtils.getMaterializationPath(materialization));
    ByteString schemaString = namespaceService.getDataset(matKey).getRecordSchema();
    BatchSchema newSchema = BatchSchema.deserialize(schemaString);
    BatchSchema oldSchema = ((CachedMaterializationDescriptor) old).getMaterialization().getSchema();
    return !oldSchema.equals(newSchema);
  } catch (NamespaceException e) {
    return true;
  }
}
 
Example 19
Source Project: dremio-oss   Source File: LocalJobsService.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void recordExtraInfo(String name, byte[] bytes) {
  //TODO DX-10977 the reflection manager should rely on its own observer to store this information in a separate store
  if(job.getJobAttempt().getExtraInfoList() == null) {
    job.getJobAttempt().setExtraInfoList(new ArrayList<ExtraInfo>());
  }

  job.getJobAttempt().getExtraInfoList().add(new ExtraInfo()
      .setData(ByteString.copyFrom(bytes))
      .setName(name));
  storeJob(job);
  super.recordExtraInfo(name, bytes);
}
 
Example 20
Source Project: protostuff   Source File: NullArrayElementTest.java    License: Apache License 2.0 5 votes vote down vote up
public void testNullMid() throws IOException
{
    Schema<PojoWithNonPrimitiveArrays> schema =
            getSchema(PojoWithNonPrimitiveArrays.class);

    Pipe.Schema<PojoWithNonPrimitiveArrays> pipeSchema =
            ((RuntimeSchema<PojoWithNonPrimitiveArrays>) schema).getPipeSchema();

    PojoWithNonPrimitiveArrays p = new PojoWithNonPrimitiveArrays(
            new Boolean[] { true, null, false },
            new Character[] { 'a', null, 'b' },
            new Short[] { 1, null, 2 },
            new Integer[] { 1, null, 2 },
            new Long[] { 1l, null, 2l },
            new Float[] { 1.1f, null, 2.2f },
            new Double[] { 1.1d, null, 2.2d },
            new String[] { "a", null, "b" },
            new ByteString[] { ByteString.copyFromUtf8("a"), null, ByteString.copyFromUtf8("b") },
            new byte[][] { new byte[] { 'a' }, null, new byte[] { 'b' } },
            new BigDecimal[] { new BigDecimal(1.1d), null, new BigDecimal(2.2d) },
            new BigInteger[] { new BigInteger("1"), null, new BigInteger("2") },
            new Date[] { new Date(), null, new Date() },
            new Size[] { Size.MEDIUM, null, Size.LARGE },
            new SomePojo[] { new SomePojo("a"), null, new SomePojo("b") }
            );

    byte[] data = toByteArray(p, schema);

    PojoWithNonPrimitiveArrays pFromByteArray = schema.newMessage();
    mergeFrom(data, 0, data.length, pFromByteArray, schema);
    assertEquals(p, pFromByteArray);

    PojoWithNonPrimitiveArrays pFromStream = schema.newMessage();
    ByteArrayInputStream in = new ByteArrayInputStream(data);
    mergeFrom(in, pFromStream, schema);
    assertEquals(p, pFromStream);

    roundTrip(p, schema, pipeSchema);
}
 
Example 21
Source Project: protostuff   Source File: NullArrayElementTest.java    License: Apache License 2.0 5 votes vote down vote up
public void testNullAll() throws IOException
{
    Schema<PojoWithNonPrimitiveArrays> schema =
            getSchema(PojoWithNonPrimitiveArrays.class);

    Pipe.Schema<PojoWithNonPrimitiveArrays> pipeSchema =
            ((RuntimeSchema<PojoWithNonPrimitiveArrays>) schema).getPipeSchema();

    PojoWithNonPrimitiveArrays p = new PojoWithNonPrimitiveArrays(
            new Boolean[] { null, null, null },
            new Character[] { null, null, null },
            new Short[] { null, null, null },
            new Integer[] { null, null, null },
            new Long[] { null, null, null },
            new Float[] { null, null, null },
            new Double[] { null, null, null },
            new String[] { null, null, null },
            new ByteString[] { null, null, null },
            new byte[][] { null, null, null },
            new BigDecimal[] { null, null, null },
            new BigInteger[] { null, null, null },
            new Date[] { null, null, null },
            new Size[] { null, null, null },
            new SomePojo[] { null, null, null }
            );

    byte[] data = toByteArray(p, schema);

    PojoWithNonPrimitiveArrays pFromByteArray = schema.newMessage();
    mergeFrom(data, 0, data.length, pFromByteArray, schema);
    assertEquals(p, pFromByteArray);

    PojoWithNonPrimitiveArrays pFromStream = schema.newMessage();
    ByteArrayInputStream in = new ByteArrayInputStream(data);
    mergeFrom(in, pFromStream, schema);
    assertEquals(p, pFromStream);

    roundTrip(p, schema, pipeSchema);
}
 
Example 22
Source Project: dremio-oss   Source File: TestUpgrade.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Test legal upgrade if prior dremio edition is not specified or editions match
 */
@Test
public void testLegalUpgrade() throws Exception {
  final ByteString prevEdition = ByteString.copyFrom("OSS".getBytes());
  final ConfigurationEntry configurationEntry = new ConfigurationEntry();
  configurationEntry.setValue(prevEdition);
  final LegacyKVStoreProvider kvStoreProvider = new LegacyKVStoreProviderAdapter(
    new LocalKVStoreProvider(CLASSPATH_SCAN_RESULT, null, true, false),
    CLASSPATH_SCAN_RESULT);
  kvStoreProvider.start();
  final ConfigurationStore configurationStore = new ConfigurationStore(kvStoreProvider);
  new Upgrade(DACConfig.newConfig(), CLASSPATH_SCAN_RESULT, false).validateUpgrade(kvStoreProvider, "OSS");
  configurationStore.put(SupportService.DREMIO_EDITION, configurationEntry);
  new Upgrade(DACConfig.newConfig(), CLASSPATH_SCAN_RESULT, false).validateUpgrade(kvStoreProvider, "OSS");
}
 
Example 23
Source Project: dremio-oss   Source File: DatasetResource.java    License: Apache License 2.0 5 votes vote down vote up
@GET
@Path("acceleration/settings")
@Produces(APPLICATION_JSON)
public AccelerationSettingsDescriptor getAccelerationSettings() throws NamespaceException {
  final DatasetConfig config = namespaceService.getDataset(datasetPath.toNamespaceKey());
  if (config.getType() == DatasetType.VIRTUAL_DATASET) {
    final String msg = String.format("acceleration settings apply only to physical dataset. %s is a virtual dataset",
        datasetPath.toPathString());
    throw new IllegalArgumentException(msg);
  }

  final ReflectionSettings reflectionSettings = reflectionServiceHelper.getReflectionSettings();
  final AccelerationSettings settings = reflectionSettings.getReflectionSettings(datasetPath.toNamespaceKey());
  final AccelerationSettingsDescriptor descriptor = new AccelerationSettingsDescriptor()
    .setAccelerationRefreshPeriod(settings.getRefreshPeriod())
    .setAccelerationGracePeriod(settings.getGracePeriod())
    .setMethod(settings.getMethod())
    .setRefreshField(settings.getRefreshField())
    .setAccelerationNeverExpire(settings.getNeverExpire())
    .setAccelerationNeverRefresh(settings.getNeverRefresh());

  final ByteString schemaBytes = DatasetHelper.getSchemaBytes(config);
  if (schemaBytes != null) {
    final BatchSchema schema = BatchSchema.deserialize(schemaBytes.toByteArray());
    descriptor.setFieldList(FluentIterable
        .from(schema)
        .transform(new Function<Field, String>() {
          @Nullable
          @Override
          public String apply(@Nullable final Field field) {
            return field.getName();
          }
        })
        .toList()
    );
  }
  return descriptor;
}
 
Example 24
Source Project: protostuff   Source File: NullArrayElementTest.java    License: Apache License 2.0 5 votes vote down vote up
public void testNullFirstAndLast() throws IOException
{
    Schema<PojoWithNonPrimitiveArrays> schema =
            getSchema(PojoWithNonPrimitiveArrays.class);

    Pipe.Schema<PojoWithNonPrimitiveArrays> pipeSchema =
            ((RuntimeSchema<PojoWithNonPrimitiveArrays>) schema).getPipeSchema();

    PojoWithNonPrimitiveArrays p = new PojoWithNonPrimitiveArrays(
            new Boolean[] { null, true, false, null },
            new Character[] { null, 'a', 'b', null },
            new Short[] { null, 1, 2, null },
            new Integer[] { null, 1, 2, null },
            new Long[] { null, 1l, 2l, null },
            new Float[] { null, 1.1f, 2.2f, null },
            new Double[] { null, 1.1d, 2.2d, null },
            new String[] { null, "a", "b", null },
            new ByteString[] { null, ByteString.copyFromUtf8("a"), ByteString.copyFromUtf8("b"), null },
            new byte[][] { null, new byte[] { 'a' }, new byte[] { 'b' }, null },
            new BigDecimal[] { null, new BigDecimal(1.1d), new BigDecimal(2.2d), null },
            new BigInteger[] { null, new BigInteger("1"), new BigInteger("2"), null },
            new Date[] { null, new Date(), new Date(), null },
            new Size[] { null, Size.MEDIUM, Size.LARGE, null },
            new SomePojo[] { null, new SomePojo("a"), new SomePojo("b"), null }
            );

    byte[] data = toByteArray(p, schema);

    PojoWithNonPrimitiveArrays pFromByteArray = schema.newMessage();
    mergeFrom(data, 0, data.length, pFromByteArray, schema);
    assertEquals(p, pFromByteArray);

    PojoWithNonPrimitiveArrays pFromStream = schema.newMessage();
    ByteArrayInputStream in = new ByteArrayInputStream(data);
    mergeFrom(in, pFromStream, schema);
    assertEquals(p, pFromStream);

    roundTrip(p, schema, pipeSchema);
}
 
Example 25
Source Project: dremio-oss   Source File: ElasticsearchSubScan.java    License: Apache License 2.0 5 votes vote down vote up
public ElasticsearchSubScan(
  OpProps props,
  StoragePluginId pluginId,
  ElasticsearchScanSpec spec,
  List<SplitAndPartitionInfo> splits,
  List<SchemaPath> columns,
  List<String> tableSchemaPath,
  BatchSchema fullSchema,
  ByteString extendedProperty){
  super(props, fullSchema, tableSchemaPath, columns);
  this.pluginId = pluginId;
  this.splits = splits;
  this.spec = spec;
  this.extendedProperty = extendedProperty;
}
 
Example 26
Source Project: dremio-oss   Source File: ElasticsearchSubScan.java    License: Apache License 2.0 5 votes vote down vote up
@JsonCreator
public ElasticsearchSubScan(
    @JsonProperty("props") OpProps props,
    @JsonProperty("pluginId") StoragePluginId pluginId,
    @JsonProperty("spec") ElasticsearchScanSpec spec,
    @JsonProperty("columns") List<SchemaPath> columns,
    @JsonProperty("tableSchemaPath") List<String> tableSchemaPath,
    @JsonProperty("fullSchema") BatchSchema fullSchema,
    @JsonProperty("extendedProperty") ByteString extendedProperty) {
  super(props, fullSchema, tableSchemaPath, columns);
  this.pluginId = pluginId;
  this.spec = spec;
  this.extendedProperty = extendedProperty;
}
 
Example 27
Source Project: protostuff-compiler   Source File: ScalarTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void scalarConstructedObject() throws Exception {
    ScalarFieldTestMsg msg = ScalarFieldTestMsg.newBuilder()
            .setDouble(0.1d)
            .setFloat(0.2f)
            .setInt32(3)
            .setInt64(4L)
            .setUnsignedInt32(5)
            .setUnsignedInt64(6L)
            .setSignedInt32(7)
            .setSignedInt64(8L)
            .setFixed32(9)
            .setFixed64(10L)
            .setSignedFixed32(11)
            .setSignedFixed64(12L)
            .setBool(true)
            .setString("string")
            .setBytes(ByteString.copyFrom(new byte[]{1, 2, 3}))
            .build();
    assertEquals(0.1d, msg.getDouble(), 0.000001d);
    assertEquals(0.2f, msg.getFloat(), 0.000001f);
    assertEquals(3, msg.getInt32());
    assertEquals(4L, msg.getInt64());
    assertEquals(5, msg.getUnsignedInt32());
    assertEquals(6L, msg.getUnsignedInt64());
    assertEquals(7, msg.getSignedInt32());
    assertEquals(8L, msg.getSignedInt64());
    assertEquals(9, msg.getFixed32());
    assertEquals(10L, msg.getFixed64());
    assertEquals(11, msg.getSignedFixed32());
    assertEquals(12L, msg.getSignedFixed64());
    assertEquals(true, msg.getBool());
    assertEquals("string", msg.getString());
    assertEquals(ByteString.copyFrom(new byte[]{1, 2, 3}), msg.getBytes());
}
 
Example 28
Source Project: protostuff   Source File: ArraySchemas.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Object readFrom(Input input, Object owner) throws IOException
{
    if (ID_ARRAY_LEN != input.readFieldNumber(this))
        throw new ProtostuffException("Corrupt input.");

    final int len = input.readInt32();

    ByteString[] array = new ByteString[len];
    if (input instanceof GraphInput)
    {
        // update the actual reference.
        ((GraphInput) input).updateLast(array, owner);
    }

    for (int i = 0; i < len;)
    {
        switch (input.readFieldNumber(this))
        {
            case ID_ARRAY_DATA:
                array[i++] = input.readBytes();
                break;
            case ID_ARRAY_NULLCOUNT:
                i += input.readUInt32();
                break;
            default:
                throw new ProtostuffException("Corrupt input.");
        }
    }

    if (0 != input.readFieldNumber(this))
        throw new ProtostuffException("Corrupt input.");

    return array;
}
 
Example 29
Source Project: protostuff   Source File: ArraySchemas.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void writeTo(Output output, Object value) throws IOException
{
    ByteString[] array = (ByteString[]) value;
    output.writeInt32(ID_ARRAY_LEN, array.length, false);

    int nullCount = 0;
    for (int i = 0, len = array.length; i < len; i++)
    {
        ByteString v = array[i];
        if (v != null)
        {
            if (nullCount != 0)
            {
                output.writeUInt32(ID_ARRAY_NULLCOUNT, nullCount, false);
                nullCount = 0;
            }

            output.writeBytes(ID_ARRAY_DATA, v, true);
        }
        else if (allowNullArrayElement)
        {
            nullCount++;
        }
    }

    // if last element is null
    if (nullCount != 0)
        output.writeUInt32(ID_ARRAY_NULLCOUNT, nullCount, false);
}
 
Example 30
Source Project: protostuff   Source File: NullArrayElementTest.java    License: Apache License 2.0 5 votes vote down vote up
public void testNullFirst() throws IOException
{
    Schema<PojoWithNonPrimitiveArrays> schema =
            getSchema(PojoWithNonPrimitiveArrays.class);

    Pipe.Schema<PojoWithNonPrimitiveArrays> pipeSchema =
            ((RuntimeSchema<PojoWithNonPrimitiveArrays>) schema).getPipeSchema();

    PojoWithNonPrimitiveArrays p = new PojoWithNonPrimitiveArrays(
            new Boolean[] { null, true, false },
            new Character[] { null, 'a', 'b' },
            new Short[] { null, 1, 2 },
            new Integer[] { null, 1, 2 },
            new Long[] { null, 1l, 2l },
            new Float[] { null, 1.1f, 2.2f },
            new Double[] { null, 1.1d, 2.2d },
            new String[] { null, "a", "b" },
            new ByteString[] { null, ByteString.copyFromUtf8("a"), ByteString.copyFromUtf8("b") },
            new byte[][] { null, new byte[] { 'a' }, new byte[] { 'b' } },
            new BigDecimal[] { null, new BigDecimal(1.1d), new BigDecimal(2.2d) },
            new BigInteger[] { null, new BigInteger("1"), new BigInteger("2") },
            new Date[] { null, new Date(), new Date() },
            new Size[] { null, Size.MEDIUM, Size.LARGE },
            new SomePojo[] { null, new SomePojo("a"), new SomePojo("b") }
            );

    byte[] data = toByteArray(p, schema);

    PojoWithNonPrimitiveArrays pFromByteArray = schema.newMessage();
    mergeFrom(data, 0, data.length, pFromByteArray, schema);
    assertEquals(p, pFromByteArray);

    PojoWithNonPrimitiveArrays pFromStream = schema.newMessage();
    ByteArrayInputStream in = new ByteArrayInputStream(data);
    mergeFrom(in, pFromStream, schema);
    assertEquals(p, pFromStream);

    roundTrip(p, schema, pipeSchema);
}