io.protostuff.ByteString Java Examples

The following examples show how to use io.protostuff.ByteString. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestConfigurationStore.java    From dremio-oss with Apache License 2.0 6 votes vote down vote up
@Test
public void testStore() throws Exception {
  try(final LegacyKVStoreProvider kvstore =
    new LocalKVStoreProvider(DremioTest.CLASSPATH_SCAN_RESULT, null, true, false).asLegacy()) {
    kvstore.start();
    ConfigurationStore store = new ConfigurationStore(kvstore);

    ConfigurationEntry supportEntry = new ConfigurationEntry();
    supportEntry.setType("mytype");
    supportEntry.setValue(ByteString.copyFrom("test string", "UTF8"));
    store.put("key", supportEntry);

    ConfigurationEntry retrieved = store.get("key");
    Assert.assertEquals(retrieved.getType(), supportEntry.getType());
    Assert.assertEquals(retrieved.getValue(), supportEntry.getValue());

    store.delete("key");
    retrieved = store.get("key");
    Assert.assertEquals(retrieved, null);
  }
}
 
Example #2
Source File: WriterOptions.java    From dremio-oss with Apache License 2.0 6 votes vote down vote up
@JsonCreator
public WriterOptions(
  @JsonProperty("ringCount") Integer ringCount,
  @JsonProperty("partitionColumns") List<String> partitionColumns,
  @JsonProperty("sortColumns") List<String> sortColumns,
  @JsonProperty("distributionColumns") List<String> distributionColumns,
  @JsonProperty("partitionDistributionStrategy") PartitionDistributionStrategy partitionDistributionStrategy,
  @JsonProperty("singleWriter") boolean singleWriter,
  @JsonProperty("recordLimit") long recordLimit,
  @JsonProperty("icebergWriterOperation") IcebergWriterOperation icebergWriterOperation,
  @JsonProperty("extendedProperty") ByteString extendedProperty
  ) {
  this.ringCount = ringCount;
  this.partitionColumns = partitionColumns;
  this.sortColumns = sortColumns;
  this.distributionColumns = distributionColumns;
  this.partitionDistributionStrategy = partitionDistributionStrategy;
  this.singleWriter = singleWriter;
  this.recordLimit = recordLimit;
  this.icebergWriterOperation = icebergWriterOperation;
  this.extendedProperty = extendedProperty;
}
 
Example #3
Source File: FileFormat.java    From dremio-oss with Apache License 2.0 6 votes vote down vote up
@JsonIgnore
@SuppressWarnings({ "rawtypes", "unchecked" })
public FileConfig asFileConfig() {
  buffer.clear();
  FileConfig fc = new FileConfig();
  fc.setType(getFileType());
  fc.setName(name);
  fc.setOwner(owner);
  fc.setCtime(ctime);
  fc.setType(getFileType());
  fc.setTag(getVersion());
  fc.setLocation(location);
  byte[] bytes = ProtobufIOUtil.toByteArray(this, (Schema) getPrivateSchema(), buffer);
  fc.setExtendedConfig(ByteString.copyFrom(bytes));
  fc.setFullPathList(fullPath);
  return fc;
}
 
Example #4
Source File: DatasetRetrievalOptions.java    From dremio-oss with Apache License 2.0 6 votes vote down vote up
private <T extends MetadataOption> void addDatasetOptions(Class<T> clazz, DatasetConfig datasetConfig, List<T> outOptions) {
  if(datasetConfig == null) {
    return;
  }

  List<MetadataOption> options = new ArrayList<>();
  if(datasetConfig.getPhysicalDataset() != null && datasetConfig.getPhysicalDataset().getFormatSettings() != null) {
    options.add(new FileConfigOption(datasetConfig.getPhysicalDataset().getFormatSettings()));
  }

  if(datasetConfig.getReadDefinition() != null && datasetConfig.getReadDefinition().getSortColumnsList() != null) {
    options.add(new SortColumnsOption(datasetConfig.getReadDefinition().getSortColumnsList()));
  }

  BatchSchema schema = DatasetHelper.getSchemaBytes(datasetConfig) != null ? CalciteArrowHelper.fromDataset(datasetConfig) : null;
  if(schema != null) {
    options.add(new CurrentSchemaOption(schema));
  }

  if (datasetConfig.getReadDefinition() != null && datasetConfig.getReadDefinition().getExtendedProperty() != null) {
    options.add(new ExtendedPropertyOption(os -> ByteString.writeTo(os, datasetConfig.getReadDefinition().getExtendedProperty())));
  }

  List<T> addOptions = options.stream().filter(o -> clazz.isAssignableFrom(o.getClass())).map(o -> clazz.cast(o)).collect(Collectors.toList());
  outOptions.addAll(addOptions);
}
 
Example #5
Source File: TestHomeFiles.java    From dremio-oss with Apache License 2.0 6 votes vote down vote up
@Test
public void testNASFileStore() throws Exception {

  final CatalogServiceImpl catalog = (CatalogServiceImpl) l(CatalogService.class);
  final SourceConfig config = catalog.getManagedSource(HomeFileSystemStoragePlugin.HOME_PLUGIN_NAME).getId().getClonedConfig();
  final ByteString oldConfig = config.getConfig();
  final HomeFileConf nasHomeFileStore = new HomeFileConf(Path.of("file:///" + BaseTestServer.folder1.getRoot().toString() + "/" + "testNASFileStore/").toString(), "localhost");
  nasHomeFileStore.getFilesystemAndCreatePaths("localhost");
  config.setConnectionConf(nasHomeFileStore);
  catalog.getSystemUserCatalog().updateSource(config);
  HomeFileTool tool = l(HomeFileTool.class);

  try {
    runTests(nasHomeFileStore);
  } finally {
    tool.clear();
    // reset plugin
    SourceConfig backConfig = catalog.getManagedSource(HomeFileSystemStoragePlugin.HOME_PLUGIN_NAME).getId().getClonedConfig();
    backConfig.setConfig(oldConfig);
    catalog.getSystemUserCatalog().updateSource(backConfig);
  }
}
 
Example #6
Source File: DatasetsUtil.java    From dremio-oss with Apache License 2.0 6 votes vote down vote up
public static List<org.apache.arrow.vector.types.pojo.Field> getArrowFieldsFromDatasetConfig(DatasetConfig datasetConfig) {
  List<org.apache.arrow.vector.types.pojo.Field> fields = Lists.newArrayList();
  final ByteString schemaBytes = DatasetHelper.getSchemaBytes(datasetConfig);
  if (schemaBytes == null) {
    return null;
  }

  final BatchSchema batchSchema = BatchSchema.deserialize(schemaBytes);
  for (int i = 0; i < batchSchema.getFieldCount(); i++) {
    final org.apache.arrow.vector.types.pojo.Field field = batchSchema.getColumn(i);
    if (!NamespaceTable.SYSTEM_COLUMNS.contains(field.getName())) {
      fields.add(field);
    }
  }

  return fields;
}
 
Example #7
Source File: ParquetSubScan.java    From dremio-oss with Apache License 2.0 6 votes vote down vote up
public ParquetSubScan(
  OpProps props,
  FileConfig formatSettings,
  List<SplitAndPartitionInfo> splits,
  BatchSchema fullSchema,
  List<List<String>> tablePath,
  List<ParquetFilterCondition> conditions,
  StoragePluginId pluginId,
  List<SchemaPath> columns,
  List<String> partitionColumns,
  List<GlobalDictionaryFieldInfo> globalDictionaryEncodedColumns,
  ByteString extendedProperty
) {
  super(props, fullSchema, tablePath, columns);
  this.formatSettings = formatSettings;
  this.splits = splits;
  this.tablePath = tablePath;
  this.conditions = conditions == null ? null : ImmutableList.copyOf(conditions);
  this.pluginId = pluginId;
  this.partitionColumns = partitionColumns;
  this.globalDictionaryEncodedColumns = globalDictionaryEncodedColumns;
  this.extendedProperty = extendedProperty;
}
 
Example #8
Source File: ParquetSubScan.java    From dremio-oss with Apache License 2.0 6 votes vote down vote up
@JsonCreator
public ParquetSubScan(
  @JsonProperty("props") OpProps props,
  @JsonProperty("formatSettings") FileConfig formatSettings,
  @JsonProperty("schema") BatchSchema fullSchema,
  @JsonProperty("referencedTables") List<List<String>> tablePath,
  @JsonProperty("conditions") List<ParquetFilterCondition> conditions,
  @JsonProperty("pluginId") StoragePluginId pluginId,
  @JsonProperty("columns") List<SchemaPath> columns,
  @JsonProperty("partitionColumns") List<String> partitionColumns,
  @JsonProperty("globalDictionaryEncodedColumns") List<GlobalDictionaryFieldInfo> globalDictionaryEncodedColumns,
  @JsonProperty("extendedProperty") ByteString extendedProperty) {

  this(props, formatSettings, null, fullSchema, tablePath, conditions, pluginId, columns, partitionColumns,
    globalDictionaryEncodedColumns, extendedProperty);
}
 
Example #9
Source File: NamespaceConverter.java    From dremio-oss with Apache License 2.0 6 votes vote down vote up
public static String[] getColumnsLowerCase(DatasetConfig datasetConfig) {
  final ByteString schemaBytes = DatasetHelper.getSchemaBytes(datasetConfig);
  if (schemaBytes != null) {
    Schema schema = Schema.getRootAsSchema(schemaBytes.asReadOnlyByteBuffer());
    org.apache.arrow.vector.types.pojo.Schema s = org.apache.arrow.vector.types.pojo.Schema.convertSchema(schema);
    return s.getFields().stream().map(input -> input.getName().toLowerCase()).toArray(String[]::new);
  } else {
    // If virtual dataset was created with view fields
    if (datasetConfig.getType() == DatasetType.VIRTUAL_DATASET) {
      final List<ViewFieldType> viewFieldTypes = datasetConfig.getVirtualDataset().getSqlFieldsList();
      if (notEmpty(viewFieldTypes)) {
        return viewFieldTypes.stream().map(input -> input.getName().toLowerCase()).toArray(String[]::new);
      }
    }
  }

  return new String[0];
}
 
Example #10
Source File: EasySubScan.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up
@JsonCreator
public EasySubScan(
  @JsonProperty("props") OpProps props,
  @JsonProperty("settings") FileConfig config,
  @JsonProperty("fullSchema") BatchSchema fullSchema,
  @JsonProperty("tableSchemaPath") List<String> tablePath,
  @JsonProperty("pluginId") StoragePluginId pluginId,
  @JsonProperty("columns") List<SchemaPath> columns,
  @JsonProperty("partitionColumns") List<String> partitionColumns,
  @JsonProperty("extendedProperty") ByteString extendedProperty) {

  this(props, config, null, fullSchema, tablePath, pluginId, columns, partitionColumns, extendedProperty);
}
 
Example #11
Source File: MaterializationCache.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up
private boolean schemaChanged(MaterializationDescriptor old, Materialization materialization) {
  if (namespaceService == null) {
    return false;
  }
  try {
    //TODO is this enough ? shouldn't we use the dataset hash instead ??
    final NamespaceKey matKey = new NamespaceKey(ReflectionUtils.getMaterializationPath(materialization));
    ByteString schemaString = namespaceService.getDataset(matKey).getRecordSchema();
    BatchSchema newSchema = BatchSchema.deserialize(schemaString);
    BatchSchema oldSchema = ((CachedMaterializationDescriptor) old).getMaterialization().getSchema();
    return !oldSchema.equals(newSchema);
  } catch (NamespaceException e) {
    return true;
  }
}
 
Example #12
Source File: Bar.java    From protostuff with Apache License 2.0 5 votes vote down vote up
public Bar(int someInt, String someString, Baz baz, Status someEnum,
        ByteString someBytes, boolean someBoolean, float someFloat,
        double someDouble, long someLong)
{
    this.someInt = someInt;
    this.someString = someString;
    this.someBaz = baz;
    this.someEnum = someEnum;
    this.someBytes = someBytes;
    this.someBoolean = someBoolean;
    this.someFloat = someFloat;
    this.someDouble = someDouble;
    this.someLong = someLong;
}
 
Example #13
Source File: ParquetRecordWriter.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up
private void initIcebergColumnIDList(ByteString extendedProperty) {
  try {
    IcebergProtobuf.IcebergDatasetXAttr icebergDatasetXAttr = LegacyProtobufSerializer.parseFrom(IcebergProtobuf.IcebergDatasetXAttr.PARSER,
      extendedProperty.toByteArray());
    List<IcebergProtobuf.IcebergSchemaField> icebergColumnIDs = icebergDatasetXAttr.getColumnIdsList();
    Map<String, Integer> icebergColumns = new HashMap<>();
    icebergColumnIDs.forEach(field -> icebergColumns.put(field.getSchemaPath(), field.getId()));
    this.icebergColumnIDMap = CaseInsensitiveImmutableBiMap.newImmutableMap(icebergColumns);
  } catch (InvalidProtocolBufferException e) {
    throw new RuntimeException("Could not deserialize Parquet dataset info", e);
  }
}
 
Example #14
Source File: NullArrayElementTest.java    From protostuff with Apache License 2.0 5 votes vote down vote up
public void testNullMid() throws IOException
{
    Schema<PojoWithNonPrimitiveArrays> schema =
            getSchema(PojoWithNonPrimitiveArrays.class);

    Pipe.Schema<PojoWithNonPrimitiveArrays> pipeSchema =
            ((RuntimeSchema<PojoWithNonPrimitiveArrays>) schema).getPipeSchema();

    PojoWithNonPrimitiveArrays p = new PojoWithNonPrimitiveArrays(
            new Boolean[] { true, null, false },
            new Character[] { 'a', null, 'b' },
            new Short[] { 1, null, 2 },
            new Integer[] { 1, null, 2 },
            new Long[] { 1l, null, 2l },
            new Float[] { 1.1f, null, 2.2f },
            new Double[] { 1.1d, null, 2.2d },
            new String[] { "a", null, "b" },
            new ByteString[] { ByteString.copyFromUtf8("a"), null, ByteString.copyFromUtf8("b") },
            new byte[][] { new byte[] { 'a' }, null, new byte[] { 'b' } },
            new BigDecimal[] { new BigDecimal(1.1d), null, new BigDecimal(2.2d) },
            new BigInteger[] { new BigInteger("1"), null, new BigInteger("2") },
            new Date[] { new Date(), null, new Date() },
            new Size[] { Size.MEDIUM, null, Size.LARGE },
            new SomePojo[] { new SomePojo("a"), null, new SomePojo("b") }
            );

    byte[] data = toByteArray(p, schema);

    PojoWithNonPrimitiveArrays pFromByteArray = schema.newMessage();
    mergeFrom(data, 0, data.length, pFromByteArray, schema);
    assertEquals(p, pFromByteArray);

    PojoWithNonPrimitiveArrays pFromStream = schema.newMessage();
    ByteArrayInputStream in = new ByteArrayInputStream(data);
    mergeFrom(in, pFromStream, schema);
    assertEquals(p, pFromStream);

    roundTrip(p, schema, pipeSchema);
}
 
Example #15
Source File: ViewFieldsHelper.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up
/**
 * Get view fields from dataset config as it is in the batch schema (if not present, defaults to reltree's row type)
 * @param config dataset config
 * @return List of view fields
 */
public static List<ViewFieldType> getViewFields(DatasetConfig config) {
  final ByteString schemaBytes = DatasetHelper.getSchemaBytes(config);
  final boolean virtualDataset = config.getType() == DatasetType.VIRTUAL_DATASET;
  if (schemaBytes == null && virtualDataset) {
    return config.getVirtualDataset().getSqlFieldsList();
  }
  if (schemaBytes != null) {
    return getBatchSchemaFields(BatchSchema.deserialize(schemaBytes.toByteArray()));
  }
  return null;
}
 
Example #16
Source File: CalciteArrowHelper.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up
public static BatchSchema fromDataset(DatasetConfig config){
  try{
    ByteString bytes = DatasetHelper.getSchemaBytes(config);
    if(bytes == null){
      throw new IllegalStateException(String.format("Schema is currently unavailable for dataset %s.", PathUtils.constructFullPath(config.getFullPathList())));
    }
    return BatchSchema.deserialize(bytes);
  }catch(Exception ex){
    throw new IllegalStateException(String.format("Schema for dataset %s is corrupt.", PathUtils.constructFullPath(config.getFullPathList())), ex);
  }
}
 
Example #17
Source File: NullArrayElementTest.java    From protostuff with Apache License 2.0 5 votes vote down vote up
public void testNullLast() throws IOException
{
    Schema<PojoWithNonPrimitiveArrays> schema =
            getSchema(PojoWithNonPrimitiveArrays.class);

    Pipe.Schema<PojoWithNonPrimitiveArrays> pipeSchema =
            ((RuntimeSchema<PojoWithNonPrimitiveArrays>) schema).getPipeSchema();

    PojoWithNonPrimitiveArrays p = new PojoWithNonPrimitiveArrays(
            new Boolean[] { true, false, null },
            new Character[] { 'a', 'b', null },
            new Short[] { 1, 2, null },
            new Integer[] { 1, 2, null },
            new Long[] { 1l, 2l, null },
            new Float[] { 1.1f, 2.2f, null },
            new Double[] { 1.1d, 2.2d, null },
            new String[] { "a", "b", null },
            new ByteString[] { ByteString.copyFromUtf8("a"), ByteString.copyFromUtf8("b"), null },
            new byte[][] { new byte[] { 'a' }, new byte[] { 'b' }, null },
            new BigDecimal[] { new BigDecimal(1.1d), new BigDecimal(2.2d), null },
            new BigInteger[] { new BigInteger("1"), new BigInteger("2"), null },
            new Date[] { new Date(), new Date(), null },
            new Size[] { Size.MEDIUM, Size.LARGE, null },
            new SomePojo[] { new SomePojo("a"), new SomePojo("b"), null }
            );

    byte[] data = toByteArray(p, schema);

    PojoWithNonPrimitiveArrays pFromByteArray = schema.newMessage();
    mergeFrom(data, 0, data.length, pFromByteArray, schema);
    assertEquals(p, pFromByteArray);

    PojoWithNonPrimitiveArrays pFromStream = schema.newMessage();
    ByteArrayInputStream in = new ByteArrayInputStream(data);
    mergeFrom(in, pFromStream, schema);
    assertEquals(p, pFromStream);

    roundTrip(p, schema, pipeSchema);
}
 
Example #18
Source File: Foo.java    From protostuff with Apache License 2.0 5 votes vote down vote up
public Foo(List<Integer> someInt, List<String> someString,
        List<Bar> someBar, List<EnumSample> someEnum,
        List<ByteString> someBytes, List<Boolean> someBoolean,
        List<Float> someFloat, List<Double> someDouble, List<Long> someLong)
{
    this.someInt = someInt;
    this.someString = someString;
    this.someBar = someBar;
    this.someEnum = someEnum;
    this.someBytes = someBytes;
    this.someBoolean = someBoolean;
    this.someFloat = someFloat;
    this.someDouble = someDouble;
    this.someLong = someLong;
}
 
Example #19
Source File: LocalJobsService.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up
@Override
public void recordExtraInfo(String name, byte[] bytes) {
  //TODO DX-10977 the reflection manager should rely on its own observer to store this information in a separate store
  if(job.getJobAttempt().getExtraInfoList() == null) {
    job.getJobAttempt().setExtraInfoList(new ArrayList<ExtraInfo>());
  }

  job.getJobAttempt().getExtraInfoList().add(new ExtraInfo()
      .setData(ByteString.copyFrom(bytes))
      .setName(name));
  storeJob(job);
  super.recordExtraInfo(name, bytes);
}
 
Example #20
Source File: NullArrayElementTest.java    From protostuff with Apache License 2.0 5 votes vote down vote up
public void testNullFirst() throws IOException
{
    Schema<PojoWithNonPrimitiveArrays> schema =
            getSchema(PojoWithNonPrimitiveArrays.class);

    Pipe.Schema<PojoWithNonPrimitiveArrays> pipeSchema =
            ((RuntimeSchema<PojoWithNonPrimitiveArrays>) schema).getPipeSchema();

    PojoWithNonPrimitiveArrays p = new PojoWithNonPrimitiveArrays(
            new Boolean[] { null, true, false },
            new Character[] { null, 'a', 'b' },
            new Short[] { null, 1, 2 },
            new Integer[] { null, 1, 2 },
            new Long[] { null, 1l, 2l },
            new Float[] { null, 1.1f, 2.2f },
            new Double[] { null, 1.1d, 2.2d },
            new String[] { null, "a", "b" },
            new ByteString[] { null, ByteString.copyFromUtf8("a"), ByteString.copyFromUtf8("b") },
            new byte[][] { null, new byte[] { 'a' }, new byte[] { 'b' } },
            new BigDecimal[] { null, new BigDecimal(1.1d), new BigDecimal(2.2d) },
            new BigInteger[] { null, new BigInteger("1"), new BigInteger("2") },
            new Date[] { null, new Date(), new Date() },
            new Size[] { null, Size.MEDIUM, Size.LARGE },
            new SomePojo[] { null, new SomePojo("a"), new SomePojo("b") }
            );

    byte[] data = toByteArray(p, schema);

    PojoWithNonPrimitiveArrays pFromByteArray = schema.newMessage();
    mergeFrom(data, 0, data.length, pFromByteArray, schema);
    assertEquals(p, pFromByteArray);

    PojoWithNonPrimitiveArrays pFromStream = schema.newMessage();
    ByteArrayInputStream in = new ByteArrayInputStream(data);
    mergeFrom(in, pFromStream, schema);
    assertEquals(p, pFromStream);

    roundTrip(p, schema, pipeSchema);
}
 
Example #21
Source File: NullArrayElementTest.java    From protostuff with Apache License 2.0 5 votes vote down vote up
public void testNullAll() throws IOException
{
    Schema<PojoWithNonPrimitiveArrays> schema =
            getSchema(PojoWithNonPrimitiveArrays.class);

    Pipe.Schema<PojoWithNonPrimitiveArrays> pipeSchema =
            ((RuntimeSchema<PojoWithNonPrimitiveArrays>) schema).getPipeSchema();

    PojoWithNonPrimitiveArrays p = new PojoWithNonPrimitiveArrays(
            new Boolean[] { null, null, null },
            new Character[] { null, null, null },
            new Short[] { null, null, null },
            new Integer[] { null, null, null },
            new Long[] { null, null, null },
            new Float[] { null, null, null },
            new Double[] { null, null, null },
            new String[] { null, null, null },
            new ByteString[] { null, null, null },
            new byte[][] { null, null, null },
            new BigDecimal[] { null, null, null },
            new BigInteger[] { null, null, null },
            new Date[] { null, null, null },
            new Size[] { null, null, null },
            new SomePojo[] { null, null, null }
            );

    byte[] data = toByteArray(p, schema);

    PojoWithNonPrimitiveArrays pFromByteArray = schema.newMessage();
    mergeFrom(data, 0, data.length, pFromByteArray, schema);
    assertEquals(p, pFromByteArray);

    PojoWithNonPrimitiveArrays pFromStream = schema.newMessage();
    ByteArrayInputStream in = new ByteArrayInputStream(data);
    mergeFrom(in, pFromStream, schema);
    assertEquals(p, pFromStream);

    roundTrip(p, schema, pipeSchema);
}
 
Example #22
Source File: TestUpgrade.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up
/**
 * Test legal upgrade if prior dremio edition is not specified or editions match
 */
@Test
public void testLegalUpgrade() throws Exception {
  final ByteString prevEdition = ByteString.copyFrom("OSS".getBytes());
  final ConfigurationEntry configurationEntry = new ConfigurationEntry();
  configurationEntry.setValue(prevEdition);
  final LegacyKVStoreProvider kvStoreProvider = new LegacyKVStoreProviderAdapter(
    new LocalKVStoreProvider(CLASSPATH_SCAN_RESULT, null, true, false),
    CLASSPATH_SCAN_RESULT);
  kvStoreProvider.start();
  final ConfigurationStore configurationStore = new ConfigurationStore(kvStoreProvider);
  new Upgrade(DACConfig.newConfig(), CLASSPATH_SCAN_RESULT, false).validateUpgrade(kvStoreProvider, "OSS");
  configurationStore.put(SupportService.DREMIO_EDITION, configurationEntry);
  new Upgrade(DACConfig.newConfig(), CLASSPATH_SCAN_RESULT, false).validateUpgrade(kvStoreProvider, "OSS");
}
 
Example #23
Source File: DatasetResource.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up
@GET
@Path("acceleration/settings")
@Produces(APPLICATION_JSON)
public AccelerationSettingsDescriptor getAccelerationSettings() throws NamespaceException {
  final DatasetConfig config = namespaceService.getDataset(datasetPath.toNamespaceKey());
  if (config.getType() == DatasetType.VIRTUAL_DATASET) {
    final String msg = String.format("acceleration settings apply only to physical dataset. %s is a virtual dataset",
        datasetPath.toPathString());
    throw new IllegalArgumentException(msg);
  }

  final ReflectionSettings reflectionSettings = reflectionServiceHelper.getReflectionSettings();
  final AccelerationSettings settings = reflectionSettings.getReflectionSettings(datasetPath.toNamespaceKey());
  final AccelerationSettingsDescriptor descriptor = new AccelerationSettingsDescriptor()
    .setAccelerationRefreshPeriod(settings.getRefreshPeriod())
    .setAccelerationGracePeriod(settings.getGracePeriod())
    .setMethod(settings.getMethod())
    .setRefreshField(settings.getRefreshField())
    .setAccelerationNeverExpire(settings.getNeverExpire())
    .setAccelerationNeverRefresh(settings.getNeverRefresh());

  final ByteString schemaBytes = DatasetHelper.getSchemaBytes(config);
  if (schemaBytes != null) {
    final BatchSchema schema = BatchSchema.deserialize(schemaBytes.toByteArray());
    descriptor.setFieldList(FluentIterable
        .from(schema)
        .transform(new Function<Field, String>() {
          @Nullable
          @Override
          public String apply(@Nullable final Field field) {
            return field.getName();
          }
        })
        .toList()
    );
  }
  return descriptor;
}
 
Example #24
Source File: NullArrayElementTest.java    From protostuff with Apache License 2.0 5 votes vote down vote up
public void testNullFirstAndLast() throws IOException
{
    Schema<PojoWithNonPrimitiveArrays> schema =
            getSchema(PojoWithNonPrimitiveArrays.class);

    Pipe.Schema<PojoWithNonPrimitiveArrays> pipeSchema =
            ((RuntimeSchema<PojoWithNonPrimitiveArrays>) schema).getPipeSchema();

    PojoWithNonPrimitiveArrays p = new PojoWithNonPrimitiveArrays(
            new Boolean[] { null, true, false, null },
            new Character[] { null, 'a', 'b', null },
            new Short[] { null, 1, 2, null },
            new Integer[] { null, 1, 2, null },
            new Long[] { null, 1l, 2l, null },
            new Float[] { null, 1.1f, 2.2f, null },
            new Double[] { null, 1.1d, 2.2d, null },
            new String[] { null, "a", "b", null },
            new ByteString[] { null, ByteString.copyFromUtf8("a"), ByteString.copyFromUtf8("b"), null },
            new byte[][] { null, new byte[] { 'a' }, new byte[] { 'b' }, null },
            new BigDecimal[] { null, new BigDecimal(1.1d), new BigDecimal(2.2d), null },
            new BigInteger[] { null, new BigInteger("1"), new BigInteger("2"), null },
            new Date[] { null, new Date(), new Date(), null },
            new Size[] { null, Size.MEDIUM, Size.LARGE, null },
            new SomePojo[] { null, new SomePojo("a"), new SomePojo("b"), null }
            );

    byte[] data = toByteArray(p, schema);

    PojoWithNonPrimitiveArrays pFromByteArray = schema.newMessage();
    mergeFrom(data, 0, data.length, pFromByteArray, schema);
    assertEquals(p, pFromByteArray);

    PojoWithNonPrimitiveArrays pFromStream = schema.newMessage();
    ByteArrayInputStream in = new ByteArrayInputStream(data);
    mergeFrom(in, pFromStream, schema);
    assertEquals(p, pFromStream);

    roundTrip(p, schema, pipeSchema);
}
 
Example #25
Source File: ElasticsearchSubScan.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up
public ElasticsearchSubScan(
  OpProps props,
  StoragePluginId pluginId,
  ElasticsearchScanSpec spec,
  List<SplitAndPartitionInfo> splits,
  List<SchemaPath> columns,
  List<String> tableSchemaPath,
  BatchSchema fullSchema,
  ByteString extendedProperty){
  super(props, fullSchema, tableSchemaPath, columns);
  this.pluginId = pluginId;
  this.splits = splits;
  this.spec = spec;
  this.extendedProperty = extendedProperty;
}
 
Example #26
Source File: ElasticsearchSubScan.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up
@JsonCreator
public ElasticsearchSubScan(
    @JsonProperty("props") OpProps props,
    @JsonProperty("pluginId") StoragePluginId pluginId,
    @JsonProperty("spec") ElasticsearchScanSpec spec,
    @JsonProperty("columns") List<SchemaPath> columns,
    @JsonProperty("tableSchemaPath") List<String> tableSchemaPath,
    @JsonProperty("fullSchema") BatchSchema fullSchema,
    @JsonProperty("extendedProperty") ByteString extendedProperty) {
  super(props, fullSchema, tableSchemaPath, columns);
  this.pluginId = pluginId;
  this.spec = spec;
  this.extendedProperty = extendedProperty;
}
 
Example #27
Source File: ScalarTest.java    From protostuff-compiler with Apache License 2.0 5 votes vote down vote up
@Test
public void scalarConstructedObject() throws Exception {
    ScalarFieldTestMsg msg = ScalarFieldTestMsg.newBuilder()
            .setDouble(0.1d)
            .setFloat(0.2f)
            .setInt32(3)
            .setInt64(4L)
            .setUnsignedInt32(5)
            .setUnsignedInt64(6L)
            .setSignedInt32(7)
            .setSignedInt64(8L)
            .setFixed32(9)
            .setFixed64(10L)
            .setSignedFixed32(11)
            .setSignedFixed64(12L)
            .setBool(true)
            .setString("string")
            .setBytes(ByteString.copyFrom(new byte[]{1, 2, 3}))
            .build();
    assertEquals(0.1d, msg.getDouble(), 0.000001d);
    assertEquals(0.2f, msg.getFloat(), 0.000001f);
    assertEquals(3, msg.getInt32());
    assertEquals(4L, msg.getInt64());
    assertEquals(5, msg.getUnsignedInt32());
    assertEquals(6L, msg.getUnsignedInt64());
    assertEquals(7, msg.getSignedInt32());
    assertEquals(8L, msg.getSignedInt64());
    assertEquals(9, msg.getFixed32());
    assertEquals(10L, msg.getFixed64());
    assertEquals(11, msg.getSignedFixed32());
    assertEquals(12L, msg.getSignedFixed64());
    assertEquals(true, msg.getBool());
    assertEquals("string", msg.getString());
    assertEquals(ByteString.copyFrom(new byte[]{1, 2, 3}), msg.getBytes());
}
 
Example #28
Source File: ArraySchemas.java    From protostuff with Apache License 2.0 5 votes vote down vote up
@Override
public Object readFrom(Input input, Object owner) throws IOException
{
    if (ID_ARRAY_LEN != input.readFieldNumber(this))
        throw new ProtostuffException("Corrupt input.");

    final int len = input.readInt32();

    ByteString[] array = new ByteString[len];
    if (input instanceof GraphInput)
    {
        // update the actual reference.
        ((GraphInput) input).updateLast(array, owner);
    }

    for (int i = 0; i < len;)
    {
        switch (input.readFieldNumber(this))
        {
            case ID_ARRAY_DATA:
                array[i++] = input.readBytes();
                break;
            case ID_ARRAY_NULLCOUNT:
                i += input.readUInt32();
                break;
            default:
                throw new ProtostuffException("Corrupt input.");
        }
    }

    if (0 != input.readFieldNumber(this))
        throw new ProtostuffException("Corrupt input.");

    return array;
}
 
Example #29
Source File: ArraySchemas.java    From protostuff with Apache License 2.0 5 votes vote down vote up
@Override
public void writeTo(Output output, Object value) throws IOException
{
    ByteString[] array = (ByteString[]) value;
    output.writeInt32(ID_ARRAY_LEN, array.length, false);

    int nullCount = 0;
    for (int i = 0, len = array.length; i < len; i++)
    {
        ByteString v = array[i];
        if (v != null)
        {
            if (nullCount != 0)
            {
                output.writeUInt32(ID_ARRAY_NULLCOUNT, nullCount, false);
                nullCount = 0;
            }

            output.writeBytes(ID_ARRAY_DATA, v, true);
        }
        else if (allowNullArrayElement)
        {
            nullCount++;
        }
    }

    // if last element is null
    if (nullCount != 0)
        output.writeUInt32(ID_ARRAY_NULLCOUNT, nullCount, false);
}
 
Example #30
Source File: UnsafeNioBufInput.java    From sofa-jraft with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("all")
@Override
public ByteString readBytes() throws IOException {
    try {
        return (ByteString) byteStringWrapMethod.invoke(null, readByteArray());
    } catch (Exception e) {
        ThrowUtil.throwException(e);
    }
    return null; // never get here
}