org.kitesdk.data.IncompatibleSchemaException Java Examples

@Test
public void testReadNullsWithPrimitivesAllowNullSchema() {
  final String name = "allowNullPrimitives";
  try {
    repo.create(NAMESPACE, name, new DatasetDescriptor.Builder()
        .schema(ReflectData.AllowNull.get().getSchema(ObjectPoJo.class))
        .build(), ObjectPoJo.class);

    // should load the dataset because PrimitivePoJo can be used to write
    final Dataset<PrimitivePoJo> dataset = repo.load(
        NAMESPACE, name, PrimitivePoJo.class);
    TestHelpers.assertThrows("AllowNull primitives cannot read nullable type",
        IncompatibleSchemaException.class, new Runnable() {
          @Override
          public void run() {
            dataset.newReader();
          }
        });

  } catch (RuntimeException e) {
    throw e;
  } finally {
    repo.delete(NAMESPACE, name);
  }
}
 

protected AbstractRefinableView(AbstractRefinableView<?> view, Schema schema, Class<E> type) {
  if (view.dataset instanceof AbstractDataset) {
    this.dataset = ((AbstractDataset<?>) view.dataset).asType(type);
  } else {
    this.dataset = Datasets.load(view.dataset.getUri(), type);
  }
  this.comparator = view.comparator;
  this.constraints = view.constraints;
  // thread-safe, so okay to reuse when views share a partition strategy
  this.keys = view.keys;
  // Resolve our type according to the given schema
  this.accessor = DataModelUtil.accessor(type, schema);
  this.entityTest = constraints.toEntityPredicate(accessor);

  Schema datasetSchema = dataset.getDescriptor().getSchema();
  this.canRead = SchemaValidationUtil.canRead(
      datasetSchema, accessor.getReadSchema());
  this.canWrite = SchemaValidationUtil.canRead(
      accessor.getWriteSchema(), datasetSchema);

  IncompatibleSchemaException.check(canRead || canWrite,
      "The type cannot be used to read from or write to the dataset:\n" +
      "Type schema: %s\nDataset schema: %s",
      getSchema(), datasetSchema);
}
 

/**
 * Checks that the {@code existing} {@link DatasetDescriptor} is compatible
 * with {@code test}.
 *
 * @param existing the current {@code DatasetDescriptor} for a dataset
 * @param test a new {@code DatasetDescriptor} for the same dataset
 */
public static void checkCompatible(DatasetDescriptor existing,
                                   DatasetDescriptor test) {
  checkNotChanged("format", existing.getFormat(), test.getFormat());

  checkNotChanged("partitioning",
      existing.isPartitioned(), test.isPartitioned());

  if (existing.isPartitioned()) {
    checkStrategyUpdate(
        existing.getPartitionStrategy(),
        test.getPartitionStrategy(),
        test.getSchema());
  }

  // check can read records written with old schema using new schema
  Schema oldSchema = existing.getSchema();
  Schema testSchema = test.getSchema();
  if (!SchemaValidationUtil.canRead(oldSchema, testSchema)) {
    throw new IncompatibleSchemaException("Schema cannot read data " +
        "written using existing schema. Schema: " + testSchema.toString(true) +
        "\nExisting schema: " + oldSchema.toString(true));
  }

}
 

@Test(expected = IncompatibleSchemaException.class)
public void testIndirectIncompatibleUpdate() {
  SchemaManager manager = SchemaManager.create(getConfiguration(), testDirectory);

  // Write two schemas that are compatible since they use optional fields.
  manager.writeSchema(SchemaBuilder.record("test")
      .fields()
      .optionalString("foo")
      .endRecord());

  manager.writeSchema(SchemaBuilder.record("test")
      .fields()
      .optionalString("bar")
      .endRecord());

  // This schema creates a schema compatible with the immediately previous
  // version, but incompatible with the original.
  manager.writeSchema(SchemaBuilder.record("test")
      .fields()
      .optionalInt("foo")
      .endRecord());
}
 

@Test
public void testIncompatibleProjection() throws IOException {
  DatasetWriter<StandardEvent> writer = null;
  try {
    writer = unbounded.newWriter();
    writer.write(sepEvent);
    writer.write(octEvent);
    writer.write(novEvent);
  } finally {
    Closeables.close(writer, false);
  }

  TestHelpers.assertThrows(
      "Should not load a dataset with an incompatible class",
      IncompatibleSchemaException.class, new Runnable() {
        @Override
        public void run() {
          repo.load("ns", unbounded.getDataset().getName(),
              IncompatibleEvent.class);
        }
      });

  TestHelpers.assertThrows("Should reject a schema that can't read or write",
      IncompatibleSchemaException.class, new Runnable() {
        @Override
        public void run() {
          unbounded.asType(IncompatibleEvent.class);
        }
      });

  TestHelpers.assertThrows("Should reject a schema that can't read or write",
      IncompatibleSchemaException.class, new Runnable() {
        @Override
        public void run() {
          unbounded.getDataset().asType(IncompatibleEvent.class);
        }
      });
}
 

@Test(expected = IncompatibleSchemaException.class)
public void testCannotCreateExisting() throws Exception {
  manager.createSchema(tableName, "TestRecord", goodMigrationRecordAddField,
      "org.kitesdk.data.hbase.avro.AvroKeyEntitySchemaParser",
      "org.kitesdk.data.hbase.avro.AvroKeySerDe",
      "org.kitesdk.data.hbase.avro.AvroEntitySerDe");
}
 

@Test
public void testIncompatibleMerge() throws Exception {
  command.datasets = Lists.newArrayList(
      "resource:schema/string.avsc",
      "resource:schema/user.avsc");

  TestHelpers.assertThrows(
      "Should reject incompatible schemas, not produce a union schema",
      IncompatibleSchemaException.class, new Callable() {
    @Override
    public Integer call() throws IOException {
      return command.run();
    }
  });
}
 

@Test
public void testReflectProjectionAsType() throws IOException {
  Dataset<StandardEvent> original = repo.create(
      "ns", "reflectProjection",
      new DatasetDescriptor.Builder()
          .schema(StandardEvent.class)
          .build(),
      StandardEvent.class);

  DatasetWriter<ReflectStandardEvent> writer = null;
  try {
    writer = original.asType(ReflectStandardEvent.class).newWriter();
    writer.write(new ReflectStandardEvent(sepEvent));
    writer.write(new ReflectStandardEvent(octEvent));
    writer.write(new ReflectStandardEvent(novEvent));
  } finally {
    Closeables.close(writer, false);
  }

  final View<ReflectSmallEvent> smallEvents = original.asType(ReflectSmallEvent.class);

  Set<ReflectSmallEvent> expected = Sets.newHashSet(
      new ReflectSmallEvent(sepEvent), new ReflectSmallEvent(octEvent),
      new ReflectSmallEvent(novEvent));

  assertContentEquals(expected, smallEvents);

  TestHelpers.assertThrows("Should not be able to write small events",
      IncompatibleSchemaException.class, new Runnable() {
        @Override
        public void run() {
          smallEvents.newWriter();
        }
      });
}
 

@Test
public void testSpecificProjectionAsType() throws IOException {
  Dataset<GenericRecord> original = Datasets.load(unbounded.getUri());

  DatasetWriter<StandardEvent> writer = null;
  try {
    writer = original.asType(StandardEvent.class).newWriter();
    writer.write(sepEvent);
    writer.write(octEvent);
    writer.write(novEvent);
  } finally {
    Closeables.close(writer, false);
  }

  final View<SmallEvent> smallEvents = original.asType(SmallEvent.class);

  Set<SmallEvent> expected = Sets.newHashSet(toSmallEvent(sepEvent),
      toSmallEvent(octEvent), toSmallEvent(novEvent));

  assertContentEquals(expected, smallEvents);

  TestHelpers.assertThrows("Should not be able to write small events",
      IncompatibleSchemaException.class, new Runnable() {
        @Override
        public void run() {
          smallEvents.newWriter();
        }
      });
}
 

@Test
public void testGenericProjectionAsSchema() throws IOException {
  Dataset<StandardEvent> original = Datasets.load(
      unbounded.getUri(), StandardEvent.class);
  Schema standardEvent = Schemas.fromAvsc(conf,
      URI.create("resource:standard_event.avsc"));
  final Schema smallEvent = Schemas.fromAvsc(conf,
      URI.create("resource:small_event.avsc"));

  DatasetWriter<GenericRecord> writer = null;
  try {
    writer = original.asSchema(standardEvent).newWriter();
    writer.write(toGeneric(sepEvent, standardEvent));
    writer.write(toGeneric(octEvent, standardEvent));
    writer.write(toGeneric(novEvent, standardEvent));
  } finally {
    Closeables.close(writer, false);
  }

  final View<GenericRecord> smallEvents = original.asSchema(smallEvent);

  Set<GenericRecord> expected = Sets.newHashSet(
      toGeneric(toSmallEvent(sepEvent), smallEvent),
      toGeneric(toSmallEvent(octEvent), smallEvent),
      toGeneric(toSmallEvent(novEvent), smallEvent));

  assertContentEquals(expected, smallEvents);

  TestHelpers.assertThrows("Should not be able to write small events",
      IncompatibleSchemaException.class, new Runnable() {
        @Override
        public void run() {
          smallEvents.newWriter();
        }
      });
}
 

protected void checkSchemaForWrite() {
  IncompatibleSchemaException.check(canWrite,
      "Cannot write data with this view's schema, " +
      "it cannot be read with the dataset's schema:\n" +
      "Current schema: %s\nDataset schema: %s",
      getSchema(), dataset.getDescriptor().getSchema());
}
 

protected AbstractRefinableView(Dataset<E> dataset, Class<E> type) {
  this.dataset = dataset;
  final DatasetDescriptor descriptor = dataset.getDescriptor();
  if (descriptor.isPartitioned()) {
    this.constraints = new Constraints(
        descriptor.getSchema(), descriptor.getPartitionStrategy());
    // TODO: is comparator used anywhere?
    this.comparator = new MarkerComparator(descriptor.getPartitionStrategy());
    this.keys = new ThreadLocal<StorageKey>() {
      @Override
      protected StorageKey initialValue() {
        return new StorageKey(descriptor.getPartitionStrategy());
      }
    };
  } else {
    this.constraints = new Constraints(descriptor.getSchema());
    this.comparator = null;
    this.keys = null;
  }
  this.accessor = DataModelUtil.accessor(type, descriptor.getSchema());
  this.entityTest = constraints.toEntityPredicate(accessor);

  Schema datasetSchema = descriptor.getSchema();
  this.canRead = SchemaValidationUtil.canRead(
      datasetSchema, accessor.getReadSchema());
  this.canWrite = SchemaValidationUtil.canRead(
      accessor.getWriteSchema(), datasetSchema);

  IncompatibleSchemaException.check(canRead || canWrite,
      "The type cannot be used to read from or write to the dataset:\n" +
      "Type schema: %s\nDataset schema: %s",
      getSchema(), descriptor.getSchema());
}
 

@Test(expected = IncompatibleSchemaException.class)
public void testIncompatibleUpdate() {
  SchemaManager manager = SchemaManager.create(getConfiguration(), testDirectory);

  // Trivially incompatible schemas should yield an exception.
  manager.writeSchema(SchemaBuilder.record("test")
      .fields()
      .requiredString("foo")
      .endRecord());

  manager.writeSchema(SchemaBuilder.record("test")
      .fields()
      .requiredString("bar")
      .endRecord());
}
 

@Test(expected = IncompatibleSchemaException.class)
public void testBadMigration3() throws Exception {
  badMigration(badMigrationRecordModifiedMapping);
}
 

@Override
Result directory(FileSystem fs, Path path, List<Result> children) throws IOException {
  // there are two possible outcomes for this method:
  // 1. all child tables are compatible and part of one dataset (Table)
  // 2. each valid child is a separate dataset (Group)
  boolean allCompatible = true; // assume compatible to start
  boolean containsUnknown = false;

  // if all are compatible
  Schema mergedSchema = null;
  Format onlyFormat = null;
  int depth = Result.Table.UNKNOWN_DEPTH;

  // if all are separate datasets
  List<Result.Table> tables = Lists.newArrayList();

  for (Result child : children) {
    if (child instanceof Result.Unknown) {
      // not compatible at this level because a data file is not supported
      allCompatible = false;
      containsUnknown = true;

    } else if (child instanceof Result.Group) {
      // not compatible at a lower level
      Result.Group group = (Result.Group) child;
      containsUnknown |= group.containsUnknown;
      if (containsUnknown || !group.tables.isEmpty()) {
        // not compatible if there was an unknown or was not empty
        allCompatible = false;
      }
      tables.addAll(group.tables);

    } else {
      Result.Table table = (Result.Table) child;
      tables.add(table); // always add table in case not compatible later

      // if all tables are currently compatible, add the latest table
      if (allCompatible) {
        try {
          mergedSchema = merge(mergedSchema, table.schema);
        } catch (IncompatibleSchemaException e) {
          allCompatible = false;
        }

        if (onlyFormat == null) {
          onlyFormat = table.format;
        } else if (onlyFormat != table.format) {
          allCompatible = false;
        }

        if (depth == Result.Table.UNKNOWN_DEPTH) {
          depth = table.depth;
        } else if (depth != table.depth) {
          depth = Result.Table.MIXED_DEPTH;
        }
      }
    }
  }

  if (allCompatible && tables.size() > 0) {
    if (tables.size() == 1) {
      // only one, use the existing location rather than higher up the path
      return tables.get(0);
    } else {
      // more than one, use the path at this level
      return new Result.Table(path, onlyFormat, mergedSchema, depth);
    }
  } else {
    return new Result.Group(tables, containsUnknown);
  }
}
 

@Test(expected = IncompatibleSchemaException.class)
public void testBadCreateIncompatibleColumn3() throws Exception {
  SchemaTool tool = new SchemaTool(new HBaseAdmin(HBaseTestUtils.getConf()),
      new DefaultSchemaManager(tablePool));
  tool.createOrMigrateSchema(tableName, badCreateIncompatibleColumn3, false);
}
 

@Test(expected = IncompatibleSchemaException.class)
public void testBadCreateIncompatibleColumn2() throws Exception {
  SchemaTool tool = new SchemaTool(new HBaseAdmin(HBaseTestUtils.getConf()),
      new DefaultSchemaManager(tablePool));
  tool.createOrMigrateSchema(tableName, badCreateIncompatibleColumn2, false);
}
 

@Test(expected = IncompatibleSchemaException.class)
public void testBadCreateIncompatibleColumn1() throws Exception {
  SchemaTool tool = new SchemaTool(new HBaseAdmin(HBaseTestUtils.getConf()),
      new DefaultSchemaManager(tablePool));
  tool.createOrMigrateSchema(tableName, badCreateIncompatibleColumn1, false);
}
 

@Test(expected = IncompatibleSchemaException.class)
public void testBadCreateIncompatibleKey2() throws Exception {
  SchemaTool tool = new SchemaTool(new HBaseAdmin(HBaseTestUtils.getConf()),
      new DefaultSchemaManager(tablePool));
  tool.createOrMigrateSchema(tableName, badCreateIncompatibleKey2, false);
}
 

@Test(expected = IncompatibleSchemaException.class)
public void testBadCreateIncompatibleKey1() throws Exception {
  SchemaTool tool = new SchemaTool(new HBaseAdmin(HBaseTestUtils.getConf()),
      new DefaultSchemaManager(tablePool));
  tool.createOrMigrateSchema(tableName, badCreateIncompatibleKey1, false);
}
 

@Test(expected = IncompatibleSchemaException.class)
public void testBadMigrationIntToLong() throws Exception {
  SchemaManager manager = new DefaultSchemaManager(tablePool);
  manager.migrateSchema(tableName, "TestRecord", goodMigrationRecordAddField);
  manager.migrateSchema(tableName, "TestRecord", badMigrationRecordIntToLong);
}
 

@Test(expected = IncompatibleSchemaException.class)
public void testBadMigrationDuplicateSchema() throws Exception {
  badMigration(testRecordv2);
}
 

@Test(expected = IncompatibleSchemaException.class)
public void testBadMigration2() throws Exception {
  badMigration(badMigrationRecordAddSubFieldNoDefault);
}
 

@Test(expected = IncompatibleSchemaException.class)
public void testBadMigration1() throws Exception {
  badMigration(badMigrationRecordAddFieldNoDefault);
}
 

@Test(expected = IncompatibleSchemaException.class)
public void testBadMigrationKeyField() throws Exception {
  badMigration(badMigrationRecordAddKeyField);
}
 

protected void checkSchemaForRead() {
  IncompatibleSchemaException.check(canRead,
      "Cannot read data with this view's schema:\n" +
      "Current schema: %s\nDataset schema: %s",
      dataset.getDescriptor().getSchema(), getSchema());
}
 

/**
 * Merges two {@link Schema} instances if they are compatible.
 * <p>
 * Two schemas are incompatible if:
 * <ul>
 * <li>The {@link Schema.Type} does not match.</li>
 * <li>For record schemas, the record name does not match</li>
 * <li>For enum schemas, the enum name does not match</li>
 * </ul>
 * <p>
 * Map value and array element types will use unions if necessary, and union
 * schemas are merged recursively.
 *
 * @param left a {@code Schema}
 * @param right a {@code Schema}
 * @return a merged {@code Schema}
 * @throws IncompatibleSchemaException if the schemas are not compatible
 */
public static Schema merge(Schema left, Schema right) {
  Schema merged = mergeOnly(left, right);
  IncompatibleSchemaException.check(merged != null,
      "Cannot merge %s and %s", left, right);
  return merged;
}