/*
* MetaDataEvolutionTest.java
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2015-2019 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.apple.foundationdb.record.metadata;
import com.apple.foundationdb.async.RankedSet;
import com.apple.foundationdb.record.RecordCoreException;
import com.apple.foundationdb.record.RecordMetaData;
import com.apple.foundationdb.record.RecordMetaDataBuilder;
import com.apple.foundationdb.record.RecordMetaDataOptionsProto;
import com.apple.foundationdb.record.RecordMetaDataProto;
import com.apple.foundationdb.record.TestRecords1Proto;
import com.apple.foundationdb.record.TestRecordsEnumProto;
import com.apple.foundationdb.record.TestRecordsWithHeaderProto;
import com.apple.foundationdb.record.evolution.TestHeaderAsGroupProto;
import com.apple.foundationdb.record.evolution.TestMergedNestedTypesProto;
import com.apple.foundationdb.record.evolution.TestNewRecordTypeProto;
import com.apple.foundationdb.record.evolution.TestSelfReferenceProto;
import com.apple.foundationdb.record.evolution.TestSelfReferenceUnspooledProto;
import com.apple.foundationdb.record.evolution.TestSplitNestedTypesProto;
import com.apple.foundationdb.record.evolution.TestUnmergedNestedTypesProto;
import com.apple.foundationdb.record.provider.common.text.AllSuffixesTextTokenizer;
import com.apple.foundationdb.record.provider.common.text.DefaultTextTokenizer;
import com.apple.foundationdb.record.provider.common.text.PrefixTextTokenizer;
import com.apple.foundationdb.record.provider.common.text.TextTokenizer;
import com.apple.foundationdb.tuple.Tuple;
import com.google.protobuf.ByteString;
import com.google.protobuf.DescriptorProtos;
import com.google.protobuf.Descriptors;
import com.google.protobuf.Descriptors.Descriptor;
import com.google.protobuf.Descriptors.FieldDescriptor;
import com.google.protobuf.Descriptors.FileDescriptor;
import com.google.protobuf.InvalidProtocolBufferException;
import org.junit.jupiter.api.Test;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.function.Consumer;
import java.util.function.UnaryOperator;
import java.util.stream.Collectors;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.is;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
/**
* Tests of the {@link MetaDataEvolutionValidator} class. This mostly consists of trying to perform illegal updates
* to the meta-data and then verifying that the update fails. Some of the tests may try doing things that
* <i>seem</i> like they should be illegal but are actually fine.
*/
public class MetaDataEvolutionValidatorTest {
@Nonnull
private final MetaDataEvolutionValidator validator = MetaDataEvolutionValidator.getDefaultInstance();
static void assertInvalid(@Nonnull String errMsg, @Nonnull MetaDataEvolutionValidator validator,
@Nonnull RecordMetaData oldMetaData, @Nonnull RecordMetaData newMetaData) {
MetaDataException err = assertThrows(MetaDataException.class, () -> validator.validate(oldMetaData, newMetaData));
assertThat(err.getMessage(), containsString(errMsg));
}
static void assertInvalid(@Nonnull String errMsg, @Nonnull RecordMetaData oldMetaData, @Nonnull RecordMetaData newMetaData) {
assertInvalid(errMsg, MetaDataEvolutionValidator.getDefaultInstance(), oldMetaData, newMetaData);
}
static void assertInvalid(@Nonnull String errMsg, @Nonnull MetaDataEvolutionValidator validator,
@Nonnull Descriptor oldUnionDescriptor, @Nonnull Descriptor newUnionDescriptor) {
MetaDataException err = assertThrows(MetaDataException.class, () -> validator.validateUnion(oldUnionDescriptor, newUnionDescriptor));
assertThat(err.getMessage(), containsString(errMsg));
}
static void assertInvalid(@Nonnull String errMsg, @Nonnull Descriptor oldUnionDescriptor, @Nonnull Descriptor newUnionDescriptor) {
assertInvalid(errMsg, MetaDataEvolutionValidator.getDefaultInstance(), oldUnionDescriptor, newUnionDescriptor);
}
static void assertInvalid(@Nonnull String errMsg, @Nonnull FileDescriptor oldFileDescriptor, @Nonnull FileDescriptor newFileDescriptor) {
assertInvalid(errMsg, oldFileDescriptor.findMessageTypeByName(RecordMetaDataBuilder.DEFAULT_UNION_NAME), newFileDescriptor.findMessageTypeByName(RecordMetaDataBuilder.DEFAULT_UNION_NAME));
}
@Test
public void doNotChangeVersion() {
// Check if a naive removal of the index without updating the version is checked
RecordMetaData metaData1 = RecordMetaData.build(TestRecords1Proto.getDescriptor());
assertInvalid("new meta-data does not have newer version", metaData1, metaData1);
RecordMetaDataBuilder metaData2Builder = RecordMetaData.newBuilder().setRecords(TestRecords1Proto.getDescriptor());
metaData2Builder.removeIndex("MySimpleRecord$str_value_indexed");
validator.validate(metaData1, metaData2Builder.getRecordMetaData());
metaData2Builder.setVersion(metaData1.getVersion());
assertInvalid("new meta-data does not have newer version", metaData1, metaData2Builder.build(false));
// If the validator allows not changing the version, it should make sure all of the changes are compatible
MetaDataEvolutionValidator validatorAcceptingSameVersion = MetaDataEvolutionValidator.newBuilder()
.setAllowNoVersionChange(true)
.build();
validatorAcceptingSameVersion.validate(metaData1, metaData1);
// Confirm with the laxer validator that the removed index is noticed
metaData2Builder.setVersion(metaData1.getVersion() + 1);
RecordMetaDataBuilder metaData3Builder = RecordMetaData.newBuilder().setRecords(TestRecords1Proto.getDescriptor());
metaData3Builder.setVersion(metaData1.getVersion() + 1);
RecordMetaData metaData3 = metaData3Builder.getRecordMetaData();
RecordMetaData metaData4 = metaData2Builder.getRecordMetaData();
assertInvalid("new meta-data does not have newer version", metaData3, metaData4);
assertInvalid("new former index has removed version that is not newer than the old meta-data version",
validatorAcceptingSameVersion, metaData3, metaData4);
}
// Schema evolution tests
@Nonnull
static RecordMetaData replaceRecordsDescriptor(@Nonnull RecordMetaData metaData, @Nonnull FileDescriptor newDescriptor,
@Nonnull Consumer<RecordMetaDataProto.MetaData.Builder> metaDataMutation) {
RecordMetaDataProto.MetaData.Builder protoBuilder = metaData.toProto().toBuilder()
.setVersion(metaData.getVersion() + 1)
.setRecords(newDescriptor.toProto())
.addDependencies(TestRecords1Proto.getDescriptor().toProto());
metaDataMutation.accept(protoBuilder);
return RecordMetaData.build(protoBuilder.build());
}
@Nonnull
static RecordMetaData replaceRecordsDescriptor(@Nonnull RecordMetaData metaData, @Nonnull FileDescriptor newDescriptor) {
return replaceRecordsDescriptor(metaData, newDescriptor, ignore -> { });
}
@Nonnull
static FileDescriptor mutateFile(@Nonnull FileDescriptor originalFile, @Nonnull Consumer<DescriptorProtos.FileDescriptorProto.Builder> fileMutation) {
DescriptorProtos.FileDescriptorProto.Builder fileBuilder = originalFile.toProto().toBuilder();
fileMutation.accept(fileBuilder);
try {
return FileDescriptor.buildFrom(fileBuilder.build(), new FileDescriptor[]{RecordMetaDataOptionsProto.getDescriptor()});
} catch (Descriptors.DescriptorValidationException e) {
throw new RecordCoreException("unable to build file descriptor", e);
}
}
@Nonnull
static FileDescriptor mutateFile(@Nonnull Consumer<DescriptorProtos.FileDescriptorProto.Builder> fileMutation) {
return mutateFile(TestRecords1Proto.getDescriptor(), fileMutation);
}
@Nonnull
static FileDescriptor mutateField(@Nonnull String messageName, @Nonnull String fieldName, @Nonnull FileDescriptor originalFile,
@Nonnull Consumer<DescriptorProtos.FieldDescriptorProto.Builder> fieldMutation) {
return mutateFile(originalFile, fileBuilder ->
fileBuilder.getMessageTypeBuilderList().forEach(message -> {
if (message.getName().equals(messageName)) {
message.getFieldBuilderList().forEach(field -> {
if (field.getName().equals(fieldName)) {
fieldMutation.accept(field);
}
});
}
})
);
}
@Nonnull
static FileDescriptor mutateField(@Nonnull String messageName, @Nonnull String fieldName, @Nonnull Consumer<DescriptorProtos.FieldDescriptorProto.Builder> fieldMutation) {
return mutateField(messageName, fieldName, TestRecords1Proto.getDescriptor(), fieldMutation);
}
@Test
public void changeSplitLongRecords() {
RecordMetaData metaData1 = RecordMetaData.build(TestRecords1Proto.getDescriptor());
assertThat(metaData1.isSplitLongRecords(), is(false));
RecordMetaData metaData2 = RecordMetaData.build(metaData1.toProto().toBuilder()
.setVersion(metaData1.getVersion() + 1)
.setSplitLongRecords(true)
.build()
);
assertInvalid("new meta-data splits long records", metaData1, metaData2);
MetaDataEvolutionValidator laxerValidator = MetaDataEvolutionValidator.newBuilder()
.setAllowUnsplitToSplit(true)
.build();
laxerValidator.validate(metaData1, metaData2);
RecordMetaData metaData3 = RecordMetaData.build(metaData2.toProto().toBuilder()
.setVersion(metaData2.getVersion() + 1)
.setSplitLongRecords(false)
.build()
);
assertInvalid("new meta-data no longer splits long records", metaData2, metaData3);
assertInvalid("new meta-data no longer splits long records", laxerValidator, metaData2, metaData3);
}
@Test
public void changeStoreRecordVersions() {
RecordMetaData metaData1 = RecordMetaData.build(TestRecords1Proto.getDescriptor());
RecordMetaData metaData2 = RecordMetaData.build(metaData1.toProto().toBuilder()
.setVersion(metaData1.getVersion() + 1)
.setStoreRecordVersions(!metaData1.isStoreRecordVersions())
.build()
);
assertNotEquals(metaData1.isStoreRecordVersions(), metaData2.isStoreRecordVersions());
validator.validate(metaData1, metaData2);
RecordMetaData metaData3 = RecordMetaData.build(metaData2.toProto().toBuilder()
.setVersion(metaData2.getVersion() + 1)
.setStoreRecordVersions(!metaData2.isStoreRecordVersions())
.build()
);
assertNotEquals(metaData2.isStoreRecordVersions(), metaData3.isStoreRecordVersions());
validator.validate(metaData2, metaData3);
}
// Protobuf evolution tests
@Test
public void swapUnionFields() {
FileDescriptor updatedDescriptor = mutateFile(fileBuilder ->
fileBuilder.getMessageTypeBuilderList().forEach(message -> {
if (message.getName().equals(RecordMetaDataBuilder.DEFAULT_UNION_NAME)) {
message.getFieldBuilderList().forEach(field -> {
if (field.getNumber() == 1) {
field.setNumber(2);
} else {
field.setNumber(1);
}
});
}
})
);
// The two record types do not have the same form, so swapping them should fail.
// However, the exact way they fail isn't super important.
RecordMetaData metaData1 = RecordMetaData.build(TestRecords1Proto.getDescriptor());
RecordMetaData metaData2 = replaceRecordsDescriptor(metaData1, updatedDescriptor);
assertInvalid("", metaData1, metaData2);
}
/**
* Validate that changes to the union descriptor that equate two previously differentiated
* types--even if they have the same form--are disallowed. In theory, this could be allowed if
* the machinery to handle indexes were sophisticated enough. It would require that every index
* either be dropped or that all indexes were previously defined on both records and are now
* defined on the combined record. It also requires that any indexes on record type be dropped.
*/
@Test
public void mergeTypes() {
// Build a descriptor with two copies of MyOtherRecord (essentially).
FileDescriptor updatedDescriptor = mutateFile(fileBuilder -> {
DescriptorProtos.DescriptorProto newMessageType = fileBuilder.getMessageTypeList().stream()
.filter(message -> message.getName().equals("MyOtherRecord"))
.findFirst()
.get()
.toBuilder()
.setName("MyOtherOtherRecord")
.build();
fileBuilder.addMessageType(newMessageType);
fileBuilder.getMessageTypeBuilderList().forEach(message -> {
if (message.getName().equals(RecordMetaDataBuilder.DEFAULT_UNION_NAME)) {
message.addField(DescriptorProtos.FieldDescriptorProto.newBuilder()
.setLabel(DescriptorProtos.FieldDescriptorProto.Label.LABEL_OPTIONAL)
.setType(DescriptorProtos.FieldDescriptorProto.Type.TYPE_MESSAGE)
.setTypeName("MyOtherOtherRecord")
.setName("_MyOtherOtherRecord")
.setNumber(message.getFieldList().stream().mapToInt(DescriptorProtos.FieldDescriptorProto::getNumber).max().orElse(0) + 1)
);
}
});
});
assertThat(updatedDescriptor.getMessageTypes().stream().map(Descriptor::getName).collect(Collectors.toSet()),
containsInAnyOrder("MySimpleRecord", "MyOtherRecord", "MyOtherOtherRecord", RecordMetaDataBuilder.DEFAULT_UNION_NAME));
RecordMetaData metaData1 = RecordMetaData.build(updatedDescriptor);
assertThat(metaData1.getRecordTypes().keySet(), containsInAnyOrder("MySimpleRecord", "MyOtherRecord", "MyOtherOtherRecord"));
FileDescriptor secondDescriptor = mutateFile(updatedDescriptor, fileBuilder -> {
DescriptorProtos.DescriptorProto.Builder myOtherOtherDescriptor = fileBuilder.getMessageTypeBuilderList().stream()
.filter(message -> message.getName().equals("MyOtherOtherRecord"))
.findFirst()
.get();
int index = fileBuilder.getMessageTypeBuilderList().indexOf(myOtherOtherDescriptor);
fileBuilder.removeMessageType(index);
fileBuilder.getMessageTypeBuilderList().forEach(message -> {
if (message.getName().equals(RecordMetaDataBuilder.DEFAULT_UNION_NAME)) {
message.getFieldBuilderList().forEach(field -> {
if (field.getTypeName().equals("MyOtherOtherRecord")) {
field.setTypeName("MyOtherRecord");
}
});
}
});
});
RecordMetaData metaData2 = replaceRecordsDescriptor(metaData1, secondDescriptor, metaDataProtoBuilder -> {
RecordMetaDataProto.RecordType.Builder myOtherOtherRecord = metaDataProtoBuilder.getRecordTypesBuilderList().stream()
.filter(recordType -> recordType.getName().equals("MyOtherOtherRecord"))
.findFirst()
.get();
int index = metaDataProtoBuilder.getRecordTypesBuilderList().indexOf(myOtherOtherRecord);
metaDataProtoBuilder.removeRecordTypes(index);
});
assertThat(metaData2.getRecordTypes().keySet(), containsInAnyOrder("MySimpleRecord", "MyOtherRecord"));
assertInvalid("record type corresponds to multiple types in old meta-data", metaData1, metaData2);
}
/**
* Validate that changes to the union descriptor that differentiate two previously equivalent
* types--even if they have the same form--are disallowed. In theory, this could be allowed if
* the machinery to handle indexes were sophisticated enough. It would require that every index
* either be dropped or that all indexes that were previously defined on the combined record type
* are now defined on both types. It also requires that any indexes on record type be dropped.
*/
@Test
public void splitTypes() {
// Add a second "MyOtherRecord" to the union descriptor
FileDescriptor updatedDescriptor = mutateFile(fileBuilder ->
fileBuilder.getMessageTypeBuilderList().forEach(message -> {
if (message.getName().equals(RecordMetaDataBuilder.DEFAULT_UNION_NAME)) {
message.addField(DescriptorProtos.FieldDescriptorProto.newBuilder()
.setLabel(DescriptorProtos.FieldDescriptorProto.Label.LABEL_OPTIONAL)
.setType(DescriptorProtos.FieldDescriptorProto.Type.TYPE_MESSAGE)
.setTypeName("MyOtherRecord")
.setName("_MyOtherOtherRecord")
.setNumber(message.getFieldList().stream().mapToInt(DescriptorProtos.FieldDescriptorProto::getNumber).max().orElse(0) + 1));
}
})
);
RecordMetaData metaData1 = RecordMetaData.build(updatedDescriptor);
assertThat(metaData1.getRecordTypes().keySet(), containsInAnyOrder("MySimpleRecord", "MyOtherRecord"));
FileDescriptor secondDescriptor = mutateFile(updatedDescriptor, fileBuilder -> {
DescriptorProtos.DescriptorProto newMessageType = fileBuilder.getMessageTypeList().stream()
.filter(message -> message.getName().equals("MyOtherRecord"))
.findFirst()
.get()
.toBuilder()
.setName("MyOtherOtherRecord")
.build();
fileBuilder.addMessageType(newMessageType);
fileBuilder.getMessageTypeBuilderList().forEach(message -> {
if (message.getName().equals(RecordMetaDataBuilder.DEFAULT_UNION_NAME)) {
message.getFieldBuilderList().forEach(field -> {
if (field.getName().equals("_MyOtherOtherRecord")) {
field.setTypeName("MyOtherOtherRecord");
}
});
}
});
});
RecordMetaData metaData2 = replaceRecordsDescriptor(metaData1, secondDescriptor, metaDataProtoBuilder ->
metaDataProtoBuilder.addRecordTypes(RecordMetaDataProto.RecordType.newBuilder()
.setName("MyOtherOtherRecord")
.setPrimaryKey(Key.Expressions.field("rec_no").toKeyExpression())
)
);
assertThat(metaData2.getRecordTypes().keySet(), containsInAnyOrder("MySimpleRecord", "MyOtherRecord", "MyOtherOtherRecord"));
assertInvalid("record type corresponds to multiple types in new meta-data", metaData1, metaData2);
}
@Test
public void changeRecordTypeName() {
final MetaDataEvolutionValidator renameDisallowingValidator = MetaDataEvolutionValidator.newBuilder()
.setDisallowTypeRenames(true)
.build();
// Update the record type name, but don't update any references in indexes
FileDescriptor updatedFile = mutateFile(fileBuilder ->
fileBuilder.getMessageTypeBuilderList().forEach(messageType -> {
if (messageType.getName().equals("MyOtherRecord")) {
messageType.setName("MyOtherOtherRecord");
} else if (messageType.getName().equals(RecordMetaDataBuilder.DEFAULT_UNION_NAME)) {
messageType.getFieldBuilderList().forEach(field -> {
if (field.getName().equals("_MyOtherRecord")) {
field.setName("_MyOtherOtherRecord");
field.setTypeName("MyOtherOtherRecord");
}
});
}
})
);
RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder().setRecords(TestRecords1Proto.getDescriptor());
metaDataBuilder.addIndex("MyOtherRecord", "num_value_3_indexed");
RecordMetaData metaData1 = metaDataBuilder.getRecordMetaData();
MetaDataException e = assertThrows(MetaDataException.class, () -> replaceRecordsDescriptor(metaData1, updatedFile));
assertThat(e.getMessage(), containsString("Unknown record type MyOtherRecord"));
validator.validateUnion(metaData1.getUnionDescriptor(), updatedFile.findMessageTypeByName(RecordMetaDataBuilder.DEFAULT_UNION_NAME));
// Changes the record type definition but not the indexes
e = assertThrows(MetaDataException.class, () -> replaceRecordsDescriptor(metaData1, updatedFile, protoBuilder ->
protoBuilder.getRecordTypesBuilderList().forEach(recordType -> {
if (recordType.getName().equals("MyOtherRecord")) {
recordType.setName("MyOtherOtherRecord");
}
})
));
assertThat(e.getMessage(), containsString("Unknown record type MyOtherRecord"));
// This should be allowed because it replaces all index definitions with the new record type as well
RecordMetaData metaData3 = replaceRecordsDescriptor(metaData1, updatedFile, protoBuilder -> {
protoBuilder.getRecordTypesBuilderList().forEach(recordType -> {
if (recordType.getName().equals("MyOtherRecord")) {
recordType.setName("MyOtherOtherRecord");
}
});
protoBuilder.getIndexesBuilderList().forEach(index -> {
List<String> recordTypes = new ArrayList<>(index.getRecordTypeList());
recordTypes.replaceAll(recordType -> recordType.equals("MyOtherRecord") ? "MyOtherOtherRecord" : recordType);
index.clearRecordType();
index.addAllRecordType(recordTypes);
});
});
assertEquals(Collections.singletonList(metaData3.getRecordType("MyOtherOtherRecord")),
metaData3.recordTypesForIndex(metaData3.getIndex("MyOtherRecord$num_value_3_indexed")));
validator.validate(metaData1, metaData3);
assertInvalid("record type name changed", renameDisallowingValidator, metaData1, metaData3);
// Validate that calling update records with the new file descriptor produces a valid evolution
RecordMetaDataBuilder metaDataBuilder4 = RecordMetaData.newBuilder().setRecords(metaData1.toProto());
metaDataBuilder4.updateRecords(updatedFile);
RecordMetaData metaData4 = metaDataBuilder4.getRecordMetaData();
assertEquals(Collections.singletonList(metaData4.getRecordType("MyOtherOtherRecord")),
metaData4.recordTypesForIndex(metaData4.getIndex("MyOtherRecord$num_value_3_indexed")));
validator.validate(metaData1, metaData4);
assertInvalid("record type name changed", renameDisallowingValidator, metaData1, metaData4);
}
@Test
public void swapRecordTypes() {
FileDescriptor updatedFile = mutateFile(fileBuilder -> {
// Update the field of the union descriptor.
fileBuilder.getMessageTypeBuilderList().forEach(messageType -> {
if (messageType.getName().equals(RecordMetaDataBuilder.DEFAULT_UNION_NAME)) {
messageType.getFieldBuilderList().forEach(field -> {
if (field.getName().equals("_MyOtherRecord")) {
field.setTypeName("MySimpleRecord");
}
if (field.getName().equals("_MySimpleRecord")) {
field.setTypeName("MyOtherRecord");
}
});
}
});
});
RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder().setRecords(TestRecords1Proto.getDescriptor());
metaDataBuilder.addIndex("MyOtherRecord", "num_value_3_indexed");
RecordMetaData metaData1 = metaDataBuilder.getRecordMetaData();
// Swap is noticed as the two records are not of compatible forms
RecordMetaData metaData2 = replaceRecordsDescriptor(metaData1, updatedFile);
assertInvalid("", metaData1, metaData2);
}
@Test
public void swapIsomorphicRecordTypesWithIndexes() {
FileDescriptor updatedFile = mutateFile(fileBuilder -> {
DescriptorProtos.DescriptorProto newMessageType = fileBuilder.getMessageTypeList().stream()
.filter(messageType -> messageType.getName().equals("MyOtherRecord"))
.findFirst()
.get()
.toBuilder()
.setName("MyOtherOtherRecord")
.build();
fileBuilder.addMessageType(newMessageType);
fileBuilder.getMessageTypeBuilderList().forEach(messageType -> {
if (messageType.getName().equals(RecordMetaDataBuilder.DEFAULT_UNION_NAME)) {
messageType.addField(DescriptorProtos.FieldDescriptorProto.newBuilder()
.setLabel(DescriptorProtos.FieldDescriptorProto.Label.LABEL_OPTIONAL)
.setType(DescriptorProtos.FieldDescriptorProto.Type.TYPE_MESSAGE)
.setTypeName("MyOtherOtherRecord")
.setName("_MyOtherOtherRecord")
.setNumber(messageType.getFieldList().stream().mapToInt(DescriptorProtos.FieldDescriptorProto::getNumber).max().orElse(0) + 1));
}
});
});
RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder().setRecords(updatedFile);
metaDataBuilder.addIndex("MyOtherRecord", "num_value_3_indexed");
metaDataBuilder.addIndex("MyOtherOtherRecord", "num_value_3_indexed");
RecordMetaData metaData1 = metaDataBuilder.getRecordMetaData();
assertThat(metaData1.getRecordTypes().keySet(), containsInAnyOrder("MySimpleRecord", "MyOtherRecord", "MyOtherOtherRecord"));
assertEquals(Collections.singletonList(metaData1.getRecordType("MyOtherRecord")),
metaData1.recordTypesForIndex(metaData1.getIndex("MyOtherRecord$num_value_3_indexed")));
assertEquals(Collections.singletonList(metaData1.getRecordType("MyOtherOtherRecord")),
metaData1.recordTypesForIndex(metaData1.getIndex("MyOtherOtherRecord$num_value_3_indexed")));
// Swap the two record types in the union descriptor.
FileDescriptor secondFile = mutateFile(updatedFile, fileBuilder ->
fileBuilder.getMessageTypeBuilderList().forEach(messageType -> {
if (messageType.getName().equals(RecordMetaDataBuilder.DEFAULT_UNION_NAME)) {
messageType.getFieldBuilderList().forEach(field -> {
if (field.getName().equals("_MyOtherRecord")) {
field.setTypeName("MyOtherOtherRecord");
}
if (field.getName().equals("_MyOtherOtherRecord")) {
field.setTypeName("MyOtherRecord");
}
});
}
})
);
// Doesn't update the record types for the index which effectively swaps the definitions
RecordMetaData metaData2 = replaceRecordsDescriptor(metaData1, secondFile);
assertThat(metaData2.getRecordTypes().keySet(), containsInAnyOrder("MySimpleRecord", "MyOtherRecord", "MyOtherOtherRecord"));
assertEquals(Collections.singletonList(metaData2.getRecordType("MyOtherRecord")),
metaData2.recordTypesForIndex(metaData2.getIndex("MyOtherRecord$num_value_3_indexed")));
assertEquals(Collections.singletonList(metaData2.getRecordType("MyOtherOtherRecord")),
metaData2.recordTypesForIndex(metaData2.getIndex("MyOtherOtherRecord$num_value_3_indexed")));
assertInvalid("new index removes record type", metaData1, metaData2);
// Replace the record types in the indexes with the new names
RecordMetaData metaData3 = replaceRecordsDescriptor(metaData2, secondFile, metaDataProtoBuilder ->
metaDataProtoBuilder.getIndexesBuilderList().forEach(index -> {
List<String> recordTypes = new ArrayList<>(index.getRecordTypeList());
recordTypes.replaceAll(recordType -> {
if (recordType.equals("MyOtherRecord")) {
return "MyOtherOtherRecord";
} else if (recordType.equals("MyOtherOtherRecord")) {
return "MyOtherRecord";
} else {
return recordType;
}
});
index.clearRecordType();
index.addAllRecordType(recordTypes);
})
);
assertEquals(Collections.singletonList(metaData3.getRecordType("MyOtherOtherRecord")),
metaData3.recordTypesForIndex(metaData3.getIndex("MyOtherRecord$num_value_3_indexed")));
assertEquals(Collections.singletonList(metaData3.getRecordType("MyOtherRecord")),
metaData3.recordTypesForIndex(metaData3.getIndex("MyOtherOtherRecord$num_value_3_indexed")));
validator.validate(metaData1, metaData3);
// Verify that using "update records" updates the index definitions
RecordMetaDataBuilder metaDataBuilder4 = RecordMetaData.newBuilder().setRecords(metaData1.toProto());
metaDataBuilder4.updateRecords(secondFile);
RecordMetaData metaData4 = metaDataBuilder4.getRecordMetaData();
assertEquals(Collections.singletonList(metaData4.getRecordType("MyOtherOtherRecord")),
metaData4.recordTypesForIndex(metaData4.getIndex("MyOtherRecord$num_value_3_indexed")));
assertEquals(Collections.singletonList(metaData4.getRecordType("MyOtherRecord")),
metaData4.recordTypesForIndex(metaData4.getIndex("MyOtherOtherRecord$num_value_3_indexed")));
validator.validate(metaData1, metaData4);
}
@Test
public void swapIsomorphicRecordTypesWithExplicitKeys() {
FileDescriptor updatedFile = mutateFile(fileBuilder -> {
DescriptorProtos.DescriptorProto newMessageType = fileBuilder.getMessageTypeList().stream()
.filter(messageType -> messageType.getName().equals("MyOtherRecord"))
.findFirst()
.get()
.toBuilder()
.setName("MyOtherOtherRecord")
.build();
fileBuilder.addMessageType(newMessageType);
fileBuilder.getMessageTypeBuilderList().forEach(messageType -> {
if (messageType.getName().equals(RecordMetaDataBuilder.DEFAULT_UNION_NAME)) {
messageType.addField(DescriptorProtos.FieldDescriptorProto.newBuilder()
.setLabel(DescriptorProtos.FieldDescriptorProto.Label.LABEL_OPTIONAL)
.setType(DescriptorProtos.FieldDescriptorProto.Type.TYPE_MESSAGE)
.setTypeName("MyOtherOtherRecord")
.setName("_MyOtherOtherRecord")
.setNumber(messageType.getFieldList().stream().mapToInt(DescriptorProtos.FieldDescriptorProto::getNumber).max().orElse(0) + 1));
}
});
});
RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder().setRecords(updatedFile);
metaDataBuilder.getRecordType("MyOtherRecord").setRecordTypeKey("other");
metaDataBuilder.getRecordType("MyOtherOtherRecord").setRecordTypeKey("other_other");
RecordMetaData metaData1 = metaDataBuilder.getRecordMetaData();
assertEquals("other", metaData1.getRecordType("MyOtherRecord").getRecordTypeKey());
assertEquals("other_other", metaData1.getRecordType("MyOtherOtherRecord").getRecordTypeKey());
// Swap the definitions in the union descriptor
FileDescriptor secondFile = mutateFile(updatedFile, fileBuilder ->
fileBuilder.getMessageTypeBuilderList().forEach(messageType -> {
if (messageType.getName().equals(RecordMetaDataBuilder.DEFAULT_UNION_NAME)) {
messageType.getFieldBuilderList().forEach(field -> {
if (field.getName().equals("_MyOtherRecord")) {
field.setTypeName("MyOtherOtherRecord");
} else if (field.getName().equals("_MyOtherOtherRecord")) {
field.setTypeName("MyOtherRecord");
}
});
}
})
);
RecordMetaData metaData2 = replaceRecordsDescriptor(metaData1, secondFile);
assertEquals("other", metaData2.getRecordType("MyOtherRecord").getRecordTypeKey());
assertEquals("other_other", metaData2.getRecordType("MyOtherOtherRecord").getRecordTypeKey());
assertInvalid("record type key changed", metaData1, metaData2);
// Swap the definitions in the record type descriptor list
RecordMetaData metaData3 = replaceRecordsDescriptor(metaData1, secondFile, metaDataProtoBuilder ->
metaDataProtoBuilder.getRecordTypesBuilderList().forEach(recordType -> {
if (recordType.getName().equals("MyOtherRecord")) {
recordType.setName("MyOtherOtherRecord");
} else if (recordType.getName().equals("MyOtherOtherRecord")) {
recordType.setName("MyOtherRecord");
}
})
);
assertEquals("other", metaData3.getRecordType("MyOtherOtherRecord").getRecordTypeKey());
assertEquals("other_other", metaData3.getRecordType("MyOtherRecord").getRecordTypeKey());
validator.validate(metaData1, metaData3);
// Verify that using "update records" updates the record type keys
RecordMetaDataBuilder metaDataBuilder4 = RecordMetaData.newBuilder().setRecords(metaData1.toProto());
metaDataBuilder4.updateRecords(secondFile);
RecordMetaData metaData4 = metaDataBuilder4.getRecordMetaData();
assertEquals("other", metaData4.getRecordType("MyOtherOtherRecord").getRecordTypeKey());
assertEquals("other_other", metaData4.getRecordType("MyOtherRecord").getRecordTypeKey());
validator.validate(metaData1, metaData4);
}
@Test
public void dropField() {
FileDescriptor updatedFile = mutateFile(fileBuilder ->
fileBuilder.getMessageTypeBuilderList().forEach(message -> {
if (message.getName().equals("MySimpleRecord")) {
int fieldNumValue2Index = 0;
while (!message.getField(fieldNumValue2Index).getName().equals("num_value_2")) {
fieldNumValue2Index++;
}
message.removeField(fieldNumValue2Index);
}
})
);
assertInvalid("field removed from message descriptor", TestRecords1Proto.getDescriptor(), updatedFile);
RecordMetaData metaData1 = RecordMetaData.build(TestRecords1Proto.getDescriptor());
RecordMetaData metaData2 = replaceRecordsDescriptor(metaData1, updatedFile);
assertInvalid("field removed from message descriptor", metaData1, metaData2);
}
@Test
public void renameField() {
FileDescriptor updatedFile = mutateField("MySimpleRecord", "num_value_2",
field -> field.setName("num_value_too"));
assertInvalid("field renamed", TestRecords1Proto.getDescriptor(), updatedFile);
RecordMetaData metaData1 = RecordMetaData.build(TestRecords1Proto.getDescriptor());
RecordMetaData metaData2 = replaceRecordsDescriptor(metaData1, updatedFile);
assertInvalid("field renamed", metaData1, metaData2);
// This updates both the field name and its indexes which means that this is actually okay.
updatedFile = mutateField("MySimpleRecord", "str_value_indexed",
field -> field.setName("str_value_still_indexed"));
RecordMetaData metaData3 = replaceRecordsDescriptor(metaData1, updatedFile, protoBuilder ->
protoBuilder.getIndexesBuilderList().forEach(index -> {
if (index.getName().equals("MySimpleRecord$str_value_indexed")) {
index.setRootExpression(Key.Expressions.field("str_value_still_indexed").toKeyExpression());
}
})
);
assertInvalid("field renamed", metaData1, metaData3);
}
@Test
public void fieldTypeChanged() throws InvalidProtocolBufferException {
FileDescriptor updatedFile = mutateField("MySimpleRecord", "str_value_indexed",
field -> field.setType(DescriptorProtos.FieldDescriptorProto.Type.TYPE_BYTES));
assertInvalid("field type changed", TestRecords1Proto.getDescriptor(), updatedFile);
RecordMetaData metaData1 = RecordMetaData.build(TestRecords1Proto.getDescriptor());
RecordMetaData metaData2 = replaceRecordsDescriptor(metaData1, updatedFile);
assertInvalid("field type changed", metaData1, metaData2);
// Allow int32 -> int64 but not int64 -> int32
assertEquals(FieldDescriptor.Type.INT32, TestRecords1Proto.MySimpleRecord.getDescriptor().findFieldByName("num_value_2").getType());
updatedFile = mutateField("MySimpleRecord", "num_value_2",
field -> field.setType(DescriptorProtos.FieldDescriptorProto.Type.TYPE_INT64));
assertEquals(FieldDescriptor.Type.INT64, updatedFile.findMessageTypeByName("MySimpleRecord").findFieldByName("num_value_2").getType());
validator.validateUnion(TestRecords1Proto.RecordTypeUnion.getDescriptor(), updatedFile.findMessageTypeByName(RecordMetaDataBuilder.DEFAULT_UNION_NAME));
assertInvalid("field type changed", updatedFile, TestRecords1Proto.getDescriptor());
RecordMetaData metaData3 = replaceRecordsDescriptor(metaData1, updatedFile);
validator.validate(metaData1, metaData3);
// Allow sint32 -> sint64 but not sint32 -> sint64
FileDescriptor intermediateFile = mutateField("MySimpleRecord", "num_value_2",
field -> field.setType(DescriptorProtos.FieldDescriptorProto.Type.TYPE_SINT32));
updatedFile = mutateField("MySimpleRecord", "num_value_2",
field -> field.setType(DescriptorProtos.FieldDescriptorProto.Type.TYPE_SINT64));
validator.validateUnion(intermediateFile.findMessageTypeByName(RecordMetaDataBuilder.DEFAULT_UNION_NAME), updatedFile.findMessageTypeByName(RecordMetaDataBuilder.DEFAULT_UNION_NAME));
assertInvalid("field type changed", updatedFile, intermediateFile);
// Do not allow sfixed32 -> sfixed64
intermediateFile = mutateField("MySimpleRecord", "num_value_2",
field -> field.setType(DescriptorProtos.FieldDescriptorProto.Type.TYPE_SFIXED32));
updatedFile = mutateField("MySimpleRecord", "num_value_2",
field -> field.setType(DescriptorProtos.FieldDescriptorProto.Type.TYPE_SFIXED64));
assertInvalid("field type changed", intermediateFile, updatedFile);
}
@Test
public void fieldChangedFromMessageToGroup() {
// The message and group types here have the same form, but messages and groups are serialized differently
assertInvalid("field type changed", TestRecordsWithHeaderProto.getDescriptor(), TestHeaderAsGroupProto.getDescriptor());
assertInvalid("field type changed", TestHeaderAsGroupProto.getDescriptor(), TestRecordsWithHeaderProto.getDescriptor());
RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder().setRecords(TestRecordsWithHeaderProto.getDescriptor());
metaDataBuilder.getRecordType("MyRecord").setPrimaryKey(Key.Expressions.field("header").nest(Key.Expressions.concatenateFields("path", "rec_no")));
RecordMetaData metaData1 = metaDataBuilder.getRecordMetaData();
RecordMetaData metaData2 = replaceRecordsDescriptor(metaData1, TestHeaderAsGroupProto.getDescriptor());
assertInvalid("field type changed", metaData1, metaData2);
}
@Test
public void enumFieldChanged() {
// Add an enum field
FileDescriptor updatedFile = mutateFile(TestRecordsEnumProto.getDescriptor(), fileBuilder ->
fileBuilder.getMessageTypeBuilderList().forEach(message -> {
if (message.getName().equals("MyShapeRecord")) {
message.getEnumTypeBuilderList().forEach(enumType -> {
if (enumType.getName().equals("Size")) {
enumType.addValue(DescriptorProtos.EnumValueDescriptorProto.newBuilder()
.setName("X_LARGE")
.setNumber(TestRecordsEnumProto.MyShapeRecord.Size.getDescriptor().getValues().stream()
.mapToInt(Descriptors.EnumValueDescriptor::getNumber).max().getAsInt() + 1));
}
});
}
})
);
validator.validateUnion(TestRecordsEnumProto.RecordTypeUnion.getDescriptor(), updatedFile.findMessageTypeByName(RecordMetaDataBuilder.DEFAULT_UNION_NAME));
RecordMetaData metaData1 = RecordMetaData.build(TestRecordsEnumProto.getDescriptor());
RecordMetaData metaData2 = replaceRecordsDescriptor(metaData1, updatedFile);
validator.validate(metaData1, metaData2);
// Dropping a value is not allowed
assertInvalid("enum removes value", updatedFile, TestRecordsEnumProto.getDescriptor());
RecordMetaData metaData3 = RecordMetaData.build(updatedFile);
RecordMetaData metaData4 = replaceRecordsDescriptor(metaData3, TestRecordsEnumProto.getDescriptor());
assertInvalid("enum removes value", metaData3, metaData4);
// Changing the value name is okay
updatedFile = mutateFile(TestRecordsEnumProto.getDescriptor(), fileBuilder ->
fileBuilder.getEnumTypeBuilderList().forEach(enumType -> {
if (enumType.getName().equals("Size")) {
enumType.getValueBuilder(0).setName("PETIT");
}
})
);
validator.validateUnion(TestRecordsEnumProto.RecordTypeUnion.getDescriptor(), updatedFile.findMessageTypeByName(RecordMetaDataBuilder.DEFAULT_UNION_NAME));
RecordMetaData metaData5 = replaceRecordsDescriptor(metaData1, updatedFile);
validator.validate(metaData1, metaData5);
}
@Test
public void selfReferenceChanged() {
// This is largely to test that messages which include themselves as nested types don't cause the validator to blow up
final Descriptor selfReferenceUnion = TestSelfReferenceProto.RecordTypeUnion.getDescriptor();
final Descriptor unspooledUnion = TestSelfReferenceUnspooledProto.RecordTypeUnion.getDescriptor();
validator.validateUnion(selfReferenceUnion, unspooledUnion);
assertInvalid("field removed", unspooledUnion, selfReferenceUnion);
FileDescriptor updatedUnspooledFile = mutateFile(TestSelfReferenceUnspooledProto.getDescriptor(), fileBuilder ->
fileBuilder.getMessageTypeBuilderList().forEach(message -> {
if (message.getName().equals("Node")) {
message.removeField(0);
}
})
);
assertNull(updatedUnspooledFile.findMessageTypeByName("Node").findFieldByName("rec_no"));
assertInvalid("field removed", TestSelfReferenceUnspooledProto.getDescriptor(), updatedUnspooledFile);
}
@Test
public void nestedTypeChangesName() {
FileDescriptor updatedFile = mutateFile(TestRecordsWithHeaderProto.getDescriptor(), fileBuilder ->
fileBuilder.getMessageTypeBuilderList().forEach(message -> {
if (message.getName().equals("HeaderRecord")) {
message.setName("Header");
} else if (message.getName().equals("MyRecord")) {
message.getFieldBuilderList().forEach(field -> {
if (field.getName().equals("header")) {
field.setTypeName("." + fileBuilder.getPackage() + ".Header");
}
});
}
})
);
assertThat(updatedFile.getMessageTypes().stream().map(Descriptor::getName).collect(Collectors.toList()), containsInAnyOrder("MyRecord", RecordMetaDataBuilder.DEFAULT_UNION_NAME, "Header"));
validator.validateUnion(TestRecordsWithHeaderProto.RecordTypeUnion.getDescriptor(), updatedFile.findMessageTypeByName(RecordMetaDataBuilder.DEFAULT_UNION_NAME));
RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder().setRecords(TestRecordsWithHeaderProto.getDescriptor());
metaDataBuilder.getRecordType("MyRecord").setPrimaryKey(Key.Expressions.field("header").nest(Key.Expressions.concatenateFields("path", "rec_no")));
RecordMetaData metaData1 = metaDataBuilder.getRecordMetaData();
RecordMetaData metaData2 = replaceRecordsDescriptor(metaData1, updatedFile);
validator.validate(metaData1, metaData2);
}
@Test
public void nestedTypeChangesFieldName() {
FileDescriptor updatedFile = mutateField("HeaderRecord", "num", TestRecordsWithHeaderProto.getDescriptor(),
field -> field.setName("numb"));
assertInvalid("field renamed", TestRecordsWithHeaderProto.getDescriptor(), updatedFile);
RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder().setRecords(TestRecordsWithHeaderProto.getDescriptor());
metaDataBuilder.getRecordType("MyRecord").setPrimaryKey(Key.Expressions.field("header").nest(Key.Expressions.concatenateFields("path", "rec_no")));
RecordMetaData metaData1 = metaDataBuilder.getRecordMetaData();
RecordMetaData metaData2 = replaceRecordsDescriptor(metaData1, updatedFile);
assertInvalid("field renamed", metaData1, metaData2);
}
@Test
public void nestedTypeChangesFieldType() {
FileDescriptor updatedFile = mutateField("HeaderRecord", "num", TestRecordsWithHeaderProto.getDescriptor(),
field -> field.setType(DescriptorProtos.FieldDescriptorProto.Type.TYPE_SFIXED32));
assertInvalid("field type changed", TestRecordsWithHeaderProto.getDescriptor(), updatedFile);
RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder().setRecords(TestRecordsWithHeaderProto.getDescriptor());
metaDataBuilder.getRecordType("MyRecord").setPrimaryKey(Key.Expressions.field("header").nest(Key.Expressions.concatenateFields("path", "rec_no")));
RecordMetaData metaData1 = metaDataBuilder.getRecordMetaData();
RecordMetaData metaData2 = replaceRecordsDescriptor(metaData1, updatedFile);
assertInvalid("field type changed", metaData1, metaData2);
}
@Test
public void nestedTypesMerged() {
validator.validateUnion(TestUnmergedNestedTypesProto.RecordTypeUnion.getDescriptor(), TestMergedNestedTypesProto.RecordTypeUnion.getDescriptor());
FileDescriptor updatedMergedFile = mutateField("OneTrueNested", "b", TestMergedNestedTypesProto.getDescriptor(),
field -> field.setName("c"));
assertInvalid("field renamed", TestUnmergedNestedTypesProto.getDescriptor(), updatedMergedFile);
}
@Test
public void nestedTypesSplit() {
validator.validateUnion(TestMergedNestedTypesProto.RecordTypeUnion.getDescriptor(), TestSplitNestedTypesProto.RecordTypeUnion.getDescriptor());
FileDescriptor updatedSplitFile = mutateField("NestedB", "b", TestSplitNestedTypesProto.getDescriptor(),
field -> field.setType(DescriptorProtos.FieldDescriptorProto.Type.TYPE_BYTES));
assertInvalid("field type changed", TestUnmergedNestedTypesProto.getDescriptor(), updatedSplitFile);
}
@Test
public void fieldLabelChanged() {
FileDescriptor oldFile = TestRecords1Proto.getDescriptor();
List<DescriptorProtos.FieldDescriptorProto.Label> labels = Arrays.asList(
DescriptorProtos.FieldDescriptorProto.Label.LABEL_OPTIONAL,
DescriptorProtos.FieldDescriptorProto.Label.LABEL_REPEATED,
DescriptorProtos.FieldDescriptorProto.Label.LABEL_REQUIRED
);
for (int i = 0; i < labels.size(); i++) {
final int itr = i;
final DescriptorProtos.FieldDescriptorProto.Label label = labels.get(itr);
final String labelText = label.name().substring(label.name().indexOf('_') + 1).toLowerCase();
final String errMsg = String.format("%s field is no longer %s", labelText, labelText);
FileDescriptor updatedFile = mutateField("MySimpleRecord", "str_value_indexed", oldFile,
field -> field.setLabel(labels.get((itr + 1) % labels.size())));
assertInvalid(errMsg, oldFile, updatedFile);
oldFile = updatedFile;
}
}
@Test
public void addRequiredField() {
FileDescriptor updatedFile = mutateFile(fileBuilder ->
fileBuilder.getMessageTypeBuilderList().forEach(message -> {
if (message.getName().equals("MySimpleRecord")) {
message.addField(DescriptorProtos.FieldDescriptorProto.newBuilder()
.setLabel(DescriptorProtos.FieldDescriptorProto.Label.LABEL_REQUIRED)
.setType(DescriptorProtos.FieldDescriptorProto.Type.TYPE_INT32)
.setName("new_int_field")
.setNumber(message.getFieldList().stream().mapToInt(DescriptorProtos.FieldDescriptorProto::getNumber).max().getAsInt() + 1)
);
}
})
);
assertInvalid("required field added to record type", TestRecords1Proto.getDescriptor(), updatedFile);
RecordMetaData metaData1 = RecordMetaData.build(TestRecords1Proto.getDescriptor());
RecordMetaData metaData2 = replaceRecordsDescriptor(metaData1, updatedFile);
assertInvalid("required field added to record type", metaData1, metaData2);
}
@Test
public void dropType() {
FileDescriptor updatedFile = mutateFile(fileBuilder ->
fileBuilder.getMessageTypeBuilderList().forEach(message -> {
if (message.getName().equals(RecordMetaDataBuilder.DEFAULT_UNION_NAME)) {
message.removeField(1);
}
})
);
assertInvalid("record type removed from union", TestRecords1Proto.getDescriptor(), updatedFile);
RecordMetaData metaData1 = RecordMetaData.build(TestRecords1Proto.getDescriptor());
MetaDataException e = assertThrows(MetaDataException.class, () -> replaceRecordsDescriptor(metaData1, updatedFile));
assertThat(e.getMessage(), containsString("Unknown record type MyOtherRecord"));
RecordMetaData metaData2 = replaceRecordsDescriptor(metaData1, updatedFile, protoBuilder -> {
protoBuilder.removeRecordTypes(1);
final List<RecordMetaDataProto.Index> indexes = protoBuilder.getIndexesList().stream()
.filter(index -> !index.getRecordTypeList().contains("MyOtherRecord"))
.collect(Collectors.toList());
protoBuilder.clearIndexes().addAllIndexes(indexes);
});
assertInvalid("record type removed from union", metaData1, metaData2);
}
@Test
public void addNewPlaceInUnionDescriptor() {
// Add a new field to the union descriptor that points to an existing record; leave the old one
FileDescriptor updatedFile = mutateFile(fileBuilder ->
fileBuilder.getMessageTypeBuilderList().forEach(message -> {
if (message.getName().equals(RecordMetaDataBuilder.DEFAULT_UNION_NAME)) {
message.getFieldBuilderList().forEach(field -> {
if (field.getName().endsWith("MySimpleRecord")) {
field.setName("_MyOldSimpleRecordField");
}
});
message.addField(DescriptorProtos.FieldDescriptorProto.newBuilder()
.setLabel(DescriptorProtos.FieldDescriptorProto.Label.LABEL_OPTIONAL)
.setType(DescriptorProtos.FieldDescriptorProto.Type.TYPE_MESSAGE)
.setTypeName("MySimpleRecord")
.setName("_MySimpleRecord")
.setNumber(1066));
}
})
);
RecordMetaData metaData1 = RecordMetaData.build(TestRecords1Proto.getDescriptor());
RecordMetaData metaData2 = replaceRecordsDescriptor(metaData1, updatedFile);
assertEquals(1066, metaData2.getUnionFieldForRecordType(metaData2.getRecordType("MySimpleRecord")).getNumber());
validator.validate(metaData1, metaData2);
assertEquals(metaData1.getRecordType("MySimpleRecord").getRecordTypeKey(), metaData2.getRecordType("MySimpleRecord").getRecordTypeKey());
// Add a new field that points to an existing record but put it in a lower position in the union which makes the record type key change
updatedFile = mutateFile(updatedFile, fileBuilder ->
fileBuilder.getMessageTypeBuilderList().forEach(message -> {
if (message.getName().equals(RecordMetaDataBuilder.DEFAULT_UNION_NAME)) {
message.removeField(0);
}
})
);
RecordMetaData metaData3 = RecordMetaData.build(updatedFile);
updatedFile = mutateFile(updatedFile, fileBuilder ->
fileBuilder.getMessageTypeBuilderList().forEach(message -> {
if (message.getName().equals(RecordMetaDataBuilder.DEFAULT_UNION_NAME)) {
message.addField(DescriptorProtos.FieldDescriptorProto.newBuilder()
.setLabel(DescriptorProtos.FieldDescriptorProto.Label.LABEL_OPTIONAL)
.setType(DescriptorProtos.FieldDescriptorProto.Type.TYPE_MESSAGE)
.setTypeName("MySimpleRecord")
.setName("_MyOtherSimpleRecord")
.setNumber(800));
}
})
);
RecordMetaData metaData4 = replaceRecordsDescriptor(metaData3, updatedFile);
RecordType recordType3 = metaData3.getRecordType("MySimpleRecord");
assertEquals(1066, metaData3.getUnionFieldForRecordType(recordType3).getNumber());
assertEquals(1066L, recordType3.getRecordTypeKey());
RecordType recordType4 = metaData4.getRecordType("MySimpleRecord");
assertEquals(1066, metaData4.getUnionFieldForRecordType(recordType4).getNumber());
assertEquals(800L, recordType4.getRecordTypeKey());
assertInvalid("record type key changed", metaData3, metaData4);
}
// Record types tests
@Nonnull
private RecordMetaData addNewRecordType(@Nonnull RecordMetaData metaData, @Nonnull Consumer<RecordMetaDataProto.RecordType.Builder> newRecordTypeHook) {
RecordMetaDataProto.RecordType.Builder newRecordTypeBuilder = RecordMetaDataProto.RecordType.newBuilder()
.setName("NewRecord")
.setPrimaryKey(Key.Expressions.field("rec_no").toKeyExpression())
.setSinceVersion(metaData.getVersion() + 1);
newRecordTypeHook.accept(newRecordTypeBuilder);
return RecordMetaData.build(metaData.toProto().toBuilder()
.setVersion(metaData.getVersion() + 1)
.addDependencies(TestRecords1Proto.getDescriptor().toProto())
.setRecords(TestNewRecordTypeProto.getDescriptor().toProto())
.addRecordTypes(newRecordTypeBuilder)
.build()
);
}
@Nonnull
private RecordMetaData addNewRecordType(@Nonnull RecordMetaData metaData) {
return addNewRecordType(metaData, ignore -> { });
}
@Test
public void newTypeWithoutSinceVersion() {
RecordMetaData metaData1 = RecordMetaData.build(TestRecords1Proto.getDescriptor());
RecordMetaData metaData2 = addNewRecordType(metaData1, RecordMetaDataProto.RecordType.Builder::clearSinceVersion);
assertInvalid("new record type is missing since version", metaData1, metaData2);
MetaDataEvolutionValidator laxerValidator = MetaDataEvolutionValidator.newBuilder()
.setAllowNoSinceVersion(true)
.build();
laxerValidator.validate(metaData1, metaData2);
}
@Test
public void newTypeWithOlderSinceVersion() {
RecordMetaData metaData1 = RecordMetaData.build(TestRecords1Proto.getDescriptor());
RecordMetaData metaData2 = addNewRecordType(metaData1, protoBuilder -> protoBuilder.setSinceVersion(metaData1.getVersion() - 1));
assertInvalid("new record type has since version older than old meta-data", metaData1, metaData2);
RecordMetaData metaData3 = addNewRecordType(metaData1, protoBuilder -> protoBuilder.setSinceVersion(metaData1.getVersion()));
assertInvalid("new record type has since version older than old meta-data", metaData1, metaData3);
}
@Test
public void recordTypeKeyChanged() {
RecordMetaData metaData1 = RecordMetaData.build(TestRecords1Proto.getDescriptor());
RecordMetaDataProto.MetaData.Builder protoBuilder = metaData1.toProto().toBuilder()
.setVersion(metaData1.getVersion() + 1);
protoBuilder.getRecordTypesBuilder(0)
.setExplicitKey(RecordMetaDataProto.Value.newBuilder()
.setStringValue("new_key"));
RecordMetaData metaData2 = RecordMetaData.build(protoBuilder.build());
assertInvalid("record type key changed", metaData1, metaData2);
}
// Former index tests
@Test
public void removeFormerIndex() {
RecordMetaDataBuilder metaData1Builder = RecordMetaData.newBuilder().setRecords(TestRecords1Proto.getDescriptor());
metaData1Builder.removeIndex("MySimpleRecord$str_value_indexed");
RecordMetaData metaData1 = metaData1Builder.getRecordMetaData();
RecordMetaData metaData2 = RecordMetaData.build(
metaData1.toProto().toBuilder().setVersion(metaData1.getVersion() + 1).clearFormerIndexes().build()
);
assertInvalid("former index removed", metaData1, metaData2);
}
@Test
public void changeFormerIndexVersion() {
RecordMetaDataBuilder metaData1Builder = RecordMetaData.newBuilder().setRecords(TestRecords1Proto.getDescriptor());
metaData1Builder.removeIndex("MySimpleRecord$str_value_indexed");
RecordMetaData metaData1 = metaData1Builder.getRecordMetaData();
RecordMetaDataProto.MetaData.Builder metaData2ProtoBuilder = metaData1.toProto()
.toBuilder()
.setVersion(metaData1.getVersion() + 1);
metaData2ProtoBuilder.setFormerIndexes(0, metaData2ProtoBuilder.getFormerIndexesBuilder(0).setRemovedVersion(metaData1.getVersion() + 1));
RecordMetaData metaData2 = RecordMetaData.build(metaData2ProtoBuilder.build());
assertInvalid("removed version of former index differs from prior version", metaData1, metaData2);
metaData2ProtoBuilder.setFormerIndexes(0, metaData2ProtoBuilder.getFormerIndexesBuilder(0).setRemovedVersion(metaData1.getVersion()).setAddedVersion(metaData1.getVersion() - 2));
metaData2 = RecordMetaData.build(metaData2ProtoBuilder.build());
assertInvalid("added version of former index differs from prior version", metaData1, metaData2);
metaData2ProtoBuilder.setFormerIndexes(0, metaData2ProtoBuilder.getFormerIndexesBuilder(0).clearAddedVersion());
metaData2 = RecordMetaData.build(metaData2ProtoBuilder.build());
assertInvalid("added version of former index differs from prior version", metaData1, metaData2);
}
@Test
public void changeFormerIndexName() {
RecordMetaDataBuilder metaData1Builder = RecordMetaData.newBuilder().setRecords(TestRecords1Proto.getDescriptor());
metaData1Builder.removeIndex("MySimpleRecord$str_value_indexed");
RecordMetaData metaData1 = metaData1Builder.getRecordMetaData();
RecordMetaDataProto.MetaData.Builder metaData2ProtoBuilder = metaData1.toProto()
.toBuilder()
.setVersion(metaData1.getVersion() + 1);
metaData2ProtoBuilder.setFormerIndexes(0, metaData2ProtoBuilder.getFormerIndexesBuilder(0).setFormerName("some_other_name"));
RecordMetaData metaData2 = RecordMetaData.build(metaData2ProtoBuilder.build());
assertInvalid("name of former index differs from prior version", metaData1, metaData2);
metaData2ProtoBuilder.setFormerIndexes(0, metaData2ProtoBuilder.getFormerIndexesBuilder(0).clearFormerName());
metaData2 = RecordMetaData.build(metaData2ProtoBuilder.build());
assertInvalid("name of former index differs from prior version", metaData1, metaData2);
// For existing former indexes, changing the name is not allowed even with the laxer validation option
MetaDataEvolutionValidator laxerValidator = MetaDataEvolutionValidator.newBuilder()
.setAllowMissingFormerIndexNames(true)
.build();
assertInvalid("name of former index differs from prior version", laxerValidator, metaData1, metaData2);
}
@Test
public void formerIndexFromThePast() {
RecordMetaData metaData1 = RecordMetaData.build(TestRecords1Proto.getDescriptor());
RecordMetaData metaData2 = RecordMetaData.build(
metaData1.toProto().toBuilder()
.setVersion(metaData1.getVersion() + 1)
.addFormerIndexes(RecordMetaDataProto.FormerIndex.newBuilder()
.setSubspaceKey(ByteString.copyFrom(Tuple.from("dummy_key").pack()))
.setRemovedVersion(metaData1.getVersion() - 1)
.build())
.build()
);
assertInvalid("new former index has removed version that is not newer than the old meta-data version", metaData1, metaData2);
RecordMetaData metaData3 = RecordMetaData.build(
metaData1.toProto().toBuilder()
.setVersion(metaData1.getVersion() + 1)
.addFormerIndexes(RecordMetaDataProto.FormerIndex.newBuilder()
.setSubspaceKey(ByteString.copyFrom(Tuple.from("dummy_key").pack()))
.setRemovedVersion(metaData1.getVersion())
.build())
.build()
);
assertInvalid("new former index has removed version that is not newer than the old meta-data version", metaData1, metaData3);
}
@Test
public void formerIndexWithoutExistingIndex() {
RecordMetaData metaData1 = RecordMetaData.build(TestRecords1Proto.getDescriptor());
RecordMetaData metaData2 = RecordMetaData.build(
metaData1.toProto().toBuilder()
.setVersion(metaData1.getVersion() + 2)
.addFormerIndexes(RecordMetaDataProto.FormerIndex.newBuilder()
.setSubspaceKey(ByteString.copyFrom(Tuple.from("dummy_key").pack()))
.setRemovedVersion(metaData1.getVersion() + 2)
.setAddedVersion(metaData1.getVersion() + 1)
.build())
.build()
);
validator.validate(metaData1, metaData2);
metaData2 = RecordMetaData.build(
metaData2.toProto().toBuilder()
.setFormerIndexes(0, metaData2.getFormerIndexes().get(0).toProto().toBuilder()
.setAddedVersion(metaData1.getVersion())
.build())
.build()
);
assertInvalid("former index without existing index has added version prior to old meta-data version", metaData1, metaData2);
MetaDataEvolutionValidator laxerValidator = MetaDataEvolutionValidator.newBuilder()
.setAllowOlderFormerIndexAddedVerions(true)
.build();
laxerValidator.validate(metaData1, metaData2);
}
@Test
public void indexUsedWhereFormerIndexWas() {
RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder().setRecords(TestRecords1Proto.getDescriptor());
metaDataBuilder.removeIndex("MySimpleRecord$str_value_indexed");
RecordMetaData metaData1 = metaDataBuilder.getRecordMetaData();
RecordMetaDataBuilder metaData2Builder = RecordMetaData.newBuilder().setRecords(
metaData1.toProto().toBuilder().clearFormerIndexes().build()
);
Index newIndex = new Index("newIndex", "str_value_indexed");
newIndex.setSubspaceKey("MySimpleRecord$str_value_indexed");
metaData2Builder.addIndex("MySimpleRecord", newIndex);
RecordMetaData metaData2 = metaData2Builder.getRecordMetaData();
assertInvalid("former index key used for new index in meta-data", metaData1, metaData2);
}
@Test
public void removeIndexAndChangeName() {
RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder().setRecords(TestRecords1Proto.getDescriptor());
// FIXME: Calling getRecordMetaData appears to pollute the FormerIndexes list
RecordMetaData metaData1 = RecordMetaData.build(metaDataBuilder.getRecordMetaData().toProto());
metaDataBuilder.removeIndex("MySimpleRecord$str_value_indexed");
RecordMetaDataProto.MetaData metaData2Proto = metaDataBuilder.getRecordMetaData().toProto();
RecordMetaData metaData2 = RecordMetaData.build(
metaData2Proto.toBuilder()
.setVersion(metaData2Proto.getVersion() + 1)
.removeFormerIndexes(0)
.addFormerIndexes(metaData2Proto.getFormerIndexes(0).toBuilder().setFormerName("some_other_name"))
.build()
);
assertInvalid("former index has different name", metaData1, metaData2);
// Dropping the name is fine if and only if the corresponding option is set
RecordMetaData metaData3 = RecordMetaData.newBuilder().setRecords(
metaData2Proto.toBuilder()
.setVersion(metaData2Proto.getVersion() + 1)
.removeFormerIndexes(0)
.addFormerIndexes(metaData2Proto.getFormerIndexes(0).toBuilder().clearFormerName())
.build()
).getRecordMetaData();
assertInvalid("former index has different name", metaData1, metaData3);
MetaDataEvolutionValidator laxerValidator = MetaDataEvolutionValidator.newBuilder()
.setAllowMissingFormerIndexNames(true)
.build();
laxerValidator.validate(metaData1, metaData3);
}
@Test
public void removeIndexAndDropAddedVersion() {
RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder().setRecords(TestRecords1Proto.getDescriptor());
RecordMetaData metaData1 = RecordMetaData.build(metaDataBuilder.getRecordMetaData().toProto());
metaDataBuilder.removeIndex("MySimpleRecord$str_value_indexed");
RecordMetaDataProto.MetaData metaData2Proto = metaDataBuilder.getRecordMetaData().toProto();
RecordMetaData metaData2 = RecordMetaData.build(
metaData2Proto.toBuilder()
.removeFormerIndexes(0)
.addFormerIndexes(metaData2Proto.getFormerIndexes(0).toBuilder().clearAddedVersion())
.build()
);
assertInvalid("former index reports added version older than replacing index", metaData1, metaData2);
// With the option set, it should validate
MetaDataEvolutionValidator laxerValidator = MetaDataEvolutionValidator.newBuilder()
.setAllowOlderFormerIndexAddedVerions(true)
.build();
laxerValidator.validate(metaData1, metaData2);
}
/**
* This test is supposed to validate that the "default" way of removing an index actually updates fields
* in a safe way. As such, it is more to validate the methods on a RecordMetaDataBuilder that mutate version
* information than the evolution validator.
*/
@Test
public void defaultIndexRemovalPath() {
final String newIndexName = "MySimpleRecord$num_value_2";
RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder().setRecords(TestRecords1Proto.getDescriptor());
metaDataBuilder.addIndex("MySimpleRecord", newIndexName, "num_value_2");
RecordMetaData metaData1 = metaDataBuilder.getRecordMetaData();
assertNotNull(metaData1.getIndex(newIndexName));
assertEquals(metaData1.getVersion(), metaData1.getIndex(newIndexName).getAddedVersion());
assertEquals(metaData1.getVersion(), metaData1.getIndex(newIndexName).getLastModifiedVersion());
metaDataBuilder.removeIndex(newIndexName);
RecordMetaData metaData2 = metaDataBuilder.getRecordMetaData();
assertEquals(1, metaData2.getFormerIndexes().size());
final FormerIndex newFormerIndex = metaData2.getFormerIndexes().get(0);
assertEquals(newIndexName, newFormerIndex.getFormerName());
assertEquals(metaData1.getVersion(), newFormerIndex.getAddedVersion());
assertEquals(metaData2.getVersion(), newFormerIndex.getRemovedVersion());
validator.validate(metaData1, metaData2);
}
// Index tests
@Nonnull
private RecordMetaDataProto.Index changeOption(@Nonnull RecordMetaDataProto.Index indexProto, @Nonnull String key, @Nullable String value) {
RecordMetaDataProto.Index.Builder builder = indexProto.toBuilder();
boolean found = false;
for (int i = 0; i < builder.getOptionsCount(); i++) {
final RecordMetaDataProto.Index.Option option = builder.getOptions(i);
if (key.equals(option.getKey())) {
if (value == null) {
builder.removeOptions(i);
} else {
builder.setOptions(i, RecordMetaDataProto.Index.Option.newBuilder().setKey(key).setValue(value));
}
found = true;
break;
}
}
if (!found && value != null) {
builder.addOptions(RecordMetaDataProto.Index.Option.newBuilder().setKey(key).setValue(value));
}
return builder.build();
}
@Nonnull
private RecordMetaDataProto.Index makeUnique(@Nonnull RecordMetaDataProto.Index indexProto) {
return changeOption(indexProto, IndexOptions.UNIQUE_OPTION, "true");
}
@Nonnull
private RecordMetaDataProto.Index clearOptions(@Nonnull RecordMetaDataProto.Index indexProto) {
return indexProto.toBuilder().clearOptions().build();
}
@Nonnull
private RecordMetaData replaceIndex(@Nonnull RecordMetaData metaData, @Nonnull String indexName, UnaryOperator<RecordMetaDataProto.Index> indexReplacement) {
RecordMetaDataProto.MetaData metaDataProto = metaData.toProto();
RecordMetaDataProto.MetaData.Builder metaDataProtoBuilder = metaDataProto.toBuilder();
metaDataProtoBuilder.setVersion(metaData.getVersion() + 1);
for (int i = 0; i < metaDataProto.getIndexesCount(); i ++) {
RecordMetaDataProto.Index indexProto = metaDataProto.getIndexes(i);
if (indexProto.getName().equals(indexName)) {
metaDataProtoBuilder.setIndexes(i, indexReplacement.apply(indexProto));
}
}
return RecordMetaData.build(metaDataProtoBuilder.build());
}
@Test
public void silentlyRemoveIndex() {
RecordMetaData metaData1 = RecordMetaData.build(TestRecords1Proto.getDescriptor());
RecordMetaData metaData2 = RecordMetaData.build(
metaData1.toProto().toBuilder().setVersion(metaData1.getVersion() + 1).removeIndexes(0).build()
);
assertInvalid("index missing in new meta-data", metaData1, metaData2);
}
@Test
public void newIndexFromThePast() {
RecordMetaData metaData1 = RecordMetaData.build(TestRecords1Proto.getDescriptor());
Index newIndex = new Index("newIndex", Key.Expressions.field("num_value_2"));
newIndex.setAddedVersion(metaData1.getVersion() - 1);
newIndex.setLastModifiedVersion(metaData1.getVersion() - 1);
RecordMetaDataBuilder metaData2Builder = RecordMetaData.newBuilder().setRecords(TestRecords1Proto.getDescriptor());
metaData2Builder.addIndex(metaData2Builder.getRecordType("MySimpleRecord"), newIndex);
metaData2Builder.setVersion(metaData1.getVersion() + 1);
RecordMetaData metaData2 = metaData2Builder.getRecordMetaData();
assertInvalid("new index has version that is not newer than the old meta-data version", metaData1, metaData2);
RecordMetaDataBuilder metaData3Builder = RecordMetaData.newBuilder().setRecords(TestRecords1Proto.getDescriptor());
newIndex.setAddedVersion(metaData1.getVersion());
metaData3Builder.addIndex(metaData2Builder.getRecordType("MySimpleRecord"), newIndex);
metaData2Builder.setVersion(metaData1.getVersion() + 1);
RecordMetaData metaData3 = metaData2.getRecordMetaData();
assertInvalid("new index has version that is not newer than the old meta-data version", metaData1, metaData3);
}
@Test
public void indexSubspaceKeyChanged() {
// The index subspace key is the thing that determines whether an index is even there, so changing it
// is identical to removing the index
RecordMetaData metaData1 = RecordMetaData.build(TestRecords1Proto.getDescriptor());
RecordMetaData metaData2 = replaceIndex(metaData1, "MySimpleRecord$str_value_indexed", indexProto ->
indexProto.toBuilder().setSubspaceKey(ByteString.copyFrom(Tuple.from("dummy_key").pack())).build()
);
assertInvalid("index missing in new meta-data", metaData1, metaData2);
}
@Test
public void indexNameChanged() {
RecordMetaData metaData1 = RecordMetaData.build(TestRecords1Proto.getDescriptor());
RecordMetaData metaData2 = replaceIndex(metaData1, "MySimpleRecord$str_value_indexed", indexProto ->
indexProto.toBuilder()
.setSubspaceKey(ByteString.copyFrom(Tuple.from("MySimpleRecord$str_value_indexed").pack()))
.setName("a_different_name")
.build()
);
assertInvalid("index name changed", metaData1, metaData2);
}
@Test
public void indexAddedVersionChanged() {
RecordMetaData metaData1 = RecordMetaData.build(TestRecords1Proto.getDescriptor());
RecordMetaData metaData2 = replaceIndex(metaData1, "MySimpleRecord$str_value_indexed", indexProto ->
indexProto.toBuilder().setAddedVersion(metaData1.getVersion() + 1).setLastModifiedVersion(metaData1.getVersion() + 1).build()
);
assertInvalid("new index added version does not match old index added version", metaData1, metaData2);
metaData2 = replaceIndex(metaData2, "MySimpleRecord$str_value_indexed", indexProto ->
indexProto.toBuilder().setAddedVersion(indexProto.getAddedVersion() - 1).build()
);
assertInvalid("new index added version does not match old index added version", metaData1, metaData2);
}
@Test
public void indexLastModifiedVersionTooOld() {
RecordMetaData metaData1 = replaceIndex(RecordMetaData.build(TestRecords1Proto.getDescriptor()), "MySimpleRecord$str_value_indexed", indexProto ->
indexProto.toBuilder().setLastModifiedVersion(2).build()
);
RecordMetaData metaData2 = replaceIndex(metaData1, "MySimpleRecord$str_value_indexed", indexProto ->
indexProto.toBuilder().setLastModifiedVersion(1).build()
);
assertInvalid("old index has last-modified version newer than new index", metaData1, metaData2);
}
@Test
public void indexLastModifiedVersionChanged() {
RecordMetaData metaData1 = RecordMetaData.build(TestRecords1Proto.getDescriptor());
RecordMetaData metaData2 = replaceIndex(metaData1, "MySimpleRecord$str_value_indexed", indexProto ->
indexProto.toBuilder().setLastModifiedVersion(metaData1.getVersion() + 1).build()
);
assertInvalid("last modified version of index changed", metaData1, metaData2);
MetaDataEvolutionValidator laxerValidator = MetaDataEvolutionValidator.newBuilder()
.setAllowIndexRebuilds(true)
.build();
laxerValidator.validate(metaData1, metaData2);
}
private void validateIndexMutation(@Nonnull String errMsg, @Nonnull RecordMetaData metaData1, @Nonnull String indexName, UnaryOperator<RecordMetaDataProto.Index> indexReplacement) {
MetaDataEvolutionValidator laxerValidator = MetaDataEvolutionValidator.newBuilder()
.setAllowIndexRebuilds(true)
.build();
RecordMetaData metaData2 = replaceIndex(metaData1, indexName, indexReplacement);
assertInvalid(errMsg, metaData1, metaData2);
assertInvalid(errMsg, laxerValidator, metaData1, metaData2);
// Allow the change if and only if the last modified version is updated and the option allowing rebuilds is set
RecordMetaData metaData3 = replaceIndex(metaData2, indexName, indexProto ->
indexProto.toBuilder().setLastModifiedVersion(metaData2.getVersion()).build()
);
assertInvalid("last modified version of index changed", metaData1, metaData3);
laxerValidator.validate(metaData1, metaData3);
}
@Test
public void indexTypeChanged() {
RecordMetaData metaData1 = RecordMetaData.build(TestRecords1Proto.getDescriptor());
validateIndexMutation("index type changed", metaData1, "MySimpleRecord$str_value_indexed", indexProto ->
indexProto.toBuilder().setType(IndexTypes.RANK).build()
);
}
@Test
public void indexKeyExpressionChanged() {
RecordMetaData metaData1 = RecordMetaData.build(TestRecords1Proto.getDescriptor());
validateIndexMutation("index key expression changed", metaData1, "MySimpleRecord$str_value_indexed", indexProto ->
indexProto.toBuilder().setRootExpression(Key.Expressions.field("num_value_2").toKeyExpression()).build()
);
}
@Test
public void indexRecordTypeRemoved() {
final String indexName = "simple&other$num_value_2";
RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder().setRecords(TestRecords1Proto.getDescriptor());
metaDataBuilder.addMultiTypeIndex(Arrays.asList(metaDataBuilder.getRecordType("MySimpleRecord"), metaDataBuilder.getRecordType("MyOtherRecord")),
new Index(indexName, "num_value_2"));
RecordMetaData metaData1 = metaDataBuilder.getRecordMetaData();
validateIndexMutation("new index removes record type", metaData1, indexName, indexProto ->
indexProto.toBuilder().clearRecordType().addRecordType("MySimpleRecord").build()
);
}
@Test
public void indexRecordTypeAdded() {
RecordMetaData metaData1 = RecordMetaData.build(TestRecords1Proto.getDescriptor());
validateIndexMutation("new index adds record type that is not newer than old meta-data", metaData1, "MySimpleRecord$num_value_3_indexed", indexProto ->
indexProto.toBuilder().addRecordType("MyOtherRecord").build()
);
// Add NewRecord as a record type to the existing meta-data and index and validate that this change is okay
// because the new record type is newer the old meta-data version.
RecordMetaData tempMetaData = addNewRecordType(metaData1);
RecordMetaData metaData2 = replaceIndex(tempMetaData, "MySimpleRecord$num_value_3_indexed", indexProto ->
indexProto.toBuilder().addRecordType("NewRecord").build());
validator.validate(metaData1, metaData2); // valid if type and index change happen together
assertInvalid("new index adds record type that is not newer than old meta-data", tempMetaData, metaData2);
}
@Test
public void indexPrimaryKeyComponentsChanged() {
RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder().setRecords(TestRecords1Proto.getDescriptor());
metaDataBuilder.addIndex("MySimpleRecord", "rec_no", "rec_no");
RecordMetaData metaData1 = metaDataBuilder.getRecordMetaData();
assertThat(metaData1.getIndex("rec_no").hasPrimaryKeyComponentPositions(), is(true));
RecordMetaData tempMetaData = addNewRecordType(metaData1);
RecordMetaData metaData2 = replaceIndex(tempMetaData, "rec_no", indexProto ->
indexProto.toBuilder().addRecordType("NewRecord").build());
assertInvalid("new index drops primary key component positions", metaData1, metaData2);
// This is essentially the behavior change outlined by: https://github.com/FoundationDB/fdb-record-layer/issues/93
metaData2.getIndex("rec_no").setPrimaryKeyComponentPositions(new int[]{0});
validator.validate(metaData1, metaData2);
metaData1.getIndex("rec_no").setPrimaryKeyComponentPositions(null);
assertInvalid("new index adds primary key component positions", metaData1, metaData2);
metaData1.getIndex("rec_no").setPrimaryKeyComponentPositions(new int[]{0});
metaData2.getIndex("rec_no").setPrimaryKeyComponentPositions(new int[]{1});
assertInvalid("new index changes primary key component positions", metaData1, metaData2);
}
@Test
public void addRecordTypeWithUniversalIndex() {
RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder().setRecords(TestRecords1Proto.getDescriptor());
metaDataBuilder.addUniversalIndex(new Index("rec_no", "rec_no"));
RecordMetaData metaData1 = metaDataBuilder.getRecordMetaData();
assertNotNull(metaData1.getUniversalIndex("rec_no"));
// Keep the index universal
RecordMetaData metaData2 = addNewRecordType(metaData1);
assertNotNull(metaData2.getUniversalIndex("rec_no"));
validator.validate(metaData1, metaData2);
// Make the index a multi-type index on the original record types
RecordMetaData metaData3 = replaceIndex(metaData2, "rec_no", indexProto ->
indexProto.toBuilder().addRecordType("MySimpleRecord").addRecordType("MyOtherRecord").build());
MetaDataException e = assertThrows(MetaDataException.class, () -> metaData3.getUniversalIndex("rec_no"));
assertThat(e.getMessage(), containsString("Index rec_no not defined"));
assertInvalid("new index removes record type", metaData2, metaData3);
validator.validate(metaData1, metaData3);
}
@Test
public void uniquenessConstraintChanged() {
// Adding a uniqueness constraint should throw an error
RecordMetaData metaData1 = RecordMetaData.build(TestRecords1Proto.getDescriptor());
validateIndexMutation("index adds uniqueness constraint", metaData1, "MySimpleRecord$str_value_indexed", this::makeUnique);
// Removing the uniqueness constraint is fine
RecordMetaData metaData2 = replaceIndex(metaData1, "MySimpleRecord$str_value_indexed", this::makeUnique);
RecordMetaData metaData3 = replaceIndex(metaData2, "MySimpleRecord$str_value_indexed", this::clearOptions);
validator.validate(metaData2, metaData3);
RecordMetaData metaData4 = replaceIndex(metaData2, "MySimpleRecord$str_value_indexed", indexProto -> changeOption(indexProto, IndexOptions.UNIQUE_OPTION, "false"));
validator.validate(metaData2, metaData4);
}
@Test
public void allowedForQueriesChanged() {
// Changing this option is always fine
RecordMetaData metaData1 = RecordMetaData.build(TestRecords1Proto.getDescriptor());
RecordMetaData metaData2 = replaceIndex(metaData1, "MySimpleRecord$str_value_indexed",
indexProto -> changeOption(indexProto, IndexOptions.ALLOWED_FOR_QUERY_OPTION, "false"));
validator.validate(metaData1, metaData2);
RecordMetaData metaData3 = replaceIndex(metaData2, "MySimpleRecord$str_value_indexed",
indexProto -> changeOption(indexProto, IndexOptions.ALLOWED_FOR_QUERY_OPTION, "true"));
validator.validate(metaData2, metaData3);
RecordMetaData metaData4 = replaceIndex(metaData3, "MySimpleRecord$str_value_indexed", this::clearOptions);
validator.validate(metaData3, metaData4);
}
@Test
public void unknownOptionChanged() {
RecordMetaData metaData1 = RecordMetaData.build(TestRecords1Proto.getDescriptor());
validateIndexMutation("index option changed", metaData1, "MySimpleRecord$str_value_indexed",
indexProto -> changeOption(indexProto, "dummyOption", "dummyValue"));
RecordMetaData metaData2 = replaceIndex(metaData1, "MySimpleRecord$str_value_indexed",
indexProto -> makeUnique(changeOption(indexProto, "dummyOption", "dummyValue")));
RecordMetaData metaData3 = replaceIndex(metaData2, "MySimpleRecord$str_value_indexed",
indexProto -> changeOption(indexProto, IndexOptions.UNIQUE_OPTION, null));
validator.validate(metaData2, metaData3);
validateIndexMutation("index option changed", metaData3, "MySimpleRecord$str_value_indexed",
indexProto -> changeOption(indexProto, "dummyOption", "dummyValue2"));
}
@Test
public void rankLevelsChanged() {
final String indexName = "MySimpleRecord$rank(num_value_2)";
RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder().setRecords(TestRecords1Proto.getDescriptor());
metaDataBuilder.addIndex("MySimpleRecord", new Index(indexName, Key.Expressions.field("num_value_2").ungrouped(), IndexTypes.RANK));
RecordMetaData metaData1 = metaDataBuilder.getRecordMetaData();
validateIndexMutation("rank levels changed", metaData1, indexName,
indexProto -> changeOption(indexProto, IndexOptions.RANK_NLEVELS, "4"));
validateIndexMutation("rank levels changed", metaData1, indexName,
indexProto -> changeOption(indexProto, IndexOptions.RANK_NLEVELS, "" + RankedSet.MAX_LEVELS));
// Setting the default explicitly is fine
RecordMetaData metaData2 = replaceIndex(metaData1, indexName,
indexProto -> changeOption(indexProto, IndexOptions.RANK_NLEVELS, "" + RankedSet.DEFAULT_LEVELS));
validator.validate(metaData1, metaData2);
RecordMetaData metaData3 = replaceIndex(metaData2, indexName, this::clearOptions);
validator.validate(metaData2, metaData3);
}
@Test
public void textOptionsChanged() {
final String indexName = "MySimpleRecord$text(str_value_indexed)";
RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder().setRecords(TestRecords1Proto.getDescriptor());
metaDataBuilder.addIndex("MySimpleRecord", new Index(indexName, Key.Expressions.field("str_value_indexed"), IndexTypes.TEXT));
RecordMetaData metaData1 = metaDataBuilder.getRecordMetaData();
validateIndexMutation("text tokenizer changed", metaData1, indexName,
indexProto -> changeOption(indexProto, IndexOptions.TEXT_TOKENIZER_NAME_OPTION, AllSuffixesTextTokenizer.NAME));
// Setting the default explicitly is fine
RecordMetaData metaData2 = replaceIndex(metaData1, indexName,
indexProto -> changeOption(indexProto, IndexOptions.TEXT_TOKENIZER_NAME_OPTION, DefaultTextTokenizer.NAME));
validator.validate(metaData1, metaData2);
RecordMetaData metaData3 = replaceIndex(metaData2, indexName, this::clearOptions);
validator.validate(metaData2, metaData3);
// Increasing the tokenizer version is fine, but decreasing it is not
RecordMetaData metaData4 = replaceIndex(metaData3, indexName,
indexProto -> changeOption(indexProto, IndexOptions.TEXT_TOKENIZER_NAME_OPTION, PrefixTextTokenizer.NAME));
RecordMetaData metaData5 = replaceIndex(metaData4, indexName,
indexProto -> changeOption(indexProto, IndexOptions.TEXT_TOKENIZER_VERSION_OPTION, "" + TextTokenizer.GLOBAL_MIN_VERSION));
validator.validate(metaData4, metaData5);
RecordMetaData metaData6 = replaceIndex(metaData5, indexName,
indexProto -> changeOption(indexProto, IndexOptions.TEXT_TOKENIZER_VERSION_OPTION, "" + (TextTokenizer.GLOBAL_MIN_VERSION + 1)));
validator.validate(metaData5, metaData6);
validateIndexMutation("text tokenizer version downgraded", metaData6, indexName,
indexProto -> changeOption(indexProto, IndexOptions.TEXT_TOKENIZER_VERSION_OPTION, "" + TextTokenizer.GLOBAL_MIN_VERSION));
// Changing whether aggressive conflict ranges are allowed is safe
RecordMetaData metaData7 = replaceIndex(metaData6, indexName,
indexProto -> changeOption(indexProto, IndexOptions.TEXT_ADD_AGGRESSIVE_CONFLICT_RANGES_OPTION, "true"));
validator.validate(metaData6, metaData7);
RecordMetaData metaData8 = replaceIndex(metaData7, indexName,
indexProto -> changeOption(indexProto, IndexOptions.TEXT_ADD_AGGRESSIVE_CONFLICT_RANGES_OPTION, "false"));
validator.validate(metaData7, metaData8);
// Changing whether position lists are omitted is safe
RecordMetaData metaData9 = replaceIndex(metaData8, indexName,
indexProto -> changeOption(indexProto, IndexOptions.TEXT_OMIT_POSITIONS_OPTION, "true"));
validator.validate(metaData8, metaData9);
RecordMetaData metaData10 = replaceIndex(metaData9, indexName,
indexProto -> changeOption(indexProto, IndexOptions.TEXT_OMIT_POSITIONS_OPTION, "false"));
validator.validate(metaData9, metaData10);
}
}
