/*
 * RecordMetaDataBuilder.java
 *
 * This source file is part of the FoundationDB open source project
 *
 * Copyright 2015-2018 Apple Inc. and the FoundationDB project authors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.apple.foundationdb.record;

import com.apple.foundationdb.annotation.API;
import com.apple.foundationdb.record.metadata.FormerIndex;
import com.apple.foundationdb.record.metadata.Index;
import com.apple.foundationdb.record.metadata.IndexTypes;
import com.apple.foundationdb.record.metadata.JoinedRecordTypeBuilder;
import com.apple.foundationdb.record.metadata.Key;
import com.apple.foundationdb.record.metadata.MetaDataEvolutionValidator;
import com.apple.foundationdb.record.metadata.MetaDataException;
import com.apple.foundationdb.record.metadata.MetaDataValidator;
import com.apple.foundationdb.record.metadata.RecordType;
import com.apple.foundationdb.record.metadata.RecordTypeBuilder;
import com.apple.foundationdb.record.metadata.RecordTypeIndexesBuilder;
import com.apple.foundationdb.record.metadata.SyntheticRecordType;
import com.apple.foundationdb.record.metadata.SyntheticRecordTypeBuilder;
import com.apple.foundationdb.record.metadata.expressions.FieldKeyExpression;
import com.apple.foundationdb.record.metadata.expressions.KeyExpression;
import com.apple.foundationdb.record.metadata.expressions.LiteralKeyExpression;
import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerRegistry;
import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerRegistryImpl;
import com.apple.foundationdb.record.provider.foundationdb.MetaDataProtoEditor;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import com.google.protobuf.DescriptorProtos;
import com.google.protobuf.Descriptors;

import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Queue;
import java.util.Set;
import java.util.TreeMap;

/**
 * A builder for {@link RecordMetaData}.
 *
 * Meta-data can be built in two ways.
 *
 * <p>
 * <b>From compiled .proto</b><br>
 * Simple single field indexes and single field primary keys can be specified in the Protobuf source file using option extensions.
 * Additional indexes or more complicated primary keys need to be specified with code using this builder.
 * The {@link #setRecords(Descriptors.FileDescriptor, boolean)} method loads the meta-data information from the Protobuf source file. Indexes and other
 * properties such as version are not accessible before calling {@code setRecords}.
 * </p>
 *
 * <p>
 * <b>From a {@link RecordMetaDataProto.MetaData} Protobuf message</b><br>
 * The Protobuf form can store the complete meta-data. The {@link #setRecords(RecordMetaDataProto.MetaData, boolean)} method loads
 * the Protobuf message. Similar to loading the meta-data directly from a Protobuf file descriptor, indexes and other
 * properties are not accessible before calling {@code setRecords}.
 * The Protobuf message may contain all of the dependencies required for resolving the record types and indexes. If some
 * of the dependencies are missing (e.g., when a list of excluded dependencies is passed to {@link RecordMetaData#toProto}), before calling
 * {@code setRecords}, callers must first add the missing dependencies using
 * {@link #addDependency(Descriptors.FileDescriptor)} or {@link #addDependencies(Descriptors.FileDescriptor[])}.
 * The {@code addDependency} or {@code addDependencies}
 * methods can also be used to override the embedded dependencies.
 * @see RecordMetaData#toProto
 * </p>
 *
 */
@API(API.Status.MAINTAINED)
public class RecordMetaDataBuilder implements RecordMetaDataProvider {

    private static final Descriptors.FileDescriptor[] emptyDependencyList = new Descriptors.FileDescriptor[0];
    public static final String DEFAULT_UNION_NAME = "RecordTypeUnion";

    @Nullable
    private Descriptors.FileDescriptor recordsDescriptor;
    @Nullable
    private Descriptors.Descriptor unionDescriptor;
    @Nullable
    private Descriptors.FileDescriptor localFileDescriptor;
    @Nonnull
    private final Map<Descriptors.Descriptor, Descriptors.FieldDescriptor> unionFields;
    @Nonnull
    private final Map<String, RecordTypeBuilder> recordTypes;
    @Nonnull
    private final Map<String, SyntheticRecordTypeBuilder<?>> syntheticRecordTypes;
    @Nonnull
    private final Map<String, Index> indexes;
    @Nonnull
    private final Map<String, Index> universalIndexes;
    private boolean splitLongRecords;
    private boolean storeRecordVersions;
    private int version;
    @Nonnull
    private final List<FormerIndex> formerIndexes;
    @Nonnull
    private IndexMaintainerRegistry indexMaintainerRegistry;
    @Nonnull
    private MetaDataEvolutionValidator evolutionValidator;
    @Nullable
    private KeyExpression recordCountKey;
    @Nullable
    private RecordMetaData recordMetaData;
    @Nonnull
    private final Map<String, Descriptors.FileDescriptor> explicitDependencies;
    private long subspaceKeyCounter = 0;
    private boolean usesSubspaceKeyCounter = false;

    /**
     * Creates a blank builder.
     */
    RecordMetaDataBuilder() {
        recordTypes = new HashMap<>();
        indexes = new HashMap<>();
        universalIndexes = new HashMap<>();
        formerIndexes = new ArrayList<>();
        unionFields = new HashMap<>();
        explicitDependencies = new TreeMap<>();
        indexMaintainerRegistry = IndexMaintainerRegistryImpl.instance();
        evolutionValidator = MetaDataEvolutionValidator.getDefaultInstance();
        syntheticRecordTypes = new HashMap<>();
    }

    /**
     * Creates a new builder from the provided record types protobuf.
     * @param fileDescriptor a file descriptor containing all the record types in the metadata
     * @deprecated use {@link RecordMetaData#newBuilder()} instead
     */
    @Deprecated
    public RecordMetaDataBuilder(@Nonnull Descriptors.FileDescriptor fileDescriptor) {
        this(fileDescriptor, true);
    }

    /**
     * Creates a new builder from the provided record types protobuf.
     * @param fileDescriptor a file descriptor containing all the record types in the metadata
     * @param processExtensionOptions whether to add primary keys and indexes based on extensions in the protobuf
     * @deprecated use {@link RecordMetaData#newBuilder()} instead
     */
    @Deprecated
    public RecordMetaDataBuilder(@Nonnull Descriptors.FileDescriptor fileDescriptor,
                                 boolean processExtensionOptions) {
        this();
        loadFromFileDescriptor(fileDescriptor, processExtensionOptions);
    }

    /**
     * Creates a new builder from the provided meta-data protobuf.
     *
     * This constructor assumes that {@code metaDataProto} is not the result of {@link RecordMetaData#toProto} and will not already
     * include all the indexes defined by any original extension options, so that they still need to be processed.
     * If {@code metaDataProto} is the result of {@code toProto} and indexes also appear in extension options, a duplicate index
     * error will result. In that case, {@link #RecordMetaDataBuilder(RecordMetaDataProto.MetaData, boolean)} will be needed instead.
     *
     * @param metaDataProto the protobuf form of the meta-data
     * @deprecated use {@link RecordMetaData#newBuilder()} instead
     */
    @Deprecated
    public RecordMetaDataBuilder(@Nonnull RecordMetaDataProto.MetaData metaDataProto) {
        this(metaDataProto, true);
    }

    /**
     * Creates a new builder from the provided meta-data protobuf.
     *
     * If {@code metaDataProto} is the result of {@link RecordMetaData#toProto}, it will already
     * include all the indexes defined by any original extension options, so {@code processExtensionOptions}
     * should be {@code false}.
     *
     * @param metaDataProto the protobuf form of the meta-data
     * @param processExtensionOptions whether to add primary keys and indexes based on extensions in the protobuf
     * @deprecated use {@link RecordMetaData#newBuilder()} instead
     */
    @Deprecated
    public RecordMetaDataBuilder(@Nonnull RecordMetaDataProto.MetaData metaDataProto,
                                 boolean processExtensionOptions) {
        this(metaDataProto, new Descriptors.FileDescriptor[] { RecordMetaDataOptionsProto.getDescriptor() }, processExtensionOptions);
    }

    /**
     * Creates a new builder from the provided meta-data protobuf.
     *
     * This constructor assumes that {@code metaDataProto} is not the result of {@link RecordMetaData#toProto} and will not already
     * include all the indexes defined by any original extension options, so that they still need to be processed.
     * If {@code metaDataProto} is the result of {@code toProto} and indexes also appear in extension options, a duplicate index
     * error will result. In that case, {@link #RecordMetaDataBuilder(RecordMetaDataProto.MetaData, Descriptors.FileDescriptor[], boolean)} will be needed instead.
     *
     * @param metaDataProto the protobuf form of the meta-data
     * @param dependencies other files imported by the record types protobuf
     * @deprecated use {@link RecordMetaData#newBuilder()} instead
     */
    @Deprecated
    public RecordMetaDataBuilder(@Nonnull RecordMetaDataProto.MetaData metaDataProto,
                                 @Nonnull Descriptors.FileDescriptor[] dependencies) {
        this(metaDataProto, dependencies, true);
    }

    /**
     * Creates a new builder from the provided meta-data protobuf.
     *
     * If {@code metaDataProto} is the result of {@link RecordMetaData#toProto}, it will already
     * include all the indexes defined by any original extension options, so {@code processExtensionOptions}
     * should be {@code false}.
     *
     * @param metaDataProto the protobuf form of the meta-data
     * @param dependencies other files imported by the record types protobuf
     * @param processExtensionOptions whether to add primary keys and indexes based on extensions in the protobuf
     * @deprecated use {@link RecordMetaData#newBuilder()} instead
     */
    @Deprecated
    public RecordMetaDataBuilder(@Nonnull RecordMetaDataProto.MetaData metaDataProto,
                                 @Nonnull Descriptors.FileDescriptor[] dependencies,
                                 boolean processExtensionOptions) {
        this();
        loadFromProto(metaDataProto, dependencies, processExtensionOptions);
    }

    private void processSchemaOptions(boolean processExtensionOptions) {
        if (processExtensionOptions) {
            RecordMetaDataOptionsProto.SchemaOptions schemaOptions = recordsDescriptor.getOptions()
                    .getExtension(RecordMetaDataOptionsProto.schema);
            if (schemaOptions != null) {
                if (schemaOptions.hasSplitLongRecords()) {
                    splitLongRecords = schemaOptions.getSplitLongRecords();
                }
                if (schemaOptions.hasStoreRecordVersions()) {
                    storeRecordVersions = schemaOptions.getStoreRecordVersions();
                }
            }
        }
    }

    @SuppressWarnings("deprecation")
    private void loadProtoExceptRecords(@Nonnull RecordMetaDataProto.MetaData metaDataProto) {
        for (RecordMetaDataProto.Index indexProto : metaDataProto.getIndexesList()) {
            List<RecordTypeBuilder> recordTypeBuilders = new ArrayList<>(indexProto.getRecordTypeCount());
            for (String recordTypeName : indexProto.getRecordTypeList()) {
                recordTypeBuilders.add(getRecordType(recordTypeName));
            }
            try {
                addMultiTypeIndex(recordTypeBuilders, new Index(indexProto));
            } catch (KeyExpression.DeserializationException e) {
                throw new MetaDataProtoDeserializationException(e);
            }
        }
        for (RecordMetaDataProto.RecordType typeProto : metaDataProto.getRecordTypesList()) {
            RecordTypeBuilder typeBuilder = getRecordType(typeProto.getName());
            if (typeProto.hasPrimaryKey()) {
                try {
                    typeBuilder.setPrimaryKey(KeyExpression.fromProto(typeProto.getPrimaryKey()));
                } catch (KeyExpression.DeserializationException e) {
                    throw new MetaDataProtoDeserializationException(e);
                }
            }
            if (typeProto.hasSinceVersion()) {
                typeBuilder.setSinceVersion(typeProto.getSinceVersion());
            }
            if (typeProto.hasExplicitKey()) {
                typeBuilder.setRecordTypeKey(LiteralKeyExpression.fromProtoValue(typeProto.getExplicitKey()));
            }
        }
        if (metaDataProto.hasSplitLongRecords()) {
            splitLongRecords = metaDataProto.getSplitLongRecords();
        }
        if (metaDataProto.hasStoreRecordVersions()) {
            storeRecordVersions = metaDataProto.getStoreRecordVersions();
        }
        for (RecordMetaDataProto.FormerIndex formerIndex : metaDataProto.getFormerIndexesList()) {
            formerIndexes.add(new FormerIndex(formerIndex));
        }
        if (metaDataProto.hasRecordCountKey()) {
            try {
                recordCountKey = KeyExpression.fromProto(metaDataProto.getRecordCountKey());
            } catch (KeyExpression.DeserializationException e) {
                throw new MetaDataProtoDeserializationException(e);
            }
        }
        if (metaDataProto.hasVersion()) {
            version = metaDataProto.getVersion();
        }
        for (RecordMetaDataProto.JoinedRecordType joinedProto : metaDataProto.getJoinedRecordTypesList()) {
            JoinedRecordTypeBuilder typeBuilder = new JoinedRecordTypeBuilder(joinedProto, this);
            syntheticRecordTypes.put(typeBuilder.getName(), typeBuilder);
        }
    }

    private void loadFromProto(@Nonnull RecordMetaDataProto.MetaData metaDataProto,
                               @Nonnull Descriptors.FileDescriptor[] dependencies,
                               boolean processExtensionOptions) {
        recordsDescriptor = buildFileDescriptor(metaDataProto.getRecords(), dependencies);
        loadSubspaceKeySettingsFromProto(metaDataProto);
        initRecordTypesAndUnion(processExtensionOptions);
        loadProtoExceptRecords(metaDataProto);
        processSchemaOptions(processExtensionOptions);

        // If a local file descriptor has been set, update records types (etc.) to use the local descriptor.
        if (localFileDescriptor != null) {
            Descriptors.Descriptor localUnionDescriptor = fetchLocalUnionDescriptor();
            evolutionValidator.validateUnion(unionDescriptor, localUnionDescriptor);
            updateUnionFieldsAndRecordTypesFromLocal(localUnionDescriptor);
            unionDescriptor = localUnionDescriptor;
        }
    }

    private void loadSubspaceKeySettingsFromProto(RecordMetaDataProto.MetaData metaDataProto) {
        if (metaDataProto.hasSubspaceKeyCounter() && !metaDataProto.getUsesSubspaceKeyCounter()) {
            throw new MetaDataProtoDeserializationException(new MetaDataException("subspaceKeyCounter is set but usesSubspaceKeyCounter is not set in the meta-data proto"));
        }
        if (metaDataProto.getUsesSubspaceKeyCounter() && !metaDataProto.hasSubspaceKeyCounter()) {
            throw new MetaDataProtoDeserializationException(new MetaDataException("usesSubspaceKeyCounter is set but subspaceKeyCounter is not set in the meta-data proto"));
        }
        if (!usesSubspaceKeyCounter()) {
            // Only read from the proto if user has not explicitly enabled it already.
            usesSubspaceKeyCounter = metaDataProto.getUsesSubspaceKeyCounter();
        }
        subspaceKeyCounter = Long.max(subspaceKeyCounter, metaDataProto.getSubspaceKeyCounter()); // User might have set the counter already.
    }

    private void loadFromFileDescriptor(@Nonnull Descriptors.FileDescriptor fileDescriptor,
                                        boolean processExtensionOptions) {
        recordsDescriptor = fileDescriptor;
        initRecordTypesAndUnion(processExtensionOptions);
        processSchemaOptions(processExtensionOptions);
    }

    private void initRecordTypesAndUnion(boolean processExtensionOptions) {
        if (recordsDescriptor == null) {
            // Should not happen as this method should only be called when a file descriptor has been set.
            throw new RecordCoreException("cannot initiate records from null file descriptor");
        }
        unionDescriptor = fetchUnionDescriptor(recordsDescriptor);
        validateRecords(recordsDescriptor, unionDescriptor);
        fillUnionFields(processExtensionOptions);
    }

    @Nonnull
    private static Descriptors.Descriptor fetchUnionDescriptor(@Nonnull Descriptors.FileDescriptor fileDescriptor) {
        @Nullable Descriptors.Descriptor union = null;
        for (Descriptors.Descriptor descriptor : fileDescriptor.getMessageTypes()) {
            RecordMetaDataOptionsProto.RecordTypeOptions recordTypeOptions = descriptor.getOptions()
                    .getExtension(RecordMetaDataOptionsProto.record);
            if (recordTypeOptions != null && recordTypeOptions.hasUsage()) {
                switch (recordTypeOptions.getUsage()) {
                    case UNION:
                        if (union != null) {
                            throw new MetaDataException("Only one union descriptor is allowed");
                        }
                        union = descriptor;
                        continue;
                    case NESTED:
                        if (DEFAULT_UNION_NAME.equals(descriptor.getName())) {
                            throw new MetaDataException("Message type " + DEFAULT_UNION_NAME + " cannot have NESTED usage");
                        }
                        continue;
                    case RECORD:
                    case UNSET:
                    default:
                        break;
                }
            }
            if (DEFAULT_UNION_NAME.equals(descriptor.getName())) {
                if (union != null) {
                    throw new MetaDataException("Only one union descriptor is allowed");
                }
                union = descriptor;
                continue;
            }
        }
        if (union == null) {
            throw new MetaDataException("Union descriptor is required");
        }
        return union;
    }

    @Nonnull
    private static Map<String, Descriptors.FileDescriptor> initGeneratedDependencies(@Nonnull Map<String, DescriptorProtos.FileDescriptorProto> protoDependencies) {
        Map<String, Descriptors.FileDescriptor> generatedDependencies = new TreeMap<>();
        if (!protoDependencies.containsKey(TupleFieldsProto.getDescriptor().getName())) {
            generatedDependencies.put(TupleFieldsProto.getDescriptor().getName(), TupleFieldsProto.getDescriptor());
        }
        if (!protoDependencies.containsKey(RecordMetaDataOptionsProto.getDescriptor().getName())) {
            generatedDependencies.put(RecordMetaDataOptionsProto.getDescriptor().getName(), RecordMetaDataOptionsProto.getDescriptor());
        }
        if (!protoDependencies.containsKey(RecordMetaDataProto.getDescriptor().getName())) {
            generatedDependencies.put(RecordMetaDataProto.getDescriptor().getName(), RecordMetaDataProto.getDescriptor());
        }
        return generatedDependencies;
    }

    /**
     * Deserializes the meta-data proto into the builder. The extension options are not processed.
     * @param metaDataProto the proto of the {@link RecordMetaData}
     * @return this builder
     */
    @Nonnull
    public RecordMetaDataBuilder setRecords(@Nonnull RecordMetaDataProto.MetaData metaDataProto) {
        return setRecords(metaDataProto, false);
    }

    /**
     * Deserializes the meta-data proto into the builder.
     * @param metaDataProto the proto of the {@link RecordMetaData}
     * @param processExtensionOptions whether to add primary keys and indexes based on extensions in the protobuf
     * @return this builder
     */
    @Nonnull
    public RecordMetaDataBuilder setRecords(@Nonnull RecordMetaDataProto.MetaData metaDataProto,
                                            boolean processExtensionOptions) {
        if (recordsDescriptor != null) {
            throw new MetaDataException("Records already set.");
        }
        // Build the recordDescriptor by de-serializing the metaData proto
        Map<String, DescriptorProtos.FileDescriptorProto> protoDependencies = new TreeMap<>();
        for (DescriptorProtos.FileDescriptorProto dependency : metaDataProto.getDependenciesList()) {
            protoDependencies.put(dependency.getName(), dependency);
        }
        Map<String, Descriptors.FileDescriptor> generatedDependencies = initGeneratedDependencies(protoDependencies);
        Descriptors.FileDescriptor[] dependencies = getDependencies(metaDataProto.getRecords(), generatedDependencies, protoDependencies);
        loadFromProto(metaDataProto, dependencies, processExtensionOptions);
        return this;
    }

    /**
     * Adds the root file descriptor of the {@link RecordMetaData} and processes the extension options.
     * @param fileDescriptor the file descriptor of the record meta-data
     * @return this builder
     */
    @Nonnull
    public RecordMetaDataBuilder setRecords(@Nonnull Descriptors.FileDescriptor fileDescriptor) {
        return setRecords(fileDescriptor, true);
    }

    /**
     * Adds the root file descriptor of the {@link RecordMetaData}.
     * @param fileDescriptor the file descriptor of the record meta-data
     * @param processExtensionOptions whether to add primary keys and indexes based on extensions in the protobuf
     * @return this builder
     */
    @Nonnull
    public RecordMetaDataBuilder setRecords(@Nonnull Descriptors.FileDescriptor fileDescriptor,
                                            boolean processExtensionOptions) {
        if (recordsDescriptor != null) {
            throw new MetaDataException("Records already set.");
        }
        if (localFileDescriptor != null) {
            throw new MetaDataException("Cannot set records from file descriptor when local descriptor is specified.");
        }
        if (!explicitDependencies.isEmpty()) {
            throw new MetaDataException("Cannot set records from file descriptor when explicit dependencies are specified.");
        }
        loadFromFileDescriptor(fileDescriptor, processExtensionOptions);
        return this;
    }

    /**
     * Update the records descriptor of the record meta-data.
     *
     * <p>
     * This involves adding new record types and updating descriptors for the existing record types and union fields.
     * By contract, the extension options will be processed for all of the new record types and will not be processed
     * for any of the old record types. Also, it is not allowed to call this method when the local file descriptor is set.
     * </p>
     *
     * <p>
     * See {@link #updateRecords(Descriptors.FileDescriptor, boolean)} for more information.
     * </p>
     *
     * @param recordsDescriptor the new record descriptor
     */
    public void updateRecords(@Nonnull Descriptors.FileDescriptor recordsDescriptor) {
        updateRecords(recordsDescriptor, true);
    }

    /**
     * Update the records descriptor of the record meta-data.
     *
     * <p>
     * This adds any new record types and updates the descriptors for existing record types.
     * By contract, the extension options will not be processed for the old record types. If {@code processExtensionOptions}
     * is set, extension options will be processed only for the new record types. Also, this method may not be called
     * when {@code localFileDescriptor} is set. This method bumps the meta-data version and sets the {@linkplain RecordType#getSinceVersion() since version}
     * for the new record types (and their indexes).
     * </p>
     *
     * <p>
     * To avoid accidental changes, this method only updates the record types (i.e., updates the message
     * descriptors of the old record types and adds new record types with their primary keys and indexes). This method
     * does not process schema options. To update the schema options, use {@link #setSplitLongRecords(boolean)} and
     * {@link #setStoreRecordVersions(boolean)}.
     * </p>
     *
     * @param newRecordsDescriptor the new record descriptor
     * @param processExtensionOptions whether to add primary keys and indexes using the extensions in the protobuf (only for the new record types)
     */
    public void updateRecords(@Nonnull Descriptors.FileDescriptor newRecordsDescriptor, boolean processExtensionOptions) {
        if (recordsDescriptor == null) {
            throw new MetaDataException("Records descriptor is not set yet");
        }
        if (localFileDescriptor != null) {
            throw new MetaDataException("Updating the records descriptor is not allowed when the local file descriptor is set");
        }
        if (unionDescriptor == null) {
            throw new RecordCoreException("cannot update record types as no previous union descriptor has been set");
        }
        Descriptors.Descriptor newUnionDescriptor = fetchUnionDescriptor(newRecordsDescriptor);
        validateRecords(newRecordsDescriptor, newUnionDescriptor);
        evolutionValidator.validateUnion(unionDescriptor, newUnionDescriptor);
        version++; // Bump the meta-data version
        updateUnionFieldsAndRecordTypes(newUnionDescriptor, processExtensionOptions);
        recordsDescriptor = newRecordsDescriptor;
        unionDescriptor = newUnionDescriptor;
    }

    /**
     * Sets the local file descriptor. A local meta-data file descriptor may contain newer versions of record types or indexes.
     *
     * <p>
     * This method is handy when two versions of the record meta-data are compatible (i.e., they follow the
     * <a href="https://foundationdb.github.io/fdb-record-layer/SchemaEvolution.html">Record Layer guidelines on schema evolution</a>),
     * but their descriptors are not equal, e.g., a statically-generated proto and its serialized version stored
     * in a meta-data store (i.e., {@link com.apple.foundationdb.record.provider.foundationdb.FDBMetaDataStore}). A
     * record store created using the meta-data store may not be able to store a record created
     * by the statically-generated proto file because the meta-data and record have mismatched descriptors. Using this method,
     * the meta-data can use the same version of the descriptor as the record.
     * </p>
     *
     * <p>
     * This should only be used when the records descriptor is set through a meta-data Protobuf message (i.e, it is followed by a call
     * to {@link #setRecords(RecordMetaDataProto.MetaData)} or {@link #setRecords(RecordMetaDataProto.MetaData, boolean)}).
     * This will not work if the records are set using a file descriptor. Note also that once the local file descriptor is
     * set, the {@link RecordMetaData} object that is produced from this builder may differ from the original Protobuf definition
     * in ways that make serializing the meta-data back to Protobuf unsafe. As a result, calling {@link RecordMetaData#toProto() toProto()}
     * on the produced {@code RecordMetaData} is disallowed.
     * </p>
     *
     * <p>
     * To verify that the file descriptor supplied here and the file descriptor included in the meta-data proto are
     * compatible, the two descriptors are compared using a {@link MetaDataEvolutionValidator}. The user may provide
     * their own validator by calling {@link #setEvolutionValidator(MetaDataEvolutionValidator)}. By default, this
     * will use that class's {@linkplain MetaDataEvolutionValidator#getDefaultInstance() default instance}.
     * </p>
     *
     * @param localFileDescriptor a file descriptor that contains updated record types
     * @return this builder
     */
    @Nonnull
    public RecordMetaDataBuilder setLocalFileDescriptor(@Nonnull Descriptors.FileDescriptor localFileDescriptor) {
        if (recordsDescriptor != null) {
            throw new MetaDataException("Records already set.");
        }
        this.localFileDescriptor = localFileDescriptor;
        return this;
    }

    @Nonnull
    private Descriptors.Descriptor buildSyntheticUnion(@Nonnull Descriptors.FileDescriptor parentFileDescriptor) {
        if (unionDescriptor == null) {
            throw new RecordCoreException("cannot build a synthetic union descriptor as no prior existing union descriptor has been set");
        }
        DescriptorProtos.FileDescriptorProto.Builder builder = DescriptorProtos.FileDescriptorProto.newBuilder();
        builder.setName("_synthetic_" + parentFileDescriptor.getName());
        builder.addMessageType(MetaDataProtoEditor.createSyntheticUnion(parentFileDescriptor, unionDescriptor));
        builder.addDependency(parentFileDescriptor.getName());
        return fetchUnionDescriptor(buildFileDescriptor(builder.build(), new Descriptors.FileDescriptor[]{parentFileDescriptor}));
    }

    @Nonnull
    private Descriptors.Descriptor fetchLocalUnionDescriptor() {
        if (localFileDescriptor == null) {
            // Should not be reached as caller should guard this call on checking for the local file descriptor
            throw new RecordCoreException("cannot fetch local union descriptor as no local file is set");
        }
        Descriptors.Descriptor localUnionDescriptor;
        if (MetaDataProtoEditor.hasUnion(localFileDescriptor)) {
            localUnionDescriptor = fetchUnionDescriptor(localFileDescriptor);
        } else {
            // The local file descriptor does not have a union. Synthesize it.
            localUnionDescriptor = buildSyntheticUnion(localFileDescriptor);
        }
        validateRecords(localFileDescriptor, localUnionDescriptor);
        return localUnionDescriptor;
    }

    /**
     * Adds a dependency to the list of dependencies. It will be used for loading the {@link RecordMetaData} from a meta-data proto.
     * @param fileDescriptor the file descriptor of the dependency
     * @return this builder
     */
    @Nonnull
    public RecordMetaDataBuilder addDependency(@Nonnull Descriptors.FileDescriptor fileDescriptor) {
        if (recordsDescriptor != null) {
            throw new MetaDataException("Records already set. Adding dependencies not allowed.");
        }
        explicitDependencies.put(fileDescriptor.getName(), fileDescriptor);
        return this;
    }

    /**
     * Adds dependencies to be used for loading the {@link RecordMetaData} from a meta-data proto.
     * @param fileDescriptors a list of dependencies
     * @return this builder
     */
    @Nonnull
    public RecordMetaDataBuilder addDependencies(@Nonnull Descriptors.FileDescriptor[] fileDescriptors) {
        if (recordsDescriptor != null) {
            throw new MetaDataException("Records already set. Adding dependencies not allowed.");
        }
        for (Descriptors.FileDescriptor fileDescriptor : fileDescriptors) {
            explicitDependencies.put(fileDescriptor.getName(), fileDescriptor);
        }
        return this;
    }

    private Descriptors.FileDescriptor[] getDependencies(@Nonnull DescriptorProtos.FileDescriptorProto proto,
                                                         @Nonnull Map<String, Descriptors.FileDescriptor> generatedDependencies,
                                                         @Nonnull Map<String, DescriptorProtos.FileDescriptorProto> protoDependencies) {
        if (proto.getDependencyCount() == 0) {
            return emptyDependencyList;
        }

        Descriptors.FileDescriptor[] dependencies = new Descriptors.FileDescriptor[proto.getDependencyCount()];
        for (int index = 0; index < proto.getDependencyCount(); index++) {
            String key = proto.getDependency(index);
            if (this.explicitDependencies.containsKey(key)) {
                // Provided by caller.
                dependencies[index] = this.explicitDependencies.get(key);
            } else if (generatedDependencies.containsKey(key)) {
                // Generated already.
                dependencies[index] = generatedDependencies.get(key);
            } else if (protoDependencies.containsKey(key)) {
                // Not seen before. Build it.
                DescriptorProtos.FileDescriptorProto dependency = protoDependencies.get(key);
                dependencies[index] = buildFileDescriptor(dependency, getDependencies(dependency, generatedDependencies, protoDependencies));
                generatedDependencies.put(key, dependencies[index]);
            } else {
                // Unknown dependency.
                throw new MetaDataException(String.format("Dependency %s not found", key));
            }
        }
        return dependencies;
    }

    private static void validateRecords(@Nonnull Descriptors.FileDescriptor fileDescriptor, @Nonnull Descriptors.Descriptor unionDescriptor) {
        validateDataTypes(fileDescriptor);
        validateUnion(fileDescriptor, unionDescriptor);
    }

    private static void validateDataTypes(@Nonnull Descriptors.FileDescriptor fileDescriptor) {
        Queue<Descriptors.Descriptor> toValidate = new ArrayDeque<>(fileDescriptor.getMessageTypes());
        Set<Descriptors.Descriptor> seen = new HashSet<>();
        while (!toValidate.isEmpty()) {
            Descriptors.Descriptor descriptor = toValidate.remove();
            if (seen.add(descriptor)) {
                for (Descriptors.FieldDescriptor field : descriptor.getFields()) {
                    switch (field.getType()) {
                        case INT32:
                        case INT64:
                        case SFIXED32:
                        case SFIXED64:
                        case SINT32:
                        case SINT64:
                        case BOOL:
                        case STRING:
                        case BYTES:
                        case FLOAT:
                        case DOUBLE:
                        case ENUM:
                            // These types are allowed ; nothing to do.
                            break;
                        case MESSAGE:
                        case GROUP:
                            if (!seen.contains(field.getMessageType())) {
                                toValidate.add(field.getMessageType());
                            }
                            break;
                        case FIXED32:
                        case FIXED64:
                        case UINT32:
                        case UINT64:
                            throw new MetaDataException(
                                    "Field " + field.getName()
                                    + " in message " + descriptor.getFullName()
                                    + " has illegal unsigned type " + field.getType().name());
                        default:
                            throw new MetaDataException(
                                    "Field " + field.getName()
                                    + " in message " + descriptor.getFullName()
                                    + " has unknown type " + field.getType().name());
                    }
                }
            }
        }
    }

    private static void validateUnion(@Nonnull Descriptors.FileDescriptor fileDescriptor, @Nonnull Descriptors.Descriptor unionDescriptor) {
        for (Descriptors.FieldDescriptor unionField : unionDescriptor.getFields()) {
            // Only message types allowed.
            if (unionField.getType() != Descriptors.FieldDescriptor.Type.MESSAGE) {
                throw new MetaDataException("Union field " + unionField.getName() +
                                            " is not a message");
            }
            // No repeating fields.
            if (unionField.isRepeated()) {
                throw new MetaDataException("Union field " + unionField.getName() +
                                            " should not be repeated");
            }
            Descriptors.Descriptor descriptor = unionField.getMessageType();
            // RecordTypeUnion is reserved for union descriptor and cannot appear as a union fields
            if (DEFAULT_UNION_NAME.equals(descriptor.getName())) {
                throw new MetaDataException("Union message type " + descriptor.getName()
                                            + " cannot be a union field.");
            }

            // All union fields must be RECORD (i.e., they cannot be NESTED/UNIONs). The same rule applies to imported union fields too.
            RecordMetaDataOptionsProto.RecordTypeOptions recordTypeOptions = descriptor.getOptions()
                    .getExtension(RecordMetaDataOptionsProto.record);
            if (recordTypeOptions != null &&
                    recordTypeOptions.hasUsage() &&
                    recordTypeOptions.getUsage() != RecordMetaDataOptionsProto.RecordTypeOptions.Usage.RECORD) {
                throw new MetaDataException("Union field " + unionField.getName() +
                                            " has type " + descriptor.getName() +
                                            " which is not a record");
            }
        }

        // All RECORD message types defined in this proto must be present in the union.
        for (Descriptors.Descriptor descriptor : fileDescriptor.getMessageTypes()) {
            RecordMetaDataOptionsProto.RecordTypeOptions recordTypeOptions = descriptor.getOptions()
                    .getExtension(RecordMetaDataOptionsProto.record);
            if (recordTypeOptions != null && recordTypeOptions.hasUsage()) {
                switch (recordTypeOptions.getUsage()) {
                    case RECORD:
                        if (DEFAULT_UNION_NAME.equals(descriptor.getName())) {
                            if (unionHasMessageType(unionDescriptor, descriptor)) {
                                throw new MetaDataException("Union message type " + descriptor.getName()
                                                            + " cannot be a union field.");
                            }
                        } else if (!unionHasMessageType(unionDescriptor, descriptor)) {
                            throw new MetaDataException("Record message type " + descriptor.getName()
                                                        + " must be a union field.");
                        }
                        break;
                    case UNION:
                        // Already checked above that none of the union fields is UNION.
                    case NESTED:
                        // Already checked above that none of the union fields is NESTED.
                    case UNSET:
                        // Unset message types can appear in either union fields or as nested messages. The only possible issue
                        // is that the message type's name might equal DEFAULT_UNION_NAME. This is already checked above.
                    default:
                        break;
                }
            }
        }
    }

    private static boolean unionHasMessageType(@Nonnull Descriptors.Descriptor unionDescriptor, @Nonnull Descriptors.Descriptor descriptor) {
        return unionDescriptor.getFields().stream().anyMatch(field -> descriptor == field.getMessageType());
    }


    private void updateUnionFieldsAndRecordTypes(@Nonnull Descriptors.Descriptor union, boolean processExtensionOptions) {
        final Map<String, RecordTypeBuilder> oldRecordTypes = ImmutableMap.copyOf(recordTypes);
        recordTypes.clear();
        unionFields.clear();
        for (Descriptors.FieldDescriptor unionField : union.getFields()) {
            Descriptors.Descriptor newDescriptor = unionField.getMessageType();
            Descriptors.Descriptor oldDescriptor = findOldDescriptor(unionField, union);
            if (unionFields.containsKey(newDescriptor)) {
                if (!recordTypes.containsKey(newDescriptor.getName())) {
                    // Union field was seen before but the record type is unknown? This must not happen.
                    throw new MetaDataException("Unknown record type for union field " + unionField.getName());
                }
                // For existing record types, the preferred field is the last one, except if there is one whose name matches.
                remapUnionField(newDescriptor, unionField);
            } else if (oldDescriptor == null) {
                // New field and record type.
                RecordTypeBuilder recordType = processRecordType(unionField, processExtensionOptions);
                if (recordType.getSinceVersion() != null && recordType.getSinceVersion() != version) {
                    throw new MetaDataException(String.format("Record type version (%d) does not match meta-data version (%d)",
                            recordType.getSinceVersion(), version));
                } else {
                    recordType.setSinceVersion(version);
                }
                unionFields.put(newDescriptor, unionField);
            } else {
                updateRecordType(oldRecordTypes, oldDescriptor, newDescriptor);
                unionFields.put(newDescriptor, unionField);
            }
        }
    }

    private void updateUnionFieldsAndRecordTypesFromLocal(@Nonnull Descriptors.Descriptor union) {
        final Map<String, RecordTypeBuilder> oldRecordTypes = ImmutableMap.copyOf(recordTypes);
        final Map<Descriptors.Descriptor, Descriptors.FieldDescriptor> oldUnionFields = ImmutableMap.copyOf(unionFields);
        recordTypes.clear();
        unionFields.clear();
        for (Descriptors.FieldDescriptor unionField : union.getFields()) {
            Descriptors.Descriptor newDescriptor = unionField.getMessageType();
            Descriptors.Descriptor oldDescriptor = findOldDescriptor(unionField, union);
            if (oldDescriptor == null) {
                // If updating from a local union descriptor, do not process any new types.
                continue;
            }
            if (unionFields.containsKey(newDescriptor)) {
                if (!recordTypes.containsKey(newDescriptor.getName())) {
                    // Union field was seen before but the record type is unknown? This must not happen.
                    throw new MetaDataException("Unknown record type for union field " + unionField.getName());
                }
                // When pulling from a local file descriptor, the preferred field is the one that has the same
                // position as the field in the old union descriptor (so that the byte representation is the same).
                Descriptors.FieldDescriptor oldUnionField = oldUnionFields.get(oldDescriptor);
                if (unionField.getNumber() == oldUnionField.getNumber()) {
                    unionFields.put(newDescriptor, unionField);
                }
            } else {
                updateRecordType(oldRecordTypes, oldDescriptor, newDescriptor);
                unionFields.put(newDescriptor, unionField);
            }
        }
    }

    @Nullable
    private static Descriptors.Descriptor getCorrespondingFieldType(@Nonnull Descriptors.Descriptor descriptor, @Nonnull Descriptors.FieldDescriptor field) {
        Descriptors.FieldDescriptor correspondingField = descriptor.findFieldByNumber(field.getNumber());
        if (correspondingField != null) {
            return correspondingField.getMessageType();
        }
        return null;
    }

    @Nullable
    private Descriptors.Descriptor findOldDescriptor(@Nonnull Descriptors.FieldDescriptor newUnionField, @Nonnull Descriptors.Descriptor newUnion) {
        if (unionDescriptor == null) {
            throw new RecordCoreException("cannot get field from union as it has not been set");
        }
        // If there is a corresponding field in the old union, use that field.
        Descriptors.Descriptor correspondingFieldType = getCorrespondingFieldType(unionDescriptor, newUnionField);
        if (correspondingFieldType != null) {
            return correspondingFieldType;
        }
        // Look for a field in the new union of the same type as this one and look for a matching field in the old union
        final Descriptors.Descriptor newDescriptor = newUnionField.getMessageType();
        for (Descriptors.FieldDescriptor otherNewUnionField : newUnion.getFields()) {
            if (otherNewUnionField.getMessageType() == newDescriptor) {
                final Descriptors.Descriptor otherCorrespondingFieldType = getCorrespondingFieldType(unionDescriptor, otherNewUnionField);
                if (otherCorrespondingFieldType != null) {
                    return otherCorrespondingFieldType;
                }
            }
        }
        return null;
    }

    private void updateRecordType(@Nonnull Map<String, RecordTypeBuilder> oldRecordTypes,
                                  @Nonnull Descriptors.Descriptor oldDescriptor,
                                  @Nonnull Descriptors.Descriptor newDescriptor) {
        // Create a new record type based off the old one
        RecordTypeBuilder oldRecordType = oldRecordTypes.get(oldDescriptor.getName());
        RecordTypeBuilder newRecordType = new RecordTypeBuilder(newDescriptor, oldRecordType);
        recordTypes.put(newRecordType.getName(), newRecordType); // update the record type builder
    }

    private void fillUnionFields(boolean processExtensionOptions) {
        if (unionDescriptor == null) {
            throw new RecordCoreException("cannot fill union fiends as no union descriptor has been set");
        }
        if (!unionFields.isEmpty()) {
            throw new RecordCoreException("cannot set union fields twice");
        }
        for (Descriptors.FieldDescriptor unionField : unionDescriptor.getFields()) {
            Descriptors.Descriptor descriptor = unionField.getMessageType();
            if (!unionFields.containsKey(descriptor)) {
                processRecordType(unionField, processExtensionOptions);
                unionFields.put(descriptor, unionField);
            } else {
                // The preferred field is the last one, except if there is one whose name matches.
                remapUnionField(descriptor, unionField);
            }
        }
    }

    private void remapUnionField(@Nonnull Descriptors.Descriptor descriptor, @Nonnull Descriptors.FieldDescriptor unionField) {
        unionFields.compute(descriptor, (d, f) -> {
            // Prefer the field that has the name in the right format or, if neither do, the one with the larger field number.
            final String canonicalName = "_" + d.getName();
            if (f == null || unionField.getName().equals(canonicalName) || !f.getName().equals(canonicalName) && unionField.getNumber() > f.getNumber()) {
                return unionField;
            } else {
                return f;
            }
        });
    }

    @Nonnull
    private RecordTypeBuilder processRecordType(@Nonnull Descriptors.FieldDescriptor unionField, boolean processExtensionOptions) {
        Descriptors.Descriptor descriptor = unionField.getMessageType();
        RecordTypeBuilder recordType = new RecordTypeBuilder(descriptor);
        if (recordTypes.putIfAbsent(recordType.getName(), recordType) != null) {
            throw new MetaDataException("There is already a record type named " + recordType.getName());
        }
        if (processExtensionOptions) {
            RecordMetaDataOptionsProto.RecordTypeOptions recordTypeOptions = descriptor.getOptions()
                    .getExtension(RecordMetaDataOptionsProto.record);
            if (recordTypeOptions != null && recordTypeOptions.hasSinceVersion()) {
                recordType.setSinceVersion(recordTypeOptions.getSinceVersion());
            }
            if (recordTypeOptions != null && recordTypeOptions.hasRecordTypeKey()) {
                recordType.setRecordTypeKey(LiteralKeyExpression.fromProto(recordTypeOptions.getRecordTypeKey()).getValue());
            }
            protoFieldOptions(recordType);
        }
        return recordType;
    }

    private void protoFieldOptions(RecordTypeBuilder recordType) {
        // Add indexes from custom options.
        for (Descriptors.FieldDescriptor fieldDescriptor : recordType.getDescriptor().getFields()) {
            RecordMetaDataOptionsProto.FieldOptions fieldOptions = fieldDescriptor.getOptions()
                    .getExtension(RecordMetaDataOptionsProto.field);
            if (fieldOptions != null) {
                protoFieldOptions(recordType, fieldDescriptor, fieldOptions);
            }
        }
    }

    @SuppressWarnings("deprecation")
    private void protoFieldOptions(RecordTypeBuilder recordType, Descriptors.FieldDescriptor fieldDescriptor, RecordMetaDataOptionsProto.FieldOptions fieldOptions) {
        Descriptors.Descriptor descriptor = recordType.getDescriptor();
        if (fieldOptions.hasIndex() || fieldOptions.hasIndexed()) {
            String type;
            Map<String, String> options;
            if (fieldOptions.hasIndex()) {
                RecordMetaDataOptionsProto.FieldOptions.IndexOption indexOption = fieldOptions.getIndex();
                type = indexOption.getType();
                options = Index.buildOptions(indexOption.getOptionsList(), indexOption.getUnique());
            } else {
                type = Index.indexTypeToType(fieldOptions.getIndexed());
                options = Index.indexTypeToOptions(fieldOptions.getIndexed());
            }
            final FieldKeyExpression field = Key.Expressions.fromDescriptor(fieldDescriptor);
            final KeyExpression expr;
            if (type.equals(IndexTypes.RANK)) {
                expr = field.ungrouped();
            } else {
                expr = field;
            }
            final Index index = new Index(descriptor.getName() + "$" + fieldDescriptor.getName(),
                    expr, Index.EMPTY_VALUE, type, options);
            addIndex(recordType, index);
        } else if (fieldOptions.getPrimaryKey()) {
            if (recordType.getPrimaryKey() != null) {
                throw new MetaDataException("Only one primary key per record type is allowed have: " +
                                            recordType.getPrimaryKey() + "; adding on " + fieldDescriptor.getName());
            } else {
                if (fieldDescriptor.isRepeated()) {
                    // TODO maybe default to concatenate for this.
                    throw new MetaDataException("Primary key cannot be set on a repeated field");
                } else {
                    recordType.setPrimaryKey(Key.Expressions.fromDescriptor(fieldDescriptor));
                }
            }
        }
    }

    private static Descriptors.FileDescriptor buildFileDescriptor(@Nonnull DescriptorProtos.FileDescriptorProto fileDescriptorProto,
                                                                  @Nonnull Descriptors.FileDescriptor[] dependencies) {
        try {
            return Descriptors.FileDescriptor.buildFrom(fileDescriptorProto, dependencies);
        } catch (Descriptors.DescriptorValidationException ex) {
            throw new MetaDataException("Error converting from protobuf", ex);
        }
    }

    @Nonnull
    public Descriptors.Descriptor getUnionDescriptor() {
        return unionDescriptor;
    }

    @Nonnull
    public Descriptors.FieldDescriptor getUnionFieldForRecordType(@Nonnull RecordType recordType) {
        final Descriptors.FieldDescriptor unionField = unionFields.get(recordType.getDescriptor());
        if (unionField == null) {
            throw new MetaDataException("Record type " + recordType.getName() + " is not in the union");
        }
        return unionField;
    }

    @Nonnull
    public RecordTypeBuilder getRecordType(@Nonnull String name) {
        RecordTypeBuilder recordType = recordTypes.get(name);
        if (recordType == null) {
            throw new MetaDataException("Unknown record type " + name);
        }
        return recordType;
    }

    @Nonnull
    @API(API.Status.EXPERIMENTAL)
    @SuppressWarnings("squid:S1452")
    public SyntheticRecordTypeBuilder<?> getSyntheticRecordType(@Nonnull String name) {
        SyntheticRecordTypeBuilder<?> recordType = syntheticRecordTypes.get(name);
        if (recordType == null) {
            throw new MetaDataException("Unknown synthetic record type " + name);
        }
        return recordType;
    }

    /**
     * Get the next record type key for a synthetic record type.
     *
     * These keys are negative, unlike stored record types which are initially positive.
     * This isn't strictly speaking necessary, but simplifies debugging.
     * @return a new unique record type key
     */
    @Nonnull
    private Long getNextRecordTypeKey() {
        long minKey = 0;
        for (SyntheticRecordTypeBuilder<?> syntheticRecordType : syntheticRecordTypes.values()) {
            if (syntheticRecordType.getRecordTypeKey() instanceof Number) {
                long key = ((Number)syntheticRecordType.getRecordTypeKey()).longValue();
                if (minKey > key) {
                    minKey = key;
                }
            }
        }
        return minKey - 1;
    }

    /**
     * Add a new joined record type.
     * @param name the name of the new record type
     * @return a new uninitialized joined record type
     */
    @Nonnull
    @API(API.Status.EXPERIMENTAL)
    public JoinedRecordTypeBuilder addJoinedRecordType(@Nonnull String name) {
        if (recordTypes.containsKey(name)) {
            throw new MetaDataException("There is already a record type named " + name);
        }
        if (syntheticRecordTypes.containsKey(name)) {
            throw new MetaDataException("There is already a synthetic record type named " + name);
        }
        JoinedRecordTypeBuilder recordType = new JoinedRecordTypeBuilder(name, getNextRecordTypeKey(), this);
        syntheticRecordTypes.put(name, recordType);
        return recordType;
    }

    /**
     * Get a record type or synthetic record type by name for use with {@link #addIndex}.
     * @param name the name of the record type
     * @return the possibly synthetic record type
     */
    public RecordTypeIndexesBuilder getIndexableRecordType(@Nonnull String name) {
        RecordTypeIndexesBuilder recordType = recordTypes.get(name);
        if (recordType == null) {
            recordType = syntheticRecordTypes.get(name);
        }
        if (recordType == null) {
            throw new MetaDataException("Unknown record type " + name);
        }
        return recordType;
    }

    @Nonnull
    public Index getIndex(@Nonnull String indexName) {
        Index index = indexes.get(indexName);
        if (null == index) {
            throw new MetaDataException("Index " + indexName + " not defined");
        }
        return index;
    }

    // Common code shared by all the methods that add indexes. It runs some validation
    // and bumps the version if necessary.
    private void addIndexCommon(@Nonnull Index index) {
        if (recordsDescriptor == null) {
            throw new MetaDataException("No records added yet");
        }
        if (indexes.containsKey(index.getName())) {
            throw new MetaDataException("Index " + index.getName() + " already defined");
        }
        if (index.getLastModifiedVersion() <= 0) {
            index.setLastModifiedVersion(++version);
        } else if (index.getLastModifiedVersion() > version) {
            version = index.getLastModifiedVersion();
        }
        if (index.getAddedVersion() <= 0) {
            index.setAddedVersion(index.getLastModifiedVersion());
        }
        if (usesSubspaceKeyCounter && !index.hasExplicitSubspaceKey()) {
            index.setSubspaceKey(++subspaceKeyCounter);
        }
        indexes.put(index.getName(), index);
    }

    /**
     * Adds a new index. This index can either be a universal index or an index for
     * a single record type.
     * @param recordType if null this index will exist for all record types
     * @param index the index to be added
     */
    public void addIndex(@Nullable RecordTypeIndexesBuilder recordType, @Nonnull Index index) {
        addIndexCommon(index);
        if (recordType != null) {
            recordType.getIndexes().add(index);
        } else {
            universalIndexes.put(index.getName(), index);
        }
    }

    /**
     * Adds a new index.
     * @param recordType name of the record type
     * @param index the index to be added
     */
    public void addIndex(@Nonnull String recordType, @Nonnull Index index) {
        addIndex(getIndexableRecordType(recordType), index);
    }

    /**
     * Adds a new index.
     * @param recordType name of the record type
     * @param indexName the name of the new index
     * @param indexExpression the root expression of the new index
     */
    public void addIndex(@Nonnull String recordType, @Nonnull String indexName, @Nonnull KeyExpression indexExpression) {
        addIndex(recordType, new Index(indexName, indexExpression));
    }

    /**
     * Adds a new index.
     * @param recordType name of the record type
     * @param indexName the name of the new index
     * @param fieldName the record field to be indexed
     */
    public void addIndex(@Nonnull String recordType, @Nonnull String indexName, @Nonnull String fieldName) {
        addIndex(recordType, new Index(indexName, fieldName));
    }

    /**
     * Adds a new index on a single field.
     * @param recordType name of the record type
     * @param fieldName the record field to be indexed
     */
    public void addIndex(@Nonnull String recordType, @Nonnull String fieldName) {
        addIndex(recordType, recordType + "$" + fieldName, fieldName);
    }

    /**
     * Adds a new index that contains multiple record types.
     * If the list is null or empty, the resulting index will include all record types.
     * If the list has one element it will just be a normal single record type index.
     * @param recordTypes a list of record types that the index will include
     * @param index the index to be added
     */
    public void addMultiTypeIndex(@Nullable List<? extends RecordTypeIndexesBuilder> recordTypes, @Nonnull Index index) {
        addIndexCommon(index);
        if (recordTypes == null || recordTypes.isEmpty()) {
            universalIndexes.put(index.getName(), index);
        } else if (recordTypes.size() == 1) {
            recordTypes.get(0).getIndexes().add(index);
        } else {
            for (RecordTypeIndexesBuilder recordType : recordTypes) {
                recordType.getMultiTypeIndexes().add(index);
            }
        }
    }

    /**
     * Adds a new index on all record types.
     * @param index the index to be added
     */
    public void addUniversalIndex(@Nonnull Index index) {
        addIndexCommon(index);
        universalIndexes.put(index.getName(), index);
    }

    public void removeIndex(@Nonnull String name) {
        Index index = indexes.remove(name);
        if (index == null) {
            throw new MetaDataException("No index named " + name + " defined");
        }
        for (RecordTypeBuilder recordType : recordTypes.values()) {
            recordType.getIndexes().remove(index);
            recordType.getMultiTypeIndexes().remove(index);
        }
        universalIndexes.remove(name);
        formerIndexes.add(new FormerIndex(index.getSubspaceKey(), index.getAddedVersion(), ++version, name));
    }

    public void addFormerIndex(@Nonnull FormerIndex formerIndex) {
        formerIndexes.add(formerIndex);
    }

    public boolean isSplitLongRecords() {
        return splitLongRecords;
    }

    public void setSplitLongRecords(boolean splitLongRecords) {
        if (recordsDescriptor == null) {
            throw new MetaDataException("No records added yet");
        }
        if (this.splitLongRecords != splitLongRecords) {
            version += 1;
            this.splitLongRecords = splitLongRecords;
        }
    }

    public boolean isStoreRecordVersions() {
        return storeRecordVersions;
    }

    public void setStoreRecordVersions(boolean storeRecordVersions) {
        if (recordsDescriptor == null) {
            throw new MetaDataException("No records added yet");
        }
        if (this.storeRecordVersions != storeRecordVersions) {
            version += 1;
            this.storeRecordVersions = storeRecordVersions;
        }
    }

    /**
     * Get the record count key, if any.
     * @return the record count key of {@code null}
     * @deprecated use {@code COUNT} type indexes instead
     */
    @Nullable
    @Deprecated
    @API(API.Status.DEPRECATED)
    public KeyExpression getRecordCountKey() {
        return recordCountKey;
    }

    /**
     * Set the key used for maintaining record counts.
     * @param recordCountKey grouping key for counting
     * @deprecated use {@code COUNT} type indexes instead
     */
    @Deprecated
    @API(API.Status.DEPRECATED)
    public void setRecordCountKey(KeyExpression recordCountKey) {
        if (recordsDescriptor == null) {
            throw new MetaDataException("No records added yet");
        }
        if (!Objects.equals(this.recordCountKey, recordCountKey)) {
            version += 1;
            this.recordCountKey = recordCountKey;
        }
    }

    public int getVersion() {
        return version;
    }

    public void setVersion(int version) {
        if (recordsDescriptor == null) {
            throw new MetaDataException("No records added yet");
        }
        this.version = version;
    }

    /**
     * Enable counter-based subspace keys assignment.
     *
     * <p>
     * If enabled, index subspace keys will be set using a counter instead of defaulting to the indexes' names. This
     * must be called prior to setting the records descriptor (for example {@link #setRecords(Descriptors.FileDescriptor)}).
     * </p>
     *
     * <p>
     * Existing clients should be careful about enabling this feature. The name of an index is the default
     * value of its subspace key when counter-based subspace keys are disabled. If a subspace key was not set
     * explicitly before, enabling the counter-based scheme will change the index's subspace key. Note that
     * it is important that the subspace key of an index that has data does not change.
     * See {@link Index#setSubspaceKey(Object)} for more details.
     * </p>
     *
     * @return this builder
     */
    @Nonnull
    public RecordMetaDataBuilder enableCounterBasedSubspaceKeys() {
        if (recordsDescriptor != null) {
            throw new MetaDataException("Records descriptor has already been set.");
        }
        this.usesSubspaceKeyCounter = true;
        return this;
    }

    /**
     * Checks if counter-based subspace key assignment is used.
     * @return {@code true} if the subspace key counter is used
     */
    public boolean usesSubspaceKeyCounter() {
        return usesSubspaceKeyCounter;
    }

    /**
     * Get the current value of the index subspace key counter. If it is not enabled, the value will be 0.
     * @return the current value of the index subspace key counter
     * @see #enableCounterBasedSubspaceKeys()
     */
    public long getSubspaceKeyCounter() {
        return subspaceKeyCounter;
    }

    /**
     * Set the initial value of the subspace key counter. This method can be handy when users want to assign
     * subspace keys using a counter, but their indexes already have subspace keys that may conflict with the
     * counter-based assignment.
     *
     * <p>
     * Note that the new counter must be greater than the current value. Also, users must first enable this feature by
     * calling {@link #enableCounterBasedSubspaceKeys()} before updating the counter value.
     * </p>
     *
     * @param subspaceKeyCounter the new value
     * @return this builder
     * @see #enableCounterBasedSubspaceKeys()
     */
    @Nonnull
    public RecordMetaDataBuilder setSubspaceKeyCounter(long subspaceKeyCounter) {
        if (!usesSubspaceKeyCounter()) {
            throw new MetaDataException("Counter-based subspace keys not enabled");
        }
        if (subspaceKeyCounter <= this.subspaceKeyCounter) {
            throw new MetaDataException(String.format("Subspace key counter must be set to a value greater than its current value (%d)",
                    this.subspaceKeyCounter));
        }
        this.subspaceKeyCounter = subspaceKeyCounter;
        return this;
    }

    /**
     * If there is only one record type, get it.
     * @return the only type defined for this store.
     */
    @Nonnull
    public RecordTypeBuilder getOnlyRecordType() {
        if (recordTypes.size() != 1) {
            throw new MetaDataException("Must have exactly one record type defined.");
        }
        return recordTypes.values().iterator().next();
    }

    /**
     * Get the index registry used for validation.
     * @return the index maintainer registry
     */
    @Nonnull
    public IndexMaintainerRegistry getIndexMaintainerRegistry() {
        return indexMaintainerRegistry;
    }

    /**
     * Set the index registry used for validation.
     *
     * If a record store has a custom index maintainer registry, that same registry may need to be used to properly
     * validate the meta-data.
     * @param indexMaintainerRegistry the index maintainer registry
     * @see com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase.BaseBuilder#setIndexMaintainerRegistry
     */
    public void setIndexMaintainerRegistry(@Nonnull IndexMaintainerRegistry indexMaintainerRegistry) {
        this.indexMaintainerRegistry = indexMaintainerRegistry;
    }

    /**
     * Get the validator used to compare the local file descriptor to the descriptor included
     * in the meta-data proto. By default, this instance is set to the {@link MetaDataEvolutionValidator}'s
     * {@linkplain MetaDataEvolutionValidator#getDefaultInstance() default instance}, but the
     * user may provide their own through {@link #setEvolutionValidator(MetaDataEvolutionValidator)}
     * if they want to tweak certain validator options.
     *
     * @return the validator used to check the local file descriptor against the one in the meta-data proto
     * @see #setLocalFileDescriptor(Descriptors.FileDescriptor)
     * @see MetaDataEvolutionValidator
     */
    @Nonnull
    public MetaDataEvolutionValidator getEvolutionValidator() {
        return evolutionValidator;
    }

    /**
     * Set the validator used to compare the local file descriptor to the descriptor included
     * in the meta-data proto. As this validator is used only to check whether the local file descriptor
     * is compatible with the records descriptor set in {@link #setRecords(RecordMetaDataProto.MetaData) setRecords()}
     * through the meta-data proto, this method must be called before {@code setRecords()}.
     *
     * @param evolutionValidator the validator used to check the local file descriptor against the one in the meta-data proto
     * @return this builder
     * @see #setLocalFileDescriptor(Descriptors.FileDescriptor)
     * @see MetaDataEvolutionValidator
     */
    @Nonnull
    public RecordMetaDataBuilder setEvolutionValidator(@Nonnull MetaDataEvolutionValidator evolutionValidator) {
        if (recordsDescriptor != null) {
            throw new MetaDataException("Records already set.");
        }
        this.evolutionValidator = evolutionValidator;
        return this;
    }

    @Nonnull
    @Override
    public RecordMetaData getRecordMetaData() {
        if (recordMetaData == null || recordMetaData.getVersion() != version) {
            recordMetaData = build();
        }
        return recordMetaData;
    }

    /**
     * Build and validate meta-data.
     * @return new validated meta-data
     */
    @Nonnull
    public RecordMetaData build() {
        return build(true);
    }

    /**
     * Build and validate meta-data with specific index registry.
     * @param validate {@code true} to validate the new meta-data
     * @return new meta-data
     */
    @Nonnull
    public RecordMetaData build(boolean validate) {
        Map<String, RecordType> builtRecordTypes = Maps.newHashMapWithExpectedSize(recordTypes.size());
        Map<String, SyntheticRecordType<?>> builtSyntheticRecordTypes = Maps.newHashMapWithExpectedSize(syntheticRecordTypes.size());
        RecordMetaData metaData = new RecordMetaData(recordsDescriptor, getUnionDescriptor(), unionFields,
                builtRecordTypes, builtSyntheticRecordTypes,
                indexes, universalIndexes, formerIndexes,
                splitLongRecords, storeRecordVersions, version, subspaceKeyCounter, usesSubspaceKeyCounter, recordCountKey, localFileDescriptor != null);
        for (RecordTypeBuilder recordTypeBuilder : recordTypes.values()) {
            KeyExpression primaryKey = recordTypeBuilder.getPrimaryKey();
            if (primaryKey != null) {
                builtRecordTypes.put(recordTypeBuilder.getName(), recordTypeBuilder.build(metaData));
                for (Index index : recordTypeBuilder.getIndexes()) {
                    index.setPrimaryKeyComponentPositions(buildPrimaryKeyComponentPositions(index.getRootExpression(), primaryKey));
                }
            } else {
                throw new MetaDataException("Record type " +
                                            recordTypeBuilder.getName() +
                                            " must have a primary key");
            }
        }
        if (!syntheticRecordTypes.isEmpty()) {
            DescriptorProtos.FileDescriptorProto.Builder fileBuilder = DescriptorProtos.FileDescriptorProto.newBuilder();
            fileBuilder.setName("_synthetic");
            fileBuilder.addDependency(unionDescriptor.getFile().getName());
            syntheticRecordTypes.values().forEach(recordTypeBuilder -> recordTypeBuilder.buildDescriptor(fileBuilder));
            final Descriptors.FileDescriptor fileDescriptor;
            try {
                final Descriptors.FileDescriptor[] dependencies = new Descriptors.FileDescriptor[] { unionDescriptor.getFile() };
                fileDescriptor = Descriptors.FileDescriptor.buildFrom(fileBuilder.build(), dependencies);
            } catch (Descriptors.DescriptorValidationException ex) {
                throw new MetaDataException("Could not build synthesized file descriptor", ex);
            }
            for (SyntheticRecordTypeBuilder<?> recordTypeBuilder : syntheticRecordTypes.values()) {
                builtSyntheticRecordTypes.put(recordTypeBuilder.getName(), recordTypeBuilder.build(metaData, fileDescriptor));
            }
        }
        if (validate) {
            final MetaDataValidator validator = new MetaDataValidator(metaData, indexMaintainerRegistry);
            validator.validate();
        }
        return metaData;
    }

    // Note that there is no harm in this returning null for very complex overlaps; that just results in some duplication.
    @Nullable
    public static int[] buildPrimaryKeyComponentPositions(@Nonnull KeyExpression indexKey, @Nonnull KeyExpression primaryKey) {
        List<KeyExpression> indexKeys = indexKey.normalizeKeyForPositions();
        List<KeyExpression> primaryKeys = primaryKey.normalizeKeyForPositions();
        int[] positions = new int[primaryKeys.size()];
        for (int i = 0; i < positions.length; i++) {
            positions[i] = indexKeys.indexOf(primaryKeys.get(i));
        }
        if (Arrays.stream(positions).anyMatch(p -> p >= 0)) {
            return positions;
        } else {
            return null;
        }
    }

    /**
     * Exception thrown when meta-data cannot be loaded from serialized form.
     */
    @SuppressWarnings("serial")
    public static class MetaDataProtoDeserializationException extends MetaDataException {
        public MetaDataProtoDeserializationException(@Nullable Throwable cause) {
            super("Error converting from protobuf", cause);
        }
    }
}