/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.phoenix.filter;

import static com.google.common.base.Preconditions.checkArgument;
import static org.apache.phoenix.schema.PTable.QualifierEncodingScheme.NON_ENCODED_QUALIFIERS;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.EOFException;
import java.io.IOException;
import java.util.BitSet;
import java.util.NoSuchElementException;
import java.util.TreeSet;

import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.util.Writables;
import org.apache.hadoop.io.WritableUtils;
import org.apache.phoenix.expression.Expression;
import org.apache.phoenix.expression.KeyValueColumnExpression;
import org.apache.phoenix.expression.visitor.ExpressionVisitor;
import org.apache.phoenix.expression.visitor.StatelessTraverseAllExpressionVisitor;
import org.apache.phoenix.schema.PTable.QualifierEncodingScheme;
import org.apache.phoenix.schema.tuple.BaseTuple;
import org.apache.phoenix.util.ByteUtil;
import org.apache.phoenix.util.ServerUtil;

/**
 * Filter used for tables that use number based column qualifiers generated by one of the encoding schemes in
 * {@link QualifierEncodingScheme}. Because the qualifiers are number based, instead of using a map of cells to track
 * the columns that have been found, we can use an array of cells where the index into the array would be derived by the
 * number based column qualifier. See {@link EncodedCQIncrementalResultTuple}. Using this filter helps us to directly
 * seek to the next row when the column qualifier that we have encountered is greater than the maxQualifier that we
 * expect. This helps in speeding up the queries filtering on key value columns.
 * 
 * TODO: derived this from MultiKeyValueComparisonFilter to reduce the copy/paste from that class.
 */
public class MultiEncodedCQKeyValueComparisonFilter extends BooleanExpressionFilter {
    // Smallest qualifier for the columns that are being projected and filtered on
    private int minQualifier;

    // Largest qualifier for the columns that are being projected and filtered on
    private int maxQualifier;

    private QualifierEncodingScheme encodingScheme;

    // Smallest qualifier for the columns in where expression
    private int whereExpressionMinQualifier;

    // Largest qualifier for the columns in where expression
    private int whereExpressionMaxQualifier;

    private FilteredKeyValueHolder filteredKeyValues;
    
    // BitSet to track the qualifiers in where expression that we expect to find while filtering a row
    private BitSet whereExpressionQualifiers;

    // Set to track the column families of the columns in where expression
    private TreeSet<byte[]> cfSet;

    // Boolean that tells us whether the result of expression evaluation as and when we filter key values in a row
    private Boolean matchedColumn;

    // Tuple used to store the relevant key values found while filtering a row
    private EncodedCQIncrementalResultTuple inputTuple = new EncodedCQIncrementalResultTuple();
    
    // Member variable to cache the size of whereExpressionQualifiers
    private int expectedCardinality;
    
    private byte[] essentialCF = ByteUtil.EMPTY_BYTE_ARRAY;
    private boolean allCFs;

    private static final byte[] UNITIALIZED_KEY_BUFFER = new byte[0];
    
    public MultiEncodedCQKeyValueComparisonFilter() {}

    public MultiEncodedCQKeyValueComparisonFilter(Expression expression, QualifierEncodingScheme scheme, boolean allCFs, byte[] essentialCF) {
        super(expression);
        checkArgument(scheme != NON_ENCODED_QUALIFIERS, "Filter can only be used for encoded qualifiers");
        this.encodingScheme = scheme;
        this.allCFs = allCFs;
        this.essentialCF = essentialCF == null ? ByteUtil.EMPTY_BYTE_ARRAY : essentialCF;
        initFilter(expression);
    }
    
    private final class FilteredKeyValueHolder {
        // Cell values corresponding to columns in where expression that were found while filtering a row.
        private Cell[] filteredCells;

        // BitSet to track whether qualifiers in where expression were found when filtering a row
        private BitSet filteredQualifiers;
        
        // Using an explicit counter instead of relying on the cardinality of the bitset as computing the 
        // cardinality could be slightly more expensive than just incrementing an integer
        private int numKeyValues;
        
        private FilteredKeyValueHolder(int size) {
            filteredCells = new Cell[size];
            filteredQualifiers = new BitSet(size);
        }

        private void setCell(int qualifier, Cell c) {
            int index = qualifier - whereExpressionMinQualifier;
            filteredCells[index] = c;
            filteredQualifiers.set(index);
            numKeyValues++;
        }

        private Cell getCell(int qualifier) {
            int index = qualifier - whereExpressionMinQualifier;
            return filteredQualifiers.get(index) ? filteredCells[index] : null;
        }

        private void clear() {
            // Note here that we are only clearing out the filteredQualifiers bitset. We are not setting all the
            // entries in filteredKeyValues to null or allocating a new Cell array as that would be expensive.
            filteredQualifiers.clear();
            numKeyValues = 0;
        }
        
        /**
         * This method really shouldn't be the way for getting hold of cells. It was
         * just added to keep the tuple.get(index) method happy.
         */
        public Cell getCellAtIndex(int index) {
            int bitIndex;
            for (bitIndex = filteredQualifiers.nextSetBit(0); bitIndex >= 0 && index >= 0; bitIndex = filteredQualifiers
                    .nextSetBit(bitIndex + 1)) {
                index--;
            }
            if (bitIndex < 0) { throw new NoSuchElementException(); }
            return filteredCells[bitIndex];
        }

        @Override
        public String toString() {
            StringBuilder sb = new StringBuilder(100);
            int length = filteredQualifiers.length();
            for (int i = 0; i < length; i++) {
                sb.append(filteredCells[i].toString());
            }
            return sb.toString();
        }
        
        private boolean allColumnsFound() {
            return numKeyValues == expectedCardinality;
        }
        
        private int numKeyValues() {
            return numKeyValues;
        }

    }
    
    private void initFilter(Expression expression) {
        cfSet = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
        final BitSet expressionQualifiers = new BitSet(20);
        final Pair<Integer, Integer> range = new Pair<>();
        ExpressionVisitor<Void> visitor = new StatelessTraverseAllExpressionVisitor<Void>() {
            @Override
            public Void visit(KeyValueColumnExpression expression) {
                int qualifier = encodingScheme.decode(expression.getColumnQualifier());
                if (range.getFirst() == null) {
                    range.setFirst(qualifier);
                    range.setSecond(qualifier);
                } else if (qualifier < range.getFirst()) {
                    range.setFirst(qualifier);
                } else if (qualifier > range.getSecond()) {
                    range.setSecond(qualifier);
                }
                cfSet.add(expression.getColumnFamily());
                expressionQualifiers.set(qualifier);
                return null;
            }
        };
        expression.accept(visitor);
        // Set min and max qualifiers for columns in the where expression
        whereExpressionMinQualifier = range.getFirst();
        whereExpressionMaxQualifier = range.getSecond();
        
        int size = whereExpressionMaxQualifier - whereExpressionMinQualifier + 1;
        filteredKeyValues = new FilteredKeyValueHolder(size);
        
        // Initialize the bitset and mark the qualifiers for columns in where expression
        whereExpressionQualifiers = new BitSet(size);
        for (int i = whereExpressionMinQualifier; i <= whereExpressionMaxQualifier; i++) {
            if (expressionQualifiers.get(i)) {
                whereExpressionQualifiers.set(i - whereExpressionMinQualifier);
            }
        }
        expectedCardinality = whereExpressionQualifiers.cardinality();
    }
    
    private boolean isQualifierForColumnInWhereExpression(int qualifier) {
        return qualifier >= whereExpressionMinQualifier ? whereExpressionQualifiers.get(qualifier - whereExpressionMinQualifier) : false;
    }
    
    @Override
    public ReturnCode filterKeyValue(Cell cell) {
        if (Boolean.TRUE.equals(this.matchedColumn)) {
          // We already found and matched the single column, all keys now pass
          return ReturnCode.INCLUDE_AND_NEXT_COL;
        }
        if (Boolean.FALSE.equals(this.matchedColumn)) {
          // We found all the columns, but did not match the expression, so skip to next row
          return ReturnCode.NEXT_ROW;
        }
        inputTuple.setKey(cell);
        int qualifier = encodingScheme.decode(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength());
        if (isQualifierForColumnInWhereExpression(qualifier)) {
            filteredKeyValues.setCell(qualifier, cell);
            // We found a new column, so we can re-evaluate
            this.matchedColumn = this.evaluate(inputTuple);
            if (this.matchedColumn == null) {
                if (inputTuple.isImmutable()) {
                    this.matchedColumn = Boolean.FALSE;
                } else {
                    return ReturnCode.INCLUDE_AND_NEXT_COL;
                }
            }
            return this.matchedColumn ? ReturnCode.INCLUDE_AND_NEXT_COL : ReturnCode.NEXT_ROW;
        }
        // The qualifier is not one of the qualifiers in the expression. So decide whether
        // we would need to include it in our result.
        if (qualifier < minQualifier) {
            // Qualifier is smaller than the minimum expected qualifier. Look at the next column.
            return ReturnCode.NEXT_COL;
        }
        // TODO: I don't think we would ever hit this case of encountering a greater than what we expect.
        // Leaving the code commented out here for future reference.
        // if (qualifier > maxQualifier) {
            // Qualifier is larger than the max expected qualifier. We are done looking at columns in this row.
            // return ReturnCode.NEXT_ROW;
        // }
        return ReturnCode.INCLUDE_AND_NEXT_COL;
    }

    @Override
    public boolean filterRow() {
        if (this.matchedColumn == null && !inputTuple.isImmutable() && expression.requiresFinalEvaluation()) {
            inputTuple.setImmutable();
            this.matchedColumn = this.evaluate(inputTuple);
        }
        return ! (Boolean.TRUE.equals(this.matchedColumn));
    }

    final class EncodedCQIncrementalResultTuple extends BaseTuple {
        private final ImmutableBytesWritable keyPtr = new ImmutableBytesWritable(UNITIALIZED_KEY_BUFFER);
        private boolean isImmutable;
        
        @Override
        public boolean isImmutable() {
            return isImmutable || filteredKeyValues.allColumnsFound();
        }
        
        public void setImmutable() {
            this.isImmutable = true;
        }
        
        private void setKey(Cell value) {
            keyPtr.set(value.getRowArray(), value.getRowOffset(), value.getRowLength());
        }
        
        @Override
        public void getKey(ImmutableBytesWritable ptr) {
            ptr.set(keyPtr.get(),keyPtr.getOffset(),keyPtr.getLength());
        }
        
        @Override
        public Cell getValue(byte[] cf, byte[] cq) {
            int qualifier = encodingScheme.decode(cq);
            return filteredKeyValues.getCell(qualifier);
        }
        
        @Override
        public String toString() {
            return filteredKeyValues.toString();
        }

        @Override
        public int size() {
            return filteredKeyValues.numKeyValues();
        }

        /**
         * This method doesn't perform well and shouldn't be the way of
         * getting hold of elements in the tuple.
         */
        @Override
        public Cell getValue(int index) {
            return filteredKeyValues.getCellAtIndex(index);
        }

        @Override
        public boolean getValue(byte[] family, byte[] qualifier,
                ImmutableBytesWritable ptr) {
            Cell cell = getValue(family, qualifier);
            if (cell == null)
                return false;
            ptr.set(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
            return true;
        }
        
        void reset() {
            isImmutable = false;
            keyPtr.set(UNITIALIZED_KEY_BUFFER);
        }
    }
    
    @Override
    public void readFields(DataInput input) throws IOException {
        try {
            this.minQualifier = WritableUtils.readVInt(input);
            this.maxQualifier = WritableUtils.readVInt(input);
            this.whereExpressionMinQualifier = WritableUtils.readVInt(input);
            this.whereExpressionMaxQualifier = WritableUtils.readVInt(input);
            this.encodingScheme = QualifierEncodingScheme.values()[WritableUtils.readVInt(input)];
            super.readFields(input);
            try {
                allCFs = input.readBoolean();
                if (!allCFs) {
                    essentialCF = Bytes.readByteArray(input);
                }
            } catch (EOFException e) { // Ignore as this will occur when a 4.10 client is used
            }
        } catch (DoNotRetryIOException e) {
            throw e;
        } catch (Throwable t) { // Catches incompatibilities during reading/writing and doesn't retry
            ServerUtil.throwIOException("MultiEncodedCQKeyValueComparisonFilter failed during writing", t);
        }
        initFilter(expression);
    }
    
    @Override
    public void write(DataOutput output) throws IOException {
        try {
            WritableUtils.writeVInt(output, minQualifier);
            WritableUtils.writeVInt(output, maxQualifier);
            WritableUtils.writeVInt(output, whereExpressionMinQualifier);
            WritableUtils.writeVInt(output, whereExpressionMaxQualifier);
            WritableUtils.writeVInt(output, encodingScheme.ordinal());
            super.write(output);
            output.writeBoolean(allCFs);
            if (!allCFs) {
                Bytes.writeByteArray(output, essentialCF);
            }
        } catch (DoNotRetryIOException e) {
            throw e;
        } catch (Throwable t) { // Catches incompatibilities during reading/writing and doesn't retry
            ServerUtil.throwIOException("MultiEncodedCQKeyValueComparisonFilter failed during writing", t);
        }
    }
    
    public void setMinMaxQualifierRange(Pair<Integer, Integer> minMaxQualifiers) {
        this.minQualifier = minMaxQualifiers.getFirst();
        this.maxQualifier = minMaxQualifiers.getSecond();
    }

    public void setMinQualifier(int minQualifier) {
        this.minQualifier = minQualifier;
    }

    public static MultiEncodedCQKeyValueComparisonFilter parseFrom(final byte [] pbBytes) throws DeserializationException {
        try {
            return (MultiEncodedCQKeyValueComparisonFilter)Writables.getWritable(pbBytes, new MultiEncodedCQKeyValueComparisonFilter());
        } catch (IOException e) {
            throw new DeserializationException(e);
        }
    }
    
    @Override
    public void reset() {
        filteredKeyValues.clear();
        matchedColumn = null;
        inputTuple.reset();
        super.reset();
    }

    @Override
    public boolean isFamilyEssential(byte[] name) {
        // Typically only the column families involved in the expression are essential.
        // The others are for columns projected in the select expression. However, depending
        // on the expression (i.e. IS NULL), we may need to include the column family
        // containing the empty key value or all column families in the case of a mapped
        // view (where we don't have an empty key value).
        return allCFs || Bytes.compareTo(name, essentialCF) == 0 || cfSet.contains(name);
    }
    
    
}