Java Code Examples for org.apache.hadoop.hbase.util.Pair#setFirst()

public FuzzyRowFilter(List<Pair<byte[], byte[]>> fuzzyKeysData) {
  List<Pair<byte[], byte[]>> fuzzyKeyDataCopy = new ArrayList<>(fuzzyKeysData.size());

  for (Pair<byte[], byte[]> aFuzzyKeysData : fuzzyKeysData) {
    if (aFuzzyKeysData.getFirst().length != aFuzzyKeysData.getSecond().length) {
      Pair<String, String> readable =
        new Pair<>(Bytes.toStringBinary(aFuzzyKeysData.getFirst()), Bytes.toStringBinary(aFuzzyKeysData.getSecond()));
      throw new IllegalArgumentException("Fuzzy pair lengths do not match: " + readable);
    }

    Pair<byte[], byte[]> p = new Pair<>();
    // create a copy of pair bytes so that they are not modified by the filter.
    p.setFirst(Arrays.copyOf(aFuzzyKeysData.getFirst(), aFuzzyKeysData.getFirst().length));
    p.setSecond(Arrays.copyOf(aFuzzyKeysData.getSecond(), aFuzzyKeysData.getSecond().length));

    // update mask ( 0 -> -1 (0xff), 1 -> 2)
    p.setSecond(preprocessMask(p.getSecond()));
    preprocessSearchKey(p);

    fuzzyKeyDataCopy.add(p);
  }
  this.fuzzyKeysData = fuzzyKeyDataCopy;
  this.tracker = new RowTracker();
}
 

private boolean getKeyExpressionCombinations(Pair<Expression, Expression> combination, StatementContext context, SelectStatement select, TableRef table, JoinType type, final List<Expression> joinExpressions, final List<Expression> hashExpressions) throws SQLException {
    if ((type != JoinType.Inner && type != JoinType.Semi) || this.noChildParentJoinOptimization)
        return false;
    
    Scan scanCopy = ScanUtil.newScan(context.getScan());
    StatementContext contextCopy = new StatementContext(statement, context.getResolver(), scanCopy, new SequenceManager(statement));
    contextCopy.setCurrentTable(table);
    List<Expression> lhsCombination = Lists.<Expression> newArrayList();
    boolean complete = WhereOptimizer.getKeyExpressionCombination(lhsCombination, contextCopy, select, joinExpressions);
    if (lhsCombination.isEmpty())
        return false;
    
    List<Expression> rhsCombination = Lists.newArrayListWithExpectedSize(lhsCombination.size());
    for (int i = 0; i < lhsCombination.size(); i++) {
        Expression lhs = lhsCombination.get(i);
        for (int j = 0; j < joinExpressions.size(); j++) {
            if (lhs == joinExpressions.get(j)) {
                rhsCombination.add(hashExpressions.get(j));
                break;
            }
        }
    }
    
    if (lhsCombination.size() == 1) {
        combination.setFirst(lhsCombination.get(0));
        combination.setSecond(rhsCombination.get(0));
    } else {
        combination.setFirst(new RowValueConstructorExpression(lhsCombination, false));
        combination.setSecond(new RowValueConstructorExpression(rhsCombination, false));
    }
    
    return type == JoinType.Semi && complete;
}
 

public int estimateByteSize(Object o, Pair<Integer, Integer> nullsVsNullRepeationCounter, PDataType baseType) {
    if (baseType.isFixedWidth()) { return baseType.getByteSize(); }
    if (baseType.isArrayType()) {
        PhoenixArray array = (PhoenixArray)o;
        int noOfElements = array.numElements;
        int totalVarSize = 0;
        int nullsRepeationCounter = 0;
        int nulls = 0;
        int totalNulls = 0;
        for (int i = 0; i < noOfElements; i++) {
            totalVarSize += array.estimateByteSize(i);
            if (!PDataType.fromTypeId((baseType.getSqlType() - PDataType.ARRAY_TYPE_BASE)).isFixedWidth()) {
                if (array.isNull(i)) {
                    nulls++;
                } else {
                    if (nulls > 0) {
                        totalNulls += nulls;
                        nulls = 0;
                        nullsRepeationCounter++;
                    }
                }
            }
        }
        if (nullsVsNullRepeationCounter != null) {
            if (nulls > 0) {
                totalNulls += nulls;
                // do not increment nullsRepeationCounter to identify trailing nulls
            }
            nullsVsNullRepeationCounter.setFirst(totalNulls);
            nullsVsNullRepeationCounter.setSecond(nullsRepeationCounter);
        }
        return totalVarSize;
    }
    // Non fixed width types must override this
    throw new UnsupportedOperationException();
}
 

public void addGuidePost(ImmutableBytesPtr cfKey, GuidePostsInfo info, long byteSize, long timestamp) {
    Pair<Long, GuidePostsInfo> newInfo = new Pair<Long, GuidePostsInfo>(byteSize, info);
    Pair<Long, GuidePostsInfo> oldInfo = guidePostsMap.put(cfKey, newInfo);
    if (oldInfo != null) {
        info.combine(oldInfo.getSecond());
        newInfo.setFirst(oldInfo.getFirst() + newInfo.getFirst());
    }
    maxTimeStamp = Math.max(maxTimeStamp, timestamp);
}
 

/**
 * Checks whether cell contains any tag with type as VISIBILITY_TAG_TYPE. This
 * tag type is reserved and should not be explicitly set by user.
 *
 * @param cell The cell under consideration
 * @param pair An optional pair of type {@code <Boolean, Tag>} which would be reused if already
 *     set and new one will be created if NULL is passed
 * @return If the boolean is false then it indicates that the cell has a RESERVERD_VIS_TAG and
 *     with boolean as true, not null tag indicates that a string modified tag was found.
 */
private Pair<Boolean, Tag> checkForReservedVisibilityTagPresence(Cell cell,
    Pair<Boolean, Tag> pair) throws IOException {
  if (pair == null) {
    pair = new Pair<>(false, null);
  } else {
    pair.setFirst(false);
    pair.setSecond(null);
  }
  // Bypass this check when the operation is done by a system/super user.
  // This is done because, while Replication, the Cells coming to the peer cluster with reserved
  // typed tags and this is fine and should get added to the peer cluster table
  if (isSystemOrSuperUser()) {
    // Does the cell contain special tag which indicates that the replicated
    // cell visiblilty tags
    // have been modified
    Tag modifiedTag = null;
    Iterator<Tag> tagsIterator = PrivateCellUtil.tagsIterator(cell);
    while (tagsIterator.hasNext()) {
      Tag tag = tagsIterator.next();
      if (tag.getType() == TagType.STRING_VIS_TAG_TYPE) {
        modifiedTag = tag;
        break;
      }
    }
    pair.setFirst(true);
    pair.setSecond(modifiedTag);
    return pair;
  }
  Iterator<Tag> tagsItr = PrivateCellUtil.tagsIterator(cell);
  while (tagsItr.hasNext()) {
    if (RESERVED_VIS_TAG_TYPES.contains(tagsItr.next().getType())) {
      return pair;
    }
  }
  pair.setFirst(true);
  return pair;
}
 

private boolean getKeyExpressionCombinations(Pair<Expression, Expression> combination, StatementContext context, SelectStatement select, TableRef table, JoinType type, final List<Expression> joinExpressions, final List<Expression> hashExpressions) throws SQLException {
    if ((type != JoinType.Inner && type != JoinType.Semi) || this.noChildParentJoinOptimization)
        return false;

    Scan scanCopy = ScanUtil.newScan(context.getScan());
    StatementContext contextCopy = new StatementContext(statement, context.getResolver(), context.getBindManager(), scanCopy, new SequenceManager(statement));
    contextCopy.setCurrentTable(table);
    List<Expression> lhsCombination = Lists.<Expression> newArrayList();
    boolean complete = WhereOptimizer.getKeyExpressionCombination(lhsCombination, contextCopy, select, joinExpressions);
    if (lhsCombination.isEmpty())
        return false;

    List<Expression> rhsCombination = Lists.newArrayListWithExpectedSize(lhsCombination.size());
    for (int i = 0; i < lhsCombination.size(); i++) {
        Expression lhs = lhsCombination.get(i);
        for (int j = 0; j < joinExpressions.size(); j++) {
            if (lhs == joinExpressions.get(j)) {
                rhsCombination.add(hashExpressions.get(j));
                break;
            }
        }
    }

    if (lhsCombination.size() == 1) {
        combination.setFirst(lhsCombination.get(0));
        combination.setSecond(rhsCombination.get(0));
    } else {
        combination.setFirst(new RowValueConstructorExpression(lhsCombination, false));
        combination.setSecond(new RowValueConstructorExpression(rhsCombination, false));
    }

    return type == JoinType.Semi && complete;
}
 

private static void adjustQualifierRange(Integer qualifier, Pair<Integer, Integer> minMaxQualifiers) {
    if (minMaxQualifiers.getFirst() == null) {
        minMaxQualifiers.setFirst(qualifier);
        minMaxQualifiers.setSecond(qualifier);
    } else {
        if (minMaxQualifiers.getFirst() > qualifier) {
            minMaxQualifiers.setFirst(qualifier);
        } else if (minMaxQualifiers.getSecond() < qualifier) {
            minMaxQualifiers.setSecond(qualifier);
        }
    }
}
 

public int estimateByteSize(Object o, Pair<Integer, Integer> nullsVsNullRepeationCounter, PDataType baseType) {
    if (baseType.isFixedWidth()) { return baseType.getByteSize(); }
    if (baseType.isArrayType()) {
        PhoenixArray array = (PhoenixArray)o;
        int noOfElements = array.numElements;
        int totalVarSize = 0;
        int nullsRepeationCounter = 0;
        int nulls = 0;
        int totalNulls = 0;
        for (int i = 0; i < noOfElements; i++) {
            totalVarSize += array.estimateByteSize(i);
            if (!PDataType.fromTypeId((baseType.getSqlType() - PDataType.ARRAY_TYPE_BASE)).isFixedWidth()) {
                if (array.isNull(i)) {
                    nulls++;
                } else {
                    if (nulls > 0) {
                        totalNulls += nulls;
                        nulls = 0;
                        nullsRepeationCounter++;
                    }
                }
            }
        }
        if (nullsVsNullRepeationCounter != null) {
            if (nulls > 0) {
                totalNulls += nulls;
                // do not increment nullsRepeationCounter to identify trailing nulls
            }
            nullsVsNullRepeationCounter.setFirst(totalNulls);
            nullsVsNullRepeationCounter.setSecond(nullsRepeationCounter);
        }
        return totalVarSize;
    }
    // Non fixed width types must override this
    throw new UnsupportedOperationException();
}
 

/**
 * Convert JanusGraph internal Mutation representation into HBase native commands.
 *
 * @param mutations    Mutations to convert into HBase commands.
 * @param putTimestamp The timestamp to use for Put commands.
 * @param delTimestamp The timestamp to use for Delete commands.
 * @return Commands sorted by key converted from JanusGraph internal representation.
 * @throws org.janusgraph.diskstorage.PermanentBackendException
 */
 @VisibleForTesting
 Map<StaticBuffer, Pair<List<Put>, Delete>> convertToCommands(Map<String, Map<StaticBuffer, KCVMutation>> mutations,
                                                              final long putTimestamp,
                                                              final long delTimestamp) throws PermanentBackendException {
    // A map of rowkey to commands (list of Puts, Delete)
    final Map<StaticBuffer, Pair<List<Put>, Delete>> commandsPerKey = new HashMap<>();

    for (Map.Entry<String, Map<StaticBuffer, KCVMutation>> entry : mutations.entrySet()) {

        String cfString = getCfNameForStoreName(entry.getKey());
        byte[] cfName = Bytes.toBytes(cfString);

        for (Map.Entry<StaticBuffer, KCVMutation> m : entry.getValue().entrySet()) {
            final byte[] key = m.getKey().as(StaticBuffer.ARRAY_FACTORY);
            KCVMutation mutation = m.getValue();

            Pair<List<Put>, Delete> commands = commandsPerKey.get(m.getKey());

            // The firt time we go through the list of input <rowkey, KCVMutation>,
            // create the holder for a particular rowkey
            if (commands == null) {
                commands = new Pair<>();
                // List of all the Puts for this rowkey, including the ones without TTL and with TTL.
                final List<Put> putList = new ArrayList<>();
                commands.setFirst(putList);
                commandsPerKey.put(m.getKey(), commands);
            }

            if (mutation.hasDeletions()) {
                if (commands.getSecond() == null) {
                    Delete d = new Delete(key);
                    compat.setTimestamp(d, delTimestamp);
                    commands.setSecond(d);
                }

                for (StaticBuffer b : mutation.getDeletions()) {
                    // commands.getSecond() is a Delete for this rowkey.
                    commands.getSecond().addColumns(cfName, b.as(StaticBuffer.ARRAY_FACTORY), delTimestamp);
                }
            }

            if (mutation.hasAdditions()) {
                // All the entries (column cells) with the rowkey use this one Put, except the ones with TTL.
                final Put putColumnsWithoutTtl = new Put(key, putTimestamp);
                // At the end of this loop, there will be one Put entry in the commands.getFirst() list that
                // contains all additions without TTL set, and possible multiple Put entries for columns
                // that have TTL set.
                for (Entry e : mutation.getAdditions()) {

                    // Deal with TTL within the entry (column cell) first
                    // HBase cell level TTL is actually set at the Mutation/Put level.
                    // Therefore we need to construct a new Put for each entry (column cell) with TTL.
                    // We can not combine them because column cells within the same rowkey may:
                    // 1. have no TTL
                    // 2. have TTL
                    // 3. have different TTL
                    final Integer ttl = (Integer) e.getMetaData().get(EntryMetaData.TTL);
                    if (null != ttl && ttl > 0) {
                        // Create a new Put
                        Put putColumnWithTtl = new Put(key, putTimestamp);
                        addColumnToPut(putColumnWithTtl, cfName, putTimestamp, e);
                        // Convert ttl from second (JanusGraph TTL) to millisec (HBase TTL)
                        // @see JanusGraphManagement#setTTL(JanusGraphSchemaType, Duration)
                        // Cast Put to Mutation for backward compatibility with HBase 0.98.x
                        // HBase supports cell-level TTL for versions 0.98.6 and above.
                        ((Mutation) putColumnWithTtl).setTTL(ttl * 1000);
                        // commands.getFirst() is the list of Puts for this rowkey. Add this
                        // Put column with TTL to the list.
                        commands.getFirst().add(putColumnWithTtl);
                    } else {
                        addColumnToPut(putColumnsWithoutTtl, cfName, putTimestamp, e);
                    }
                }
                // If there were any mutations without TTL set, add them to commands.getFirst()
                if (!putColumnsWithoutTtl.isEmpty()) {
                    commands.getFirst().add(putColumnsWithoutTtl);
                }
            }
        }
    }

    return commandsPerKey;
}
 

/**
 * Convert Titan internal Mutation representation into HBase native commands.
 *
 * @param mutations    Mutations to convert into HBase commands.
 * @param putTimestamp The timestamp to use for Put commands.
 * @param delTimestamp The timestamp to use for Delete commands.
 * @return Commands sorted by key converted from Titan internal representation.
 * @throws com.thinkaurelius.titan.diskstorage.PermanentBackendException
 */
private Map<StaticBuffer, Pair<Put, Delete>> convertToCommands(Map<String, Map<StaticBuffer, KCVMutation>> mutations,
                                                               final long putTimestamp,
                                                               final long delTimestamp) throws PermanentBackendException {
    Map<StaticBuffer, Pair<Put, Delete>> commandsPerKey = new HashMap<StaticBuffer, Pair<Put, Delete>>();

    for (Map.Entry<String, Map<StaticBuffer, KCVMutation>> entry : mutations.entrySet()) {

        String cfString = getCfNameForStoreName(entry.getKey());
        byte[] cfName = cfString.getBytes();

        for (Map.Entry<StaticBuffer, KCVMutation> m : entry.getValue().entrySet()) {
            byte[] key = m.getKey().as(StaticBuffer.ARRAY_FACTORY);
            KCVMutation mutation = m.getValue();

            Pair<Put, Delete> commands = commandsPerKey.get(m.getKey());

            if (commands == null) {
                commands = new Pair<Put, Delete>();
                commandsPerKey.put(m.getKey(), commands);
            }

            if (mutation.hasDeletions()) {
                if (commands.getSecond() == null) {
                    Delete d = new Delete(key);
                    compat.setTimestamp(d, delTimestamp);
                    commands.setSecond(d);
                }

                for (StaticBuffer b : mutation.getDeletions()) {
                    commands.getSecond().deleteColumns(cfName, b.as(StaticBuffer.ARRAY_FACTORY), delTimestamp);
                }
            }

            if (mutation.hasAdditions()) {
                if (commands.getFirst() == null) {
                    Put p = new Put(key, putTimestamp);
                    commands.setFirst(p);
                }

                for (Entry e : mutation.getAdditions()) {
                    commands.getFirst().add(cfName,
                            e.getColumnAs(StaticBuffer.ARRAY_FACTORY),
                            putTimestamp,
                            e.getValueAs(StaticBuffer.ARRAY_FACTORY));
                }
            }
        }
    }

    return commandsPerKey;
}
 

/**
 * Convert Titan internal Mutation representation into HBase native commands.
 *
 * @param mutations    Mutations to convert into HBase commands.
 * @param putTimestamp The timestamp to use for Put commands.
 * @param delTimestamp The timestamp to use for Delete commands.
 * @return Commands sorted by key converted from Titan internal representation.
 * @throws com.thinkaurelius.titan.diskstorage.PermanentBackendException
 */
private Map<StaticBuffer, Pair<Put, Delete>> convertToCommands(Map<String, Map<StaticBuffer, KCVMutation>> mutations,
                                                               final long putTimestamp,
                                                               final long delTimestamp) throws PermanentBackendException {
    Map<StaticBuffer, Pair<Put, Delete>> commandsPerKey = new HashMap<>();

    for (Map.Entry<String, Map<StaticBuffer, KCVMutation>> entry : mutations.entrySet()) {

        String cfString = getCfNameForStoreName(entry.getKey());
        byte[] cfName = cfString.getBytes();

        for (Map.Entry<StaticBuffer, KCVMutation> m : entry.getValue().entrySet()) {
            byte[] key = m.getKey().as(StaticBuffer.ARRAY_FACTORY);
            KCVMutation mutation = m.getValue();

            Pair<Put, Delete> commands = commandsPerKey.get(m.getKey());

            if (commands == null) {
                commands = new Pair<>();
                commandsPerKey.put(m.getKey(), commands);
            }

            if (mutation.hasDeletions()) {
                if (commands.getSecond() == null) {
                    Delete d = new Delete(key);
                    compat.setTimestamp(d, delTimestamp);
                    commands.setSecond(d);
                }

                for (StaticBuffer b : mutation.getDeletions()) {
                    commands.getSecond().deleteColumns(cfName, b.as(StaticBuffer.ARRAY_FACTORY), delTimestamp);
                }
            }

            if (mutation.hasAdditions()) {
                if (commands.getFirst() == null) {
                    Put p = new Put(key, putTimestamp);
                    commands.setFirst(p);
                }

                for (Entry e : mutation.getAdditions()) {
                    commands.getFirst().add(cfName,
                            e.getColumnAs(StaticBuffer.ARRAY_FACTORY),
                            putTimestamp,
                            e.getValueAs(StaticBuffer.ARRAY_FACTORY));
                }
            }
        }
    }

    return commandsPerKey;
}
 

/**
 * Update the current statistics based on the latest batch of key-values from the underlying scanner
 * 
 * @param results
 *            next batch of {@link KeyValue}s
 */
public void collectStatistics(final List<Cell> results) {
    Map<ImmutableBytesPtr, Boolean> famMap = Maps.newHashMap();
    List<GuidePostsInfo> rowTracker = null;
    if(cachedGps == null) {
        rowTracker = 
                new ArrayList<GuidePostsInfo>();
    }
    for (Cell cell : results) {
        KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
        maxTimeStamp = Math.max(maxTimeStamp, kv.getTimestamp());
        Pair<Long, GuidePostsInfo> gps;
        if (cachedGps == null) {
            ImmutableBytesPtr cfKey = new ImmutableBytesPtr(kv.getFamilyArray(), kv.getFamilyOffset(),
                    kv.getFamilyLength());
            gps = guidePostsMap.get(cfKey);
            if (gps == null) {
                gps = new Pair<Long, GuidePostsInfo>(0l, new GuidePostsInfo(0,
                        Collections.<byte[]> emptyList(), 0l));
                guidePostsMap.put(cfKey, gps);
            }
            if (famMap.get(cfKey) == null) {
                famMap.put(cfKey, true);
                rowTracker.add(gps.getSecond());
            }
        } else {
            gps = cachedGps;
        }
        int kvLength = kv.getLength();
        long byteCount = gps.getFirst() + kvLength;
        gps.setFirst(byteCount);
        if (byteCount >= guidepostDepth) {
            byte[] row = ByteUtil.copyKeyBytesIfNecessary(new ImmutableBytesWritable(kv.getRowArray(), kv
                    .getRowOffset(), kv.getRowLength()));
            if (gps.getSecond().addGuidePost(row, byteCount)) {
                gps.setFirst(0l);
            }
        }
    }
    if(cachedGps == null) {
        for (GuidePostsInfo s : rowTracker) {
            s.incrementRowCount();
        }
    } else {
        cachedGps.getSecond().incrementRowCount();
    }
}
 

/**
 * Update the current statistics based on the latest batch of key-values from the underlying scanner
 * 
 * @param results
 *            next batch of {@link KeyValue}s
 * @throws IOException 
 */
@Override
public void collectStatistics(final List<Cell> results) {
    // A guide posts depth of zero disables the collection of stats
    if (guidePostDepth == 0 || results.size() == 0) {
        return;
    }
    Map<ImmutableBytesPtr, Boolean> famMap = Maps.newHashMap();
    boolean incrementRow = false;
    Cell c = results.get(0);
    ImmutableBytesWritable row = new ImmutableBytesWritable(c.getRowArray(), c.getRowOffset(), c.getRowLength());
    /*
     * During compaction, it is possible that HBase will not return all the key values when
     * internalScanner.next() is called. So we need the below check to avoid counting a row more
     * than once.
     */
    if (currentRow == null || !row.equals(currentRow)) {
        currentRow = row;
        incrementRow = true;
    }
    for (Cell cell : results) {
        maxTimeStamp = Math.max(maxTimeStamp, cell.getTimestamp());
        Pair<Long, GuidePostsInfoBuilder> gps;
        if (cachedGuidePosts == null) {
            ImmutableBytesPtr cfKey = new ImmutableBytesPtr(cell.getFamilyArray(), cell.getFamilyOffset(),
                    cell.getFamilyLength());
            gps = guidePostsInfoWriterMap.get(cfKey);
            if (gps == null) {
                gps = new Pair<Long, GuidePostsInfoBuilder>(0l,
                        new GuidePostsInfoBuilder());
                guidePostsInfoWriterMap.put(cfKey, gps);
            }
            if (famMap.get(cfKey) == null) {
                famMap.put(cfKey, true);
                gps.getSecond().incrementRowCount();
            }
        } else {
            gps = cachedGuidePosts;
            if (incrementRow) {
                cachedGuidePosts.getSecond().incrementRowCount();
                incrementRow = false;
            }
        }
        int kvLength = KeyValueUtil.getSerializedSize(cell, true);
        long byteCount = gps.getFirst() + kvLength;
        gps.setFirst(byteCount);
        if (byteCount >= guidePostDepth) {
            if (gps.getSecond().addGuidePostOnCollection(row, byteCount, gps.getSecond().getRowCount())) {
                gps.setFirst(0l);
                gps.getSecond().resetRowCount();
            }
        }
    }
}