org.apache.hadoop.hbase.util.Pair Java Examples

The following examples show how to use org.apache.hadoop.hbase.util.Pair. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: DeployUtil.java    From Kylin with Apache License 2.0 6 votes vote down vote up
public static void deployJobJars() throws IOException {
    Pair<File, File> files = getJobJarFiles();
    File originalJobJar = files.getFirst();
    File originalCoprocessorJar = files.getSecond();

    String jobJarPath = config().getKylinJobJarPath();
    if (StringUtils.isEmpty(jobJarPath)) {
        throw new RuntimeException("deployJobJars cannot find job jar");
    }

    File targetJobJar = new File(jobJarPath);
    File jobJarRenamedAsTarget = new File(originalJobJar.getParentFile(), targetJobJar.getName());
    if (originalJobJar.equals(jobJarRenamedAsTarget) == false) {
        FileUtils.copyFile(originalJobJar, jobJarRenamedAsTarget);
    }

    File targetCoprocessorJar = new File(config().getCoprocessorLocalJar());
    File coprocessorJarRenamedAsTarget = new File(originalCoprocessorJar.getParentFile(), targetCoprocessorJar.getName());
    if (originalCoprocessorJar.equals(coprocessorJarRenamedAsTarget) == false) {
        FileUtils.copyFile(originalCoprocessorJar, coprocessorJarRenamedAsTarget);
    }

    CliCommandExecutor cmdExec = config().getCliCommandExecutor();
    cmdExec.copyFile(jobJarRenamedAsTarget.getAbsolutePath(), targetJobJar.getParent());
    cmdExec.copyFile(coprocessorJarRenamedAsTarget.getAbsolutePath(), targetCoprocessorJar.getParent());
}
 
Example #2
Source File: TestZKProcedureControllers.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Override
public Pair<ZKProcedureCoordinator, List<ZKProcedureMemberRpcs>> start(
        ZKWatcher watcher, String operationName,
        ProcedureCoordinator coordinator, String controllerName,
        ProcedureMember member, List<String> expected) throws Exception {
  // start the controller
  ZKProcedureCoordinator controller = new ZKProcedureCoordinator(
      watcher, operationName, CONTROLLER_NODE_NAME);
  controller.start(coordinator);

  // make a cohort controller for each expected node

  List<ZKProcedureMemberRpcs> cohortControllers = new ArrayList<>();
  for (String nodeName : expected) {
    ZKProcedureMemberRpcs cc = new ZKProcedureMemberRpcs(watcher, operationName);
    cc.start(nodeName, member);
    cohortControllers.add(cc);
  }
  return new Pair<>(controller, cohortControllers);
}
 
Example #3
Source File: IndexLoadBalancerIT.java    From phoenix with Apache License 2.0 6 votes vote down vote up
public boolean checkForColocation(HMaster master, String tableName, String indexTableName)
        throws IOException, InterruptedException {
    List<Pair<byte[], ServerName>> uTableStartKeysAndLocations =
            getStartKeysAndLocations(master, tableName);
    List<Pair<byte[], ServerName>> iTableStartKeysAndLocations =
            getStartKeysAndLocations(master, indexTableName);

    boolean regionsColocated = true;
    if (uTableStartKeysAndLocations.size() != iTableStartKeysAndLocations.size()) {
        regionsColocated = false;
    } else {
        for (int i = 0; i < uTableStartKeysAndLocations.size(); i++) {
            Pair<byte[], ServerName> uStartKeyAndLocation = uTableStartKeysAndLocations.get(i);
            Pair<byte[], ServerName> iStartKeyAndLocation = iTableStartKeysAndLocations.get(i);

            if (Bytes.compareTo(uStartKeyAndLocation.getFirst(), iStartKeyAndLocation
                    .getFirst()) == 0) {
                if (uStartKeyAndLocation.getSecond().equals(iStartKeyAndLocation.getSecond())) {
                    continue;
                }
            }
            regionsColocated = false;
        }
    }
    return regionsColocated;
}
 
Example #4
Source File: HRegionServer.java    From hbase with Apache License 2.0 6 votes vote down vote up
protected void initializeMemStoreChunkCreator() {
  if (MemStoreLAB.isEnabled(conf)) {
    // MSLAB is enabled. So initialize MemStoreChunkPool
    // By this time, the MemstoreFlusher is already initialized. We can get the global limits from
    // it.
    Pair<Long, MemoryType> pair = MemorySizeUtil.getGlobalMemStoreSize(conf);
    long globalMemStoreSize = pair.getFirst();
    boolean offheap = this.regionServerAccounting.isOffheap();
    // When off heap memstore in use, take full area for chunk pool.
    float poolSizePercentage = offheap? 1.0F:
        conf.getFloat(MemStoreLAB.CHUNK_POOL_MAXSIZE_KEY, MemStoreLAB.POOL_MAX_SIZE_DEFAULT);
    float initialCountPercentage = conf.getFloat(MemStoreLAB.CHUNK_POOL_INITIALSIZE_KEY,
        MemStoreLAB.POOL_INITIAL_SIZE_DEFAULT);
    int chunkSize = conf.getInt(MemStoreLAB.CHUNK_SIZE_KEY, MemStoreLAB.CHUNK_SIZE_DEFAULT);
    // init the chunkCreator
    ChunkCreator.initialize(chunkSize, offheap, globalMemStoreSize, poolSizePercentage,
      initialCountPercentage, this.hMemManager);
  }
}
 
Example #5
Source File: ScanPlan.java    From phoenix with Apache License 2.0 6 votes vote down vote up
/**
 * @return Pair of numbers in which the first part is estimated number of bytes that will be
 *         scanned and the second part is estimated number of rows. Returned value is null if
 *         estimated size of data to scan is beyond a threshold.
 * @throws SQLException
 */
private static Pair<Long, Long> getEstimateOfDataSizeToScanIfWithinThreshold(StatementContext context, PTable table, Integer perScanLimit) throws SQLException {
    Scan scan = context.getScan();
    ConnectionQueryServices services = context.getConnection().getQueryServices();
    long estRowSize = SchemaUtil.estimateRowSize(table);
    long regionSize = services.getProps().getLong(HConstants.HREGION_MAX_FILESIZE,
            HConstants.DEFAULT_MAX_FILE_SIZE);
    if (perScanLimit == null || scan.getFilter() != null) {
        /*
         * If a limit is not provided or if we have a filter, then we are not able to decide whether
         * the amount of data we need to scan is less than the threshold.
         */
        return null;
    } 
    float factor =
        services.getProps().getFloat(QueryServices.LIMITED_QUERY_SERIAL_THRESHOLD,
            QueryServicesOptions.DEFAULT_LIMITED_QUERY_SERIAL_THRESHOLD);
    long threshold = (long)(factor * regionSize);
    long estimatedBytes = perScanLimit * estRowSize;
    long estimatedRows = perScanLimit;
    return (perScanLimit * estRowSize < threshold) ? new Pair<>(estimatedBytes, estimatedRows) : null;
}
 
Example #6
Source File: FuzzyRowFilter.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * @return true if and only if the fields of the filter that are serialized are equal to the
 *         corresponding fields in other. Used for testing.
 */
@Override
boolean areSerializedFieldsEqual(Filter o) {
  if (o == this) return true;
  if (!(o instanceof FuzzyRowFilter)) return false;

  FuzzyRowFilter other = (FuzzyRowFilter) o;
  if (this.fuzzyKeysData.size() != other.fuzzyKeysData.size()) return false;
  for (int i = 0; i < fuzzyKeysData.size(); ++i) {
    Pair<byte[], byte[]> thisData = this.fuzzyKeysData.get(i);
    Pair<byte[], byte[]> otherData = other.fuzzyKeysData.get(i);
    if (!(Bytes.equals(thisData.getFirst(), otherData.getFirst()) && Bytes.equals(
      thisData.getSecond(), otherData.getSecond()))) {
      return false;
    }
  }
  return true;
}
 
Example #7
Source File: ZKReplicationQueueStorage.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * Return the {lastPushedSequenceId, ZNodeDataVersion} pair. if ZNodeDataVersion is -1, it means
 * that the ZNode does not exist.
 */
@VisibleForTesting
protected Pair<Long, Integer> getLastSequenceIdWithVersion(String encodedRegionName,
    String peerId) throws KeeperException {
  Stat stat = new Stat();
  String path = getSerialReplicationRegionPeerNode(encodedRegionName, peerId);
  byte[] data = ZKUtil.getDataNoWatch(zookeeper, path, stat);
  if (data == null) {
    // ZNode does not exist, so just return version -1 to indicate that no node exist.
    return Pair.newPair(HConstants.NO_SEQNUM, -1);
  }
  try {
    return Pair.newPair(ZKUtil.parseWALPositionFrom(data), stat.getVersion());
  } catch (DeserializationException de) {
    LOG.warn("Failed to parse log position (region=" + encodedRegionName + ", peerId=" + peerId
        + "), data=" + Bytes.toStringBinary(data));
  }
  return Pair.newPair(HConstants.NO_SEQNUM, stat.getVersion());
}
 
Example #8
Source File: TupleProjector.java    From phoenix with Apache License 2.0 6 votes vote down vote up
/**
 * Iterate over the list of cells returned from the scan and return a tuple projector for the
 * dynamic columns by parsing the metadata stored for the list of dynamic columns
 * @param result list of cells
 * @param dynCols list of dynamic columns to be populated
 * @param dynColCells list of cells corresponding to dynamic columns to be populated
 * @return The tuple projector corresponding to dynamic columns or null if there are no dynamic
 * columns to process
 * @throws InvalidProtocolBufferException Thrown if there is an error parsing byte[] to protobuf
 */
public static TupleProjector getDynamicColumnsTupleProjector(List<Cell> result,
        List<PColumn> dynCols, List<Cell> dynColCells) throws InvalidProtocolBufferException {
    Set<Pair<ByteBuffer, ByteBuffer>> dynColCellQualifiers = new HashSet<>();
    populateDynColsFromResult(result, dynCols, dynColCellQualifiers);
    if (dynCols.isEmpty()) {
        return null;
    }
    populateDynamicColumnCells(result, dynColCellQualifiers, dynColCells);
    if (dynColCells.isEmpty()) {
        return null;
    }
    KeyValueSchema dynColsSchema = PhoenixRuntime.buildKeyValueSchema(dynCols);
    Expression[] expressions = new Expression[dynCols.size()];
    for (int i = 0; i < dynCols.size(); i++) {
        expressions[i] = new KeyValueColumnExpression(dynCols.get(i));
    }
    return new TupleProjector(dynColsSchema, expressions);
}
 
Example #9
Source File: CubeManager.java    From Kylin with Apache License 2.0 6 votes vote down vote up
public Pair<CubeSegment, CubeSegment> appendAndMergeSegments(CubeInstance cube, long endDate) throws IOException {
    checkNoBuildingSegment(cube);
    checkCubeIsPartitioned(cube);

    if (cube.getSegments().size() == 0)
        throw new IllegalStateException("expect at least one existing segment");

    long appendStart = calculateStartDateForAppendSegment(cube);
    CubeSegment appendSegment = newSegment(cube, appendStart, endDate);

    long startDate = cube.getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
    CubeSegment mergeSegment = newSegment(cube, startDate, endDate);

    validateNewSegments(cube, mergeSegment);
    cube.getSegments().add(appendSegment);
    cube.getSegments().add(mergeSegment);
    Collections.sort(cube.getSegments());
    updateCube(cube);

    return new Pair<CubeSegment, CubeSegment>(appendSegment, mergeSegment);
}
 
Example #10
Source File: ConnectionlessQueryServicesImpl.java    From phoenix with Apache License 2.0 6 votes vote down vote up
@Override
public MetaDataMutationResult getFunctions(PName tenantId,
        List<Pair<byte[], Long>> functionNameAndTimeStampPairs, long clientTimestamp)
        throws SQLException {
    List<PFunction> functions = new ArrayList<PFunction>(functionNameAndTimeStampPairs.size());
    for(Pair<byte[], Long> functionInfo: functionNameAndTimeStampPairs) {
        try {
            PFunction function2 = metaData.getFunction(new PTableKey(tenantId, Bytes.toString(functionInfo.getFirst())));
            functions.add(function2);
        } catch (FunctionNotFoundException e) {
            return new MetaDataMutationResult(MutationCode.FUNCTION_NOT_FOUND, 0, null);
        }
    }
    if(functions.isEmpty()) {
        return null;
    }
    return new MetaDataMutationResult(MutationCode.FUNCTION_ALREADY_EXISTS, 0, functions, true);
}
 
Example #11
Source File: ExpressionUtil.java    From phoenix with Apache License 2.0 6 votes vote down vote up
/**
 * <pre>
 * Infer OrderBys from the rowkey columns of {@link PTable},for projected table may be there is no rowkey columns,
 * so we should move forward to inspect {@link ProjectedColumn} by {@link #getOrderByFromProjectedTable}.
 * The second part of the return pair is the rowkey column offset we must skip when we create OrderBys, because for table with salted/multiTenant/viewIndexId,
 * some leading rowkey columns should be skipped.
 * </pre>
 * @param tableRef
 * @param phoenixConnection
 * @param orderByReverse
 * @return
 * @throws SQLException
 */
public static Pair<OrderBy,Integer> getOrderByFromTable(
        TableRef tableRef,
        PhoenixConnection phoenixConnection,
        boolean orderByReverse) throws SQLException {

    PTable table = tableRef.getTable();
    Pair<OrderBy,Integer> orderByAndRowKeyColumnOffset =
            getOrderByFromTableByRowKeyColumn(table, phoenixConnection, orderByReverse);
    if(orderByAndRowKeyColumnOffset.getFirst() != OrderBy.EMPTY_ORDER_BY) {
        return orderByAndRowKeyColumnOffset;
    }
    if(table.getType() == PTableType.PROJECTED) {
        orderByAndRowKeyColumnOffset =
                getOrderByFromProjectedTable(tableRef, phoenixConnection, orderByReverse);
        if(orderByAndRowKeyColumnOffset.getFirst() != OrderBy.EMPTY_ORDER_BY) {
            return orderByAndRowKeyColumnOffset;
        }
    }
    return new Pair<OrderBy,Integer>(OrderBy.EMPTY_ORDER_BY, 0);
}
 
Example #12
Source File: RequestConverter.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * Create a protocol buffer UpdateFavoredNodesRequest to update a list of favorednode mappings
 * @param updateRegionInfos a list of favored node mappings
 * @return a protocol buffer UpdateFavoredNodesRequest
 */
public static UpdateFavoredNodesRequest buildUpdateFavoredNodesRequest(
    final List<Pair<RegionInfo, List<ServerName>>> updateRegionInfos) {
  UpdateFavoredNodesRequest.Builder ubuilder = UpdateFavoredNodesRequest.newBuilder();
  if (updateRegionInfos != null && !updateRegionInfos.isEmpty()) {
    RegionUpdateInfo.Builder builder = RegionUpdateInfo.newBuilder();
    for (Pair<RegionInfo, List<ServerName>> pair : updateRegionInfos) {
      builder.setRegion(ProtobufUtil.toRegionInfo(pair.getFirst()));
      for (ServerName server : pair.getSecond()) {
        builder.addFavoredNodes(ProtobufUtil.toServerName(server));
      }
      ubuilder.addUpdateInfo(builder.build());
      builder.clear();
    }
  }
  return ubuilder.build();
}
 
Example #13
Source File: SplitTableRegionProcedure.java    From hbase with Apache License 2.0 6 votes vote down vote up
private Pair<Path, Path> splitStoreFile(HRegionFileSystem regionFs, byte[] family, HStoreFile sf)
  throws IOException {
  if (LOG.isDebugEnabled()) {
    LOG.debug("pid=" + getProcId() + " splitting started for store file: " +
        sf.getPath() + " for region: " + getParentRegion().getShortNameToLog());
  }

  final byte[] splitRow = getSplitRow();
  final String familyName = Bytes.toString(family);
  final Path path_first = regionFs.splitStoreFile(this.daughterOneRI, familyName, sf, splitRow,
      false, splitPolicy);
  final Path path_second = regionFs.splitStoreFile(this.daughterTwoRI, familyName, sf, splitRow,
     true, splitPolicy);
  if (LOG.isDebugEnabled()) {
    LOG.debug("pid=" + getProcId() + " splitting complete for store file: " +
        sf.getPath() + " for region: " + getParentRegion().getShortNameToLog());
  }
  return new Pair<Path,Path>(path_first, path_second);
}
 
Example #14
Source File: IndexScrutinyMapper.java    From phoenix with Apache License 2.0 6 votes vote down vote up
@Override
protected void map(NullWritable key, PhoenixIndexDBWritable record, Context context)
        throws IOException, InterruptedException {
    try {
        final List<Object> values = record.getValues();

        context.getCounter(PhoenixJobCounters.INPUT_RECORDS).increment(1);
        currentBatchValues.add(new Pair<>(record.getRowTs(), values));
        if (context.getCounter(PhoenixJobCounters.INPUT_RECORDS).getValue() % batchSize != 0) {
            // if we haven't hit the batch size, just report progress and move on to next record
            context.progress();
            return;
        } else {
            // otherwise, process the batch
            processBatch(context);
        }
        context.progress(); // Make sure progress is reported to Application Master.
    } catch (SQLException | IllegalArgumentException e) {
        LOGGER.error(" Error while read/write of a record ", e);
        context.getCounter(PhoenixJobCounters.FAILED_RECORDS).increment(1);
        throw new IOException(e);
    }
}
 
Example #15
Source File: TestRSGroupsAdmin2.java    From hbase with Apache License 2.0 6 votes vote down vote up
private Pair<ServerName, RegionStateNode> createTableWithRegionSplitting(RSGroupInfo rsGroupInfo,
  int tableRegionCount) throws Exception {
  final byte[] familyNameBytes = Bytes.toBytes("f");
  // All the regions created below will be assigned to the default group.
  TEST_UTIL.createMultiRegionTable(tableName, familyNameBytes, tableRegionCount);
  TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
    @Override
    public boolean evaluate() throws Exception {
      List<String> regions = getTableRegionMap().get(tableName);
      if (regions == null) {
        return false;
      }
      return getTableRegionMap().get(tableName).size() >= tableRegionCount;
    }
  });

  return randomlySetOneRegionStateToSplitting(rsGroupInfo);
}
 
Example #16
Source File: BackupObserver.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Override
public void preCommitStoreFile(final ObserverContext<RegionCoprocessorEnvironment> ctx,
    final byte[] family, final List<Pair<Path, Path>> pairs) throws IOException {
  Configuration cfg = ctx.getEnvironment().getConfiguration();
  if (pairs == null || pairs.isEmpty() || !BackupManager.isBackupEnabled(cfg)) {
    LOG.debug("skipping recording bulk load in preCommitStoreFile since backup is disabled");
    return;
  }
  try (Connection connection = ConnectionFactory.createConnection(cfg);
      BackupSystemTable tbl = new BackupSystemTable(connection)) {
    List<TableName> fullyBackedUpTables = tbl.getTablesForBackupType(BackupType.FULL);
    RegionInfo info = ctx.getEnvironment().getRegionInfo();
    TableName tableName = info.getTable();
    if (!fullyBackedUpTables.contains(tableName)) {
      if (LOG.isTraceEnabled()) {
        LOG.trace(tableName + " has not gone thru full backup");
      }
      return;
    }
    tbl.writeFilesForBulkLoadPreCommit(tableName, info.getEncodedNameAsBytes(), family, pairs);
    return;
  }
}
 
Example #17
Source File: CloneSnapshotProcedure.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
protected void serializeStateData(ProcedureStateSerializer serializer)
    throws IOException {
  super.serializeStateData(serializer);

  MasterProcedureProtos.CloneSnapshotStateData.Builder cloneSnapshotMsg =
    MasterProcedureProtos.CloneSnapshotStateData.newBuilder()
      .setUserInfo(MasterProcedureUtil.toProtoUserInfo(getUser()))
      .setSnapshot(this.snapshot)
      .setTableSchema(ProtobufUtil.toTableSchema(tableDescriptor));
  if (newRegions != null) {
    for (RegionInfo hri: newRegions) {
      cloneSnapshotMsg.addRegionInfo(ProtobufUtil.toRegionInfo(hri));
    }
  }
  if (!parentsToChildrenPairMap.isEmpty()) {
    final Iterator<Map.Entry<String, Pair<String, String>>> it =
      parentsToChildrenPairMap.entrySet().iterator();
    while (it.hasNext()) {
      final Map.Entry<String, Pair<String, String>> entry = it.next();

      MasterProcedureProtos.RestoreParentToChildRegionsPair.Builder parentToChildrenPair =
        MasterProcedureProtos.RestoreParentToChildRegionsPair.newBuilder()
        .setParentRegionName(entry.getKey())
        .setChild1RegionName(entry.getValue().getFirst())
        .setChild2RegionName(entry.getValue().getSecond());
      cloneSnapshotMsg.addParentToChildRegionsPairList(parentToChildrenPair);
    }
  }
  serializer.serialize(cloneSnapshotMsg.build());
}
 
Example #18
Source File: CubeManager.java    From Kylin with Apache License 2.0 5 votes vote down vote up
public CubeSegment mergeSegments(CubeInstance cube, final long startDate, final long endDate) throws IOException {
    checkNoBuildingSegment(cube);
    checkCubeIsPartitioned(cube);

    Pair<Long, Long> range = alignMergeRange(cube, startDate, endDate);
    CubeSegment newSegment = newSegment(cube, range.getFirst(), range.getSecond());

    validateNewSegments(cube, newSegment);
    cube.getSegments().add(newSegment);
    Collections.sort(cube.getSegments());
    updateCube(cube);

    return newSegment;
}
 
Example #19
Source File: VisibilityController.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
public List<Pair<Cell, Cell>> postAppendBeforeWAL(
    ObserverContext<RegionCoprocessorEnvironment> ctx, Mutation mutation,
    List<Pair<Cell, Cell>> cellPairs) throws IOException {
  List<Pair<Cell, Cell>> resultPairs = new ArrayList<>(cellPairs.size());
  for (Pair<Cell, Cell> pair : cellPairs) {
    resultPairs
        .add(new Pair<>(pair.getFirst(), createNewCellWithTags(mutation, pair.getSecond())));
  }
  return resultPairs;
}
 
Example #20
Source File: TestMetaTableAccessor.java    From hbase with Apache License 2.0 5 votes vote down vote up
private static List<RegionInfo> testGettingTableRegions(final Connection connection,
  final TableName name, final int regionCount) throws IOException, InterruptedException {
  List<RegionInfo> regions = MetaTableAccessor.getTableRegions(connection, name);
  assertEquals(regionCount, regions.size());
  Pair<RegionInfo, ServerName> pair =
    MetaTableAccessor.getRegion(connection, regions.get(0).getRegionName());
  assertEquals(regions.get(0).getEncodedName(), pair.getFirst().getEncodedName());
  return regions;
}
 
Example #21
Source File: CatalogJanitor.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * If daughters no longer hold reference to the parents, delete the parent.
 * @param parent RegionInfo of split offlined parent
 * @param rowContent Content of <code>parent</code> row in
 * <code>metaRegionName</code>
 * @return True if we removed <code>parent</code> from meta table and from
 * the filesystem.
 */
boolean cleanParent(final RegionInfo parent, Result rowContent)
throws IOException {
  // Check whether it is a merged region and if it is clean of references.
  if (MetaTableAccessor.hasMergeRegions(rowContent.rawCells())) {
    // Wait until clean of merge parent regions first
    return false;
  }
  // Run checks on each daughter split.
  PairOfSameType<RegionInfo> daughters = MetaTableAccessor.getDaughterRegions(rowContent);
  Pair<Boolean, Boolean> a = checkDaughterInFs(parent, daughters.getFirst());
  Pair<Boolean, Boolean> b = checkDaughterInFs(parent, daughters.getSecond());
  if (hasNoReferences(a) && hasNoReferences(b)) {
    String daughterA = daughters.getFirst() != null?
        daughters.getFirst().getShortNameToLog(): "null";
    String daughterB = daughters.getSecond() != null?
        daughters.getSecond().getShortNameToLog(): "null";
    LOG.debug("Deleting region " + parent.getShortNameToLog() +
      " because daughters -- " + daughterA + ", " + daughterB +
      " -- no longer hold references");
    ProcedureExecutor<MasterProcedureEnv> pe = this.services.getMasterProcedureExecutor();
    pe.submitProcedure(new GCRegionProcedure(pe.getEnvironment(), parent));
    // Remove from in-memory states
    this.services.getAssignmentManager().getRegionStates().deleteRegion(parent);
    this.services.getServerManager().removeRegion(parent);
    return true;
  }
  return false;
}
 
Example #22
Source File: DefaultStatisticsCollector.java    From phoenix with Apache License 2.0 5 votes vote down vote up
@Override
public GuidePostsInfo getGuidePosts(ImmutableBytesPtr fam) {
    Pair<Long, GuidePostsInfoBuilder> pair = guidePostsInfoWriterMap.get(fam);
    if (pair != null) {
        return pair.getSecond().build();
    }
    return null;
}
 
Example #23
Source File: ConnectionQueryServicesImpl.java    From phoenix with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
@Override
public MetaDataMutationResult addColumn(final List<Mutation> tableMetaData, PTableType tableType, List<Pair<byte[],Map<String,Object>>> families) throws SQLException {
    byte[][] rowKeyMetaData = new byte[3][];

    byte[] rowKey = tableMetaData.get(0).getRow();
    SchemaUtil.getVarChars(rowKey, rowKeyMetaData);
    byte[] tenantIdBytes = rowKeyMetaData[PhoenixDatabaseMetaData.TENANT_ID_INDEX];
    byte[] schemaBytes = rowKeyMetaData[PhoenixDatabaseMetaData.SCHEMA_NAME_INDEX];
    byte[] tableBytes = rowKeyMetaData[PhoenixDatabaseMetaData.TABLE_NAME_INDEX];
    byte[] tableKey = SchemaUtil.getTableKey(tenantIdBytes, schemaBytes, tableBytes);
    byte[] tableName = SchemaUtil.getTableNameAsBytes(schemaBytes, tableBytes);
    for ( Pair<byte[],Map<String,Object>>  family : families) {
        
        PTable table = latestMetaData.getTable(Bytes.toString(tableName));
        ensureFamilyCreated(table.getPhysicalName().getBytes(), tableType, family);
    }
    // Special case for call during drop table to ensure that the empty column family exists.
    // In this, case we only include the table header row, as until we add schemaBytes and tableBytes
    // as args to this function, we have no way of getting them in this case.
    // TODO: change to  if (tableMetaData.isEmpty()) once we pass through schemaBytes and tableBytes
    // Also, could be used to update property values on ALTER TABLE t SET prop=xxx
    if (tableMetaData.size() == 1 && tableMetaData.get(0).isEmpty()) {
        return null;
    }
    MetaDataMutationResult result =  metaDataCoprocessorExec(tableKey,
        new Batch.Call<MetaDataProtocol, MetaDataMutationResult>() {
            @Override
            public MetaDataMutationResult call(MetaDataProtocol instance) throws IOException {
                return instance.addColumn(tableMetaData);
            }
        });
    return result;
}
 
Example #24
Source File: Indexer.java    From phoenix with Apache License 2.0 5 votes vote down vote up
/**
 * Add the index updates to the WAL, or write to the index table, if the WAL has been disabled
 * @return <tt>true</tt> if the WAL has been updated.
 * @throws IOException
 */
private boolean doPre(Collection<Pair<Mutation, byte[]>> indexUpdates, final WALEdit edit,
    final Durability durability) throws IOException {
  // no index updates, so we are done
  if (indexUpdates == null || indexUpdates.size() == 0) {
    return false;
  }

  // if writing to wal is disabled, we never see the WALEdit updates down the way, so do the index
  // update right away
  if (durability == Durability.SKIP_WAL) {
    try {
      this.writer.write(indexUpdates);
      return false;
    } catch (Throwable e) {
      LOG.error("Failed to update index with entries:" + indexUpdates, e);
      IndexManagementUtil.rethrowIndexingException(e);
    }
  }

  // we have all the WAL durability, so we just update the WAL entry and move on
  for (Pair<Mutation, byte[]> entry : indexUpdates) {
    edit.add(new IndexedKeyValue(entry.getSecond(), entry.getFirst()));
  }

  return true;
}
 
Example #25
Source File: ThreadPoolBuilder.java    From phoenix with Apache License 2.0 5 votes vote down vote up
public ThreadPoolBuilder setMaxThread(String confkey, int defaultThreads) {
  if (defaultThreads <= 0) {
    defaultThreads = DEFAULT_MAX_THREADS;
  }
  this.maxThreads = new Pair<String, Integer>(confkey, defaultThreads);
  return this;
}
 
Example #26
Source File: DeployUtil.java    From Kylin with Apache License 2.0 5 votes vote down vote up
public static void overrideJobJarLocations() {
    Pair<File, File> files = getJobJarFiles();
    File jobJar = files.getFirst();
    File coprocessorJar = files.getSecond();

    config().overrideKylinJobJarPath(jobJar.getAbsolutePath());
    config().overrideCoprocessorLocalJar(coprocessorJar.getAbsolutePath());
}
 
Example #27
Source File: BaseTenantSpecificViewIndexIT.java    From phoenix with Apache License 2.0 5 votes vote down vote up
private void createAndPopulateTenantView(Connection conn, String tenantId, String baseTable, String valuePrefix) throws SQLException {
    String ddl = "CREATE VIEW v(v2 VARCHAR) AS SELECT * FROM " + baseTable + " WHERE k1 = 1";
    conn.createStatement().execute(ddl);
    tenantViewsToDelete.add(new Pair<String, String>(tenantId, "v"));
    for (int i = 0; i < 10; i++) {
        conn.createStatement().execute("UPSERT INTO v(k2,v1,v2) VALUES(" + i + ",'" + valuePrefix + "v1-" + (i%5) + "','" + valuePrefix + "v2-" + (i%2) + "')");
    }
    conn.commit();
}
 
Example #28
Source File: CoveredColumnIndexer.java    From phoenix with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
@Override
public Collection<Pair<Mutation, byte[]>> getIndexUpdateForFilteredRows(
    Collection<KeyValue> filtered) throws IOException {

  // stores all the return values
  IndexUpdateManager updateMap = new IndexUpdateManager();
  // batch the updates by row to make life easier and ordered
  Collection<Batch> batches = batchByRow(filtered);

  for (Batch batch : batches) {
    Put p = new Put(batch.getKvs().iterator().next().getRow());
    for (KeyValue kv : batch.getKvs()) {
      // we only need to cleanup Put entries
      byte type = kv.getType();
      Type t = KeyValue.Type.codeToType(type);
      if (!t.equals(Type.Put)) {
        continue;
      }

      // add the kv independently
      p.add(kv);
    }

    // do the usual thing as for deletes
    Collection<Batch> timeBatch = createTimestampBatchesFromMutation(p);
    LocalTableState state = new LocalTableState(env, localTable, p);
    for (Batch entry : timeBatch) {
      //just set the timestamp on the table - it already has all the future state
      state.setCurrentTimestamp(entry.getTimestamp());
      this.addDeleteUpdatesToMap(updateMap, state, entry.getTimestamp());
    }
  }
  return updateMap.toMap();
}
 
Example #29
Source File: TestAsyncTableScanMetrics.java    From hbase with Apache License 2.0 5 votes vote down vote up
private static Pair<List<Result>, ScanMetrics> doScanWithRawAsyncTable(Scan scan)
    throws IOException, InterruptedException {
  BufferingScanResultConsumer consumer = new BufferingScanResultConsumer();
  CONN.getTable(TABLE_NAME).scan(scan, consumer);
  List<Result> results = new ArrayList<>();
  for (Result result; (result = consumer.take()) != null;) {
    results.add(result);
  }
  return Pair.newPair(results, consumer.getScanMetrics());
}
 
Example #30
Source File: TestMetaTableAccessor.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test
public void testGetRegion() throws IOException, InterruptedException {
  final String name = this.name.getMethodName();
  LOG.info("Started " + name);
  // Test get on non-existent region.
  Pair<RegionInfo, ServerName> pair =
    MetaTableAccessor.getRegion(connection, Bytes.toBytes("nonexistent-region"));
  assertNull(pair);
  LOG.info("Finished " + name);
}