org.apache.hadoop.hbase.client.HConnection Java Examples

The following examples show how to use org.apache.hadoop.hbase.client.HConnection. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: Tailer.java    From zerowing with MIT License 6 votes vote down vote up
public Tailer(Configuration conf, Mongo mongo, HConnection hbase, String tailerName) {
  _conf = conf;
  _mongo = mongo;
  _hbase = hbase;
  _knownTables = new HashMap<String, HTable>();
  _translator = ConfigUtil.getTranslator(_conf);

  _stateTable = createStateTable();

  if (tailerName == null) {
    List<ServerAddress> addresses = _mongo.getAllAddress();
    tailerName = StringUtils.join(addresses, ",");
  }

  _tailerID = tailerName.getBytes();

  _skipUpdates = ConfigUtil.getSkipUpdates(_conf);
  _skipDeletes = ConfigUtil.getSkipDeletes(_conf);
  _bufferWrites = ConfigUtil.getBufferWrites(_conf);
}
 
Example #2
Source File: RunLocalTest.java    From hadoop-arch-book with Apache License 2.0 6 votes vote down vote up
private static void populateUserProfileData(HConnection connection) throws Exception {
  UserProfile up1 = new UserProfile();
  up1.userId = "101";
  up1.lastUpdatedTimeStamp = System.currentTimeMillis() - 1000;
  up1.historicAvg90PercentSingleDaySpend = 90.0;
  up1.historicAvgSingleDaySpend = 50.0;
  up1.todayMaxSpend = 0.0;
  up1.todayNumOfPurchases = 0l;
  HBaseUtils.populateUserProfile(connection, up1);

  up1.userId = "102";
  up1.lastUpdatedTimeStamp = System.currentTimeMillis() - 1000;
  up1.historicAvg90PercentSingleDaySpend = 90.0;
  up1.historicAvgSingleDaySpend = 50.0;
  up1.todayMaxSpend = 0.0;
  up1.todayNumOfPurchases = 0l;
  HBaseUtils.populateUserProfile(connection, up1);

  up1.userId = "103";
  up1.lastUpdatedTimeStamp = System.currentTimeMillis() - 1000;
  up1.historicAvg90PercentSingleDaySpend = 90.0;
  up1.historicAvgSingleDaySpend = 50.0;
  up1.todayMaxSpend = 0.0;
  up1.todayNumOfPurchases = 0l;
  HBaseUtils.populateUserProfile(connection, up1);
}
 
Example #3
Source File: CIFHbaseAdapter.java    From opensoc-streaming with Apache License 2.0 6 votes vote down vote up
@Override
public boolean initializeAdapter() {

	// Initialize HBase Table
	Configuration conf = null;
	conf = HBaseConfiguration.create();
	conf.set("hbase.zookeeper.quorum", _quorum);
	conf.set("hbase.zookeeper.property.clientPort", _port);

	try {
		LOGGER.debug("=======Connecting to HBASE===========");
		LOGGER.debug("=======ZOOKEEPER = "
				+ conf.get("hbase.zookeeper.quorum"));
		HConnection connection = HConnectionManager.createConnection(conf);
		table = connection.getTable(_tableName);
		return true;
	} catch (IOException e) {
		// TODO Auto-generated catch block
		LOGGER.debug("=======Unable to Connect to HBASE===========");
		e.printStackTrace();
	}

	return false;
}
 
Example #4
Source File: GridTableHBaseBenchmark.java    From Kylin with Apache License 2.0 6 votes vote down vote up
public static void testGridTable(double hitRatio, double indexRatio) throws IOException {
    System.out.println("Testing grid table scanning, hit ratio " + hitRatio + ", index ratio " + indexRatio);
    String hbaseUrl = "hbase"; // use hbase-site.xml on classpath

    HConnection conn = HBaseConnection.get(hbaseUrl);
    createHTableIfNeeded(conn, TEST_TABLE);
    prepareData(conn);

    Hits hits = new Hits(N_ROWS, hitRatio, indexRatio);

    for (int i = 0; i < ROUND; i++) {
        System.out.println("==================================== ROUND " + (i + 1) + " ========================================");
        testRowScanWithIndex(conn, hits.getHitsForRowScanWithIndex());
        testRowScanNoIndexFullScan(conn, hits.getHitsForRowScanNoIndex());
        testRowScanNoIndexSkipScan(conn, hits.getHitsForRowScanNoIndex());
        testColumnScan(conn, hits.getHitsForColumnScan());
    }

}
 
Example #5
Source File: ThreatHbaseAdapter.java    From opensoc-streaming with Apache License 2.0 6 votes vote down vote up
@Override
public boolean initializeAdapter() {

	// Initialize HBase Table
	Configuration conf = null;
	conf = HBaseConfiguration.create();
	conf.set("hbase.zookeeper.quorum", _quorum);
	conf.set("hbase.zookeeper.property.clientPort", _port);

	try {
		LOGGER.debug("=======Connecting to HBASE===========");
		LOGGER.debug("=======ZOOKEEPER = "
				+ conf.get("hbase.zookeeper.quorum"));
		HConnection connection = HConnectionManager.createConnection(conf);
		table = connection.getTable(_tableName);
		return true;
	} catch (IOException e) {
		// TODO Auto-generated catch block
		LOGGER.debug("=======Unable to Connect to HBASE===========");
		e.printStackTrace();
	}

	return false;
}
 
Example #6
Source File: GridTableHBaseBenchmark.java    From Kylin with Apache License 2.0 6 votes vote down vote up
private static void fullScan(HConnection conn, boolean[] hits, Stats stats) throws IOException {
    HTableInterface table = conn.getTable(TEST_TABLE);
    try {
        stats.markStart();

        Scan scan = new Scan();
        scan.addFamily(CF);
        ResultScanner scanner = table.getScanner(scan);
        int i = 0;
        for (Result r : scanner) {
            if (hits[i])
                stats.consume(r);
            dot(i, N_ROWS);
            i++;
        }

        stats.markEnd();
    } finally {
        IOUtils.closeQuietly(table);
    }
}
 
Example #7
Source File: SerializedHBaseTupleIterator.java    From Kylin with Apache License 2.0 6 votes vote down vote up
public SerializedHBaseTupleIterator(HConnection conn, List<HBaseKeyRange> segmentKeyRanges, CubeInstance cube, Collection<TblColRef> dimensions, TupleFilter filter, Collection<TblColRef> groupBy, Collection<RowValueDecoder> rowValueDecoders, StorageContext context) {

        this.context = context;
        int limit = context.getLimit();
        this.partialResultLimit = Math.max(limit, PARTIAL_DEFAULT_LIMIT);

        this.segmentIteratorList = new ArrayList<CubeSegmentTupleIterator>(segmentKeyRanges.size());
        Map<CubeSegment, List<HBaseKeyRange>> rangesMap = makeRangesMap(segmentKeyRanges);
        for (Map.Entry<CubeSegment, List<HBaseKeyRange>> entry : rangesMap.entrySet()) {
            CubeSegmentTupleIterator segIter = new CubeSegmentTupleIterator(entry.getKey(), entry.getValue(), conn, dimensions, filter, groupBy, rowValueDecoders, context);
            this.segmentIteratorList.add(segIter);
        }

        this.segmentIteratorIterator = this.segmentIteratorList.iterator();
        if (this.segmentIteratorIterator.hasNext()) {
            this.segmentIterator = this.segmentIteratorIterator.next();
        } else {
            this.segmentIterator = ITupleIterator.EMPTY_TUPLE_ITERATOR;
        }
    }
 
Example #8
Source File: ImportHBaseData.java    From Kylin with Apache License 2.0 6 votes vote down vote up
public void setup() throws IOException {

        KylinConfig.destoryInstance();
        System.setProperty(KylinConfig.KYLIN_CONF, AbstractKylinTestCase.SANDBOX_TEST_DATA);

        kylinConfig = KylinConfig.getInstanceFromEnv();
        cli = kylinConfig.getCliCommandExecutor();

        String metadataUrl = kylinConfig.getMetadataUrl();
        // split TABLE@HBASE_URL
        int cut = metadataUrl.indexOf('@');
        tableNameBase = metadataUrl.substring(0, cut);
        String hbaseUrl = cut < 0 ? metadataUrl : metadataUrl.substring(cut + 1);

        HConnection conn = HBaseConnection.get(hbaseUrl);
        try {
            hbase = new HBaseAdmin(conn);
            config = hbase.getConfiguration();
            //allTables = hbase.listTables();
        } catch (IOException e) {
            e.printStackTrace();
            throw e;
        }

        uploadTarballToRemote();
    }
 
Example #9
Source File: CubeSegmentTupleIterator.java    From Kylin with Apache License 2.0 6 votes vote down vote up
public CubeSegmentTupleIterator(CubeSegment cubeSeg, Collection<HBaseKeyRange> keyRanges, HConnection conn, Collection<TblColRef> dimensions, TupleFilter filter, Collection<TblColRef> groupBy, Collection<RowValueDecoder> rowValueDecoders, StorageContext context) {
    this.cube = cubeSeg.getCubeInstance();
    this.cubeSeg = cubeSeg;
    this.dimensions = dimensions;
    this.filter = filter;
    this.groupBy = groupBy;
    this.rowValueDecoders = rowValueDecoders;
    this.context = context;
    this.tableName = cubeSeg.getStorageLocationIdentifier();
    this.rowKeyDecoder = new RowKeyDecoder(this.cubeSeg);
    this.scanCount = 0;

    try {
        this.table = conn.getTable(tableName);
    } catch (Throwable t) {
        throw new StorageException("Error when open connection to table " + tableName, t);
    }
    this.rangeIterator = keyRanges.iterator();
    scanNextRange();
}
 
Example #10
Source File: HBaseConnection.java    From Kylin with Apache License 2.0 6 votes vote down vote up
public static HConnection get(String url) {
    // find configuration
    Configuration conf = ConfigCache.get(url);
    if (conf == null) {
        conf = HadoopUtil.newHBaseConfiguration(url);
        ConfigCache.put(url, conf);
    }

    HConnection connection = ConnPool.get(url);
    try {
        // I don't use DCL since recreate a connection is not a big issue.
        if (connection == null) {
            connection = HConnectionManager.createConnection(conf);
            ConnPool.put(url, connection);
        }
    } catch (Throwable t) {
        throw new StorageException("Error when open connection " + url, t);
    }

    return connection;
}
 
Example #11
Source File: HBaseStorageSetup.java    From titan1withtp3.1 with Apache License 2.0 6 votes vote down vote up
public synchronized static void waitForConnection(long timeout, TimeUnit timeoutUnit) {
    long before = System.currentTimeMillis();
    long after;
    long timeoutMS = TimeUnit.MILLISECONDS.convert(timeout, timeoutUnit);
    do {
        try {
            HConnection hc = HConnectionManager.createConnection(HBaseConfiguration.create());
            hc.close();
            after = System.currentTimeMillis();
            log.info("HBase server to started after about {} ms", after - before);
            return;
        } catch (IOException e) {
            log.info("Exception caught while waiting for the HBase server to start", e);
        }
        after = System.currentTimeMillis();
    } while (timeoutMS > after - before);
    after = System.currentTimeMillis();
    log.warn("HBase server did not start in {} ms", after - before);
}
 
Example #12
Source File: HBaseSimpleDemo.java    From bigdata-tutorial with Apache License 2.0 5 votes vote down vote up
public HConnection openConn() {
	if (null == hconn) {
		try {
			this.hconn = HConnectionManager.createConnection(config);
		} catch (Exception ex) {
			ex.printStackTrace();
		}
	}
	return hconn;
}
 
Example #13
Source File: HBaseClientManager.java    From bigdata-tutorial with Apache License 2.0 5 votes vote down vote up
public synchronized HConnection getConn() {
	if (null == conn) {
		try {
			this.conn = HConnectionManager.createConnection(config);
		} catch (Exception ex) {
			LOGGER.error("create conn err:", ex);
		}
	}
	return conn;
}
 
Example #14
Source File: GridTableHBaseBenchmark.java    From Kylin with Apache License 2.0 5 votes vote down vote up
private static void testColumnScan(HConnection conn, List<Pair<Integer, Integer>> colScans) throws IOException {
    Stats stats = new Stats("COLUMN_SCAN");

    HTableInterface table = conn.getTable(TEST_TABLE);
    try {
        stats.markStart();

        int nLogicCols = colScans.size();
        int nLogicRows = colScans.get(0).getSecond() - colScans.get(0).getFirst();
        
        Scan[] scans = new Scan[nLogicCols];
        ResultScanner[] scanners = new ResultScanner[nLogicCols];
        for (int i = 0; i < nLogicCols; i++) {
            scans[i] = new Scan();
            scans[i].addFamily(CF);
            scanners[i] = table.getScanner(scans[i]);
        }
        for (int i = 0; i < nLogicRows; i++) {
            for (int c = 0; c < nLogicCols; c++) {
                Result r = scanners[c].next();
                stats.consume(r);
            }
            dot(i, nLogicRows);
        }
        
        stats.markEnd();
    } finally {
        IOUtils.closeQuietly(table);
    }
}
 
Example #15
Source File: EmptyRegionChecker.java    From hbase-tools with Apache License 2.0 5 votes vote down vote up
public EmptyRegionChecker(HConnection connection, String tableName,
                          HRegionInfo regionInfo, Set<HRegionInfo> emptyRegions) {
    this.connection = connection;
    this.tableName = tableName;
    this.regionInfo = regionInfo;
    this.emptyRegions = emptyRegions;
}
 
Example #16
Source File: HBaseConnAbstractPool.java    From bigdata-tutorial with Apache License 2.0 5 votes vote down vote up
public synchronized HConnection getConn() {
	if (null == conn) {
		try {
			this.conn = HConnectionManager.createConnection(this.config);
		} catch (Exception ex) {
			LOGGER.error("create conn err:", ex);
		}
	}
	return conn;
}
 
Example #17
Source File: EmptyRegionChecker.java    From hbase-tools with Apache License 2.0 5 votes vote down vote up
public EmptyRegionChecker(HConnection connection, String tableName,
                          HRegionInfo regionInfo, Set<HRegionInfo> emptyRegions) {
    this.connection = connection;
    this.tableName = tableName;
    this.regionInfo = regionInfo;
    this.emptyRegions = emptyRegions;
}
 
Example #18
Source File: EmptyRegionChecker.java    From hbase-tools with Apache License 2.0 5 votes vote down vote up
public EmptyRegionChecker(HConnection connection, String tableName,
                          HRegionInfo regionInfo, Set<HRegionInfo> emptyRegions) {
    this.connection = connection;
    this.tableName = tableName;
    this.regionInfo = regionInfo;
    this.emptyRegions = emptyRegions;
}
 
Example #19
Source File: RunLocalTest.java    From hadoop-arch-book with Apache License 2.0 5 votes vote down vote up
private static void populateValidationRules(HConnection connection) throws Exception {
  HashSet<String> banndedVandors = new HashSet<String>();
  banndedVandors.add("badVendor");

  ValidationRules rules = new ValidationRules(banndedVandors, 2.0);

  HBaseUtils.populateValidationRules(connection, rules);
}
 
Example #20
Source File: EmptyRegionChecker.java    From hbase-tools with Apache License 2.0 5 votes vote down vote up
public EmptyRegionChecker(HConnection connection, String tableName,
                          HRegionInfo regionInfo, Set<HRegionInfo> emptyRegions) {
    this.connection = connection;
    this.tableName = tableName;
    this.regionInfo = regionInfo;
    this.emptyRegions = emptyRegions;
}
 
Example #21
Source File: HBaseUtils.java    From hadoop-arch-book with Apache License 2.0 5 votes vote down vote up
public static void populateUserProfile(HConnection connection, UserProfile userProfile) throws Exception {
  HTableInterface table = connection.getTable(HBaseTableMetaModel.profileCacheTableName);

  try {
    Put put = new Put(convertKeyToRowKey(HBaseTableMetaModel.profileCacheTableName, userProfile.userId));
    put.add(HBaseTableMetaModel.profileCacheColumnFamily, HBaseTableMetaModel.profileCacheJsonColumn, Bytes.toBytes(userProfile.getJSONObject().toString()));
    put.add(HBaseTableMetaModel.profileCacheColumnFamily, HBaseTableMetaModel.profileCacheTsColumn, Bytes.toBytes(System.currentTimeMillis()));
    table.put(put);
  } finally {
    table.close();
  }
}
 
Example #22
Source File: HBaseUtils.java    From hadoop-arch-book with Apache License 2.0 5 votes vote down vote up
public static void populateValidationRules(HConnection connection, ValidationRules rules) throws Exception {
  HTableInterface table = connection.getTable(HBaseTableMetaModel.profileCacheTableName);

  try {
    Put put = new Put(HBaseTableMetaModel.validationRulesRowKey);
    put.add(HBaseTableMetaModel.profileCacheColumnFamily, HBaseTableMetaModel.validationRulesRowKey, Bytes.toBytes(rules.getJSONObject().toString()));
    table.put(put);
  } finally {
    table.close();
  }
}
 
Example #23
Source File: HConnectionController.java    From recsys-offline with Apache License 2.0 5 votes vote down vote up
public  HConnection getHConnection() {
    HConnection con = null;
    try {
        if (null == con || con.isClosed()) {
            con = HConnectionManager.createConnection(config);
        }
    } catch (IOException e) {
        logger.error("Cannot get HBase connection.", e.getCause());
    }
    return con;
}
 
Example #24
Source File: WhoisHBaseAdapter.java    From opensoc-streaming with Apache License 2.0 5 votes vote down vote up
public boolean initializeAdapter() {
	Configuration conf = null;
	conf = HBaseConfiguration.create();
	conf.set("hbase.zookeeper.quorum", _quorum);
	conf.set("hbase.zookeeper.property.clientPort", _port);
	conf.set("zookeeper.session.timeout", "20");
	conf.set("hbase.rpc.timeout", "20");
	conf.set("zookeeper.recovery.retry", "1");
	conf.set("zookeeper.recovery.retry.intervalmill", "1");

	try {

		LOG.trace("[OpenSOC] Connecting to HBase");
		LOG.trace("[OpenSOC] ZOOKEEPER = "
				+ conf.get("hbase.zookeeper.quorum"));

		LOG.trace("[OpenSOC] CONNECTING TO HBASE WITH: " + conf);

		HConnection connection = HConnectionManager.createConnection(conf);

		LOG.trace("[OpenSOC] CONNECTED TO HBASE");

		table = connection.getTable(_table_name);

		LOG.trace("--------CONNECTED TO TABLE: " + table);

		JSONObject tester = enrich("cisco.com");

		if (tester.keySet().size() == 0)
			throw new IOException(
					"Either HBASE is misconfigured or whois table is missing");

		return true;
	} catch (IOException e) {
		e.printStackTrace();
	}

	return false;

}
 
Example #25
Source File: HBaseClientKVIterator.java    From Kylin with Apache License 2.0 5 votes vote down vote up
public HBaseClientKVIterator(HConnection hconn, String tableName, byte[] family, byte[] qualifier) throws IOException {
    this.family = family;
    this.qualifier = qualifier;

    this.table = hconn.getTable(tableName);
    this.scanner = table.getScanner(family, qualifier);
    this.iterator = scanner.iterator();
}
 
Example #26
Source File: GridTableHBaseBenchmark.java    From Kylin with Apache License 2.0 5 votes vote down vote up
private static void createHTableIfNeeded(HConnection conn, String tableName) throws IOException {
    HBaseAdmin hbase = new HBaseAdmin(conn);

    try {
        boolean tableExist = false;
        try {
            hbase.getTableDescriptor(TableName.valueOf(tableName));
            tableExist = true;
        } catch (TableNotFoundException e) {
        }

        if (tableExist) {
            System.out.println("HTable '" + tableName + "' already exists");
            return;
        }

        System.out.println("Creating HTable '" + tableName + "'");

        HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));

        HColumnDescriptor fd = new HColumnDescriptor(CF);
        fd.setBlocksize(CELL_SIZE);
        desc.addFamily(fd);
        hbase.createTable(desc);

        System.out.println("HTable '" + tableName + "' created");
    } finally {
        hbase.close();
    }
}
 
Example #27
Source File: ExportHBaseData.java    From Kylin with Apache License 2.0 5 votes vote down vote up
private void setup() throws IOException {
    long currentTIME = System.currentTimeMillis();
    exportFolder = "/tmp/hbase-export/" + currentTIME + "/";
    backupArchive = "/tmp/kylin_" + currentTIME + ".tar.gz";

    KylinConfig.destoryInstance();
    System.setProperty(KylinConfig.KYLIN_CONF, AbstractKylinTestCase.SANDBOX_TEST_DATA);

    kylinConfig = KylinConfig.getInstanceFromEnv();
    cli = kylinConfig.getCliCommandExecutor();

    String metadataUrl = kylinConfig.getMetadataUrl();
    // split TABLE@HBASE_URL
    int cut = metadataUrl.indexOf('@');
    tableNameBase = metadataUrl.substring(0, cut);
    String hbaseUrl = cut < 0 ? metadataUrl : metadataUrl.substring(cut + 1);

    HConnection conn = HBaseConnection.get(hbaseUrl);
    try {
        hbase = new HBaseAdmin(conn);
        config = hbase.getConfiguration();
        allTables = hbase.listTables();
    } catch (IOException e) {
        e.printStackTrace();
        throw e;
    }
}
 
Example #28
Source File: InvertedIndexStorageEngine.java    From Kylin with Apache License 2.0 5 votes vote down vote up
@Override
public ITupleIterator search(StorageContext context, SQLDigest sqlDigest) {
    String tableName = seg.getStorageLocationIdentifier();

    //HConnection is cached, so need not be closed
    HConnection conn = HBaseConnection.get(context.getConnUrl());
    try {
        return new EndpointTupleIterator(seg, sqlDigest.filter, sqlDigest.groupbyColumns, new ArrayList<>(sqlDigest.aggregations), context, conn);
    } catch (Throwable e) {
        e.printStackTrace();
        throw new IllegalStateException("Error when connecting to II htable " + tableName, e);
    }
}
 
Example #29
Source File: HBaseConnection.java    From Kylin with Apache License 2.0 5 votes vote down vote up
@Override
public void run() {
    for (HConnection conn : ConnPool.values()) {
        try {
            conn.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}
 
Example #30
Source File: HBaseConnection.java    From Kylin with Apache License 2.0 5 votes vote down vote up
public static void createHTableIfNeeded(HConnection conn, String tableName, String... families) throws IOException {
    HBaseAdmin hbase = new HBaseAdmin(conn);

    try {
        boolean tableExist = false;
        try {
            hbase.getTableDescriptor(TableName.valueOf(tableName));
            tableExist = true;
        } catch (TableNotFoundException e) {
        }

        if (tableExist) {
            logger.debug("HTable '" + tableName + "' already exists");
            return;
        }

        logger.debug("Creating HTable '" + tableName + "'");

        HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));

        if (null != families && families.length > 0) {
            for (String family : families) {
                HColumnDescriptor fd = new HColumnDescriptor(family);
                fd.setInMemory(true); // metadata tables are best in memory
                desc.addFamily(fd);
            }
        }
        hbase.createTable(desc);

        logger.debug("HTable '" + tableName + "' created");
    } finally {
        hbase.close();
    }
}