com.clearspring.analytics.util.Lists Java Examples
The following examples show how to use
com.clearspring.analytics.util.Lists.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ReplicationSystemProcedure.java From spliceengine with GNU Affero General Public License v3.0 | 6 votes |
private static List<String> getTablesFromSchema(String schemaName) throws SQLException{ List<String> tables = Lists.newArrayList(); String sql = "select tablename from sys.systables t, sys.sysschemas s " + "where t.schemaid=s.schemaid and s.schemaname=?"; Connection conn = SpliceAdmin.getDefaultConn(); try(PreparedStatement ps = conn.prepareStatement(sql)) { ps.setString(1, schemaName); try(ResultSet rs = ps.executeQuery()) { while (rs.next()) { tables.add(rs.getString(1)); } return tables; } } }
Example #2
Source File: AfterMergeOrRefreshResourceMerger.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
List<CubeSegment> getToRemoveSegs(CubeInstance cube, CubeSegment mergedSegment) { Segments tobe = cube.calculateToBeSegments(mergedSegment); if (!tobe.contains(mergedSegment)) throw new IllegalStateException( "For Cube " + cube + ", segment " + mergedSegment + " is expected but not in the tobe " + tobe); if (mergedSegment.getStatus() == SegmentStatusEnum.NEW) mergedSegment.setStatus(SegmentStatusEnum.READY); List<CubeSegment> toRemoveSegs = Lists.newArrayList(); for (CubeSegment s : cube.getSegments()) { if (!tobe.contains(s)) toRemoveSegs.add(s); } return toRemoveSegs; }
Example #3
Source File: ReplicationSystemProcedure.java From spliceengine with GNU Affero General Public License v3.0 | 6 votes |
public static void GET_REPLICATED_WAL_POSITIONS(short peerId, ResultSet[] resultSets) throws StandardException, SQLException { ReplicationManager replicationManager = EngineDriver.driver().manager().getReplicationManager(); List<String> walPositions = replicationManager.getReplicatedWalPositions(peerId); Connection conn = SpliceAdmin.getDefaultConn(); LanguageConnectionContext lcc = conn.unwrap(EmbedConnection.class).getLanguageConnection(); ResultColumnDescriptor[] rcds = { new GenericColumnDescriptor("walPosition", DataTypeDescriptor.getBuiltInDataTypeDescriptor(Types.VARCHAR)) }; List<ExecRow> rows = Lists.newArrayList(); for (String walPosition : walPositions) { ExecRow template = new ValueRow(1); template.setRowArray(new DataValueDescriptor[]{new SQLVarchar()}); template.getColumn(1).setValue(walPosition); rows.add(template); } IteratorNoPutResultSet inprs = new IteratorNoPutResultSet(rows, rcds, lcc.getLastActivation()); inprs.openCore(); resultSets[0] = new EmbedResultSet40(conn.unwrap(EmbedConnection.class), inprs, false, null, true); }
Example #4
Source File: CsvSourceTest.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
@Test public void testGetSourceDataFromFactTable() { CubeManager cubeMgr = CubeManager.getInstance(getTestConfig()); CubeInstance cube = cubeMgr.getCube(CUBE_NAME); TableDesc fact = MetadataConverter.extractFactTable(cube); List<ColumnDesc> colDescs = Lists.newArrayList(); Iterator<ColumnDesc> iterator = fact.columns().iterator(); while (iterator.hasNext()) { colDescs.add(iterator.next()); } NSparkCubingEngine.NSparkCubingSource cubingSource = new CsvSource().adaptToBuildEngine(NSparkCubingEngine.NSparkCubingSource.class); Dataset<Row> cubeDS = cubingSource.getSourceData(fact, ss, Maps.newHashMap()); cubeDS.take(10); StructType schema = cubeDS.schema(); for (int i = 0; i < colDescs.size(); i++) { StructField field = schema.fields()[i]; Assert.assertEquals(field.name(), colDescs.get(i).columnName()); Assert.assertEquals(field.dataType(), colDescs.get(i).dataType()); } }
Example #5
Source File: ReplicationSystemProcedure.java From spliceengine with GNU Affero General Public License v3.0 | 6 votes |
public static void GET_REPLICATED_WAL_POSITION(String wal, ResultSet[] resultSets) throws StandardException, SQLException { ReplicationManager replicationManager = EngineDriver.driver().manager().getReplicationManager(); String walPosition = replicationManager.getReplicatedWalPosition(wal); Connection conn = SpliceAdmin.getDefaultConn(); LanguageConnectionContext lcc = conn.unwrap(EmbedConnection.class).getLanguageConnection(); ResultColumnDescriptor[] rcds = { new GenericColumnDescriptor("walPosition", DataTypeDescriptor.getBuiltInDataTypeDescriptor(Types.VARCHAR)) }; List<ExecRow> rows = Lists.newArrayList(); ExecRow template = new ValueRow(1); template.setRowArray(new DataValueDescriptor[]{new SQLVarchar()}); template.getColumn(1).setValue(walPosition); rows.add(template); IteratorNoPutResultSet inprs = new IteratorNoPutResultSet(rows, rcds, lcc.getLastActivation()); inprs.openCore(); resultSets[0] = new EmbedResultSet40(conn.unwrap(EmbedConnection.class), inprs, false, null, true); }
Example #6
Source File: TestDateTimeUtils.java From envelope with Apache License 2.0 | 6 votes |
@Test public void testParserFormats() { DateTimeParser parser = new DateTimeParser(); parser.configureFormat(Lists.newArrayList( Arrays.asList("yyyy-MM-dd HH:mm:ss.SSSSS", "yyyy-MM-dd", "yyyy-MM-dd HH:mm:ss", "dd"))); assertEquals(new LocalDateTime(parser.parse("2018-09-19 23:49:29.92284")). toDateTime(DateTimeZone.UTC).toString(), "2018-09-19T23:49:29.922Z"); assertEquals(new LocalDateTime(parser.parse("2018-09-19 23:49:29")). toDateTime(DateTimeZone.UTC).toString(), "2018-09-19T23:49:29.000Z"); assertEquals(new LocalDateTime(parser.parse("2018-07-25")). toDateTime(DateTimeZone.UTC).toString(), "2018-07-25T00:00:00.000Z"); assertEquals(new LocalDateTime(parser.parse("08")). toDateTime(DateTimeZone.UTC).toString(), "2000-01-08T00:00:00.000Z"); }
Example #7
Source File: ReplicationSystemProcedure.java From spliceengine with GNU Affero General Public License v3.0 | 6 votes |
private static List<Pair<String, String>> getTablesFromDatabase() throws SQLException{ List<Pair<String, String>> tables = Lists.newArrayList(); String sql = "select schemaname, tablename from sys.systables t, sys.sysschemas s " + "where t.schemaid=s.schemaid"; Connection conn = SpliceAdmin.getDefaultConn(); try(PreparedStatement ps = conn.prepareStatement(sql)) { try(ResultSet rs = ps.executeQuery()) { while (rs.next()) { tables.add(new Pair(rs.getString(1), rs.getString(2))); } return tables; } } }
Example #8
Source File: CoreClusterManagerIT.java From heroic with Apache License 2.0 | 6 votes |
private List<AsyncFuture<Void>> pingAllNodesInShard( ClusterManager clusterManager, ClusterShard shard ) { final List<AsyncFuture<Void>> futures = Lists.newArrayList(); final List<ClusterNode> excludeIds = Lists.newArrayList(); while (true) { Optional<ClusterManager.NodeResult<AsyncFuture<Void>>> ret = clusterManager.withNodeInShardButNotWithId(shard.getShard(), excludeIds::contains, excludeIds::add, ClusterNode.Group::ping); if (!ret.isPresent()) { // No more nodes available in shard, we're done return futures; } ClusterManager.NodeResult<AsyncFuture<Void>> result = ret.get(); futures.add(result.getReturnValue()); } }
Example #9
Source File: DataStoreUtils.java From geowave with Apache License 2.0 | 6 votes |
public static List<Index> loadIndices(final DataStore dataStore, final String indexNames) { List<Index> loadedIndices = Lists.newArrayList(); // Is there a comma? final String[] indices = indexNames.split(","); final Index[] dataStoreIndices = dataStore.getIndices(); for (final String idxName : indices) { boolean found = false; for (Index index : dataStoreIndices) { if (index.getName().equals(idxName)) { loadedIndices.add(index); found = true; break; } } if (!found) { throw new ParameterException("Unable to find index with name: " + idxName); } } return Collections.unmodifiableList(loadedIndices); }
Example #10
Source File: HFileGenerationFunction.java From spliceengine with GNU Affero General Public License v3.0 | 6 votes |
public HFileGenerationFunction(OperationContext operationContext, long txnId, Long heapConglom, String compressionAlgorithm, List<BulkImportPartition> partitionList, String tableVersion, DDLMessage.TentativeIndex tentativeIndex) { this.txnId = txnId; this.operationContext = operationContext; this.heapConglom = heapConglom; this.compressionAlgorithm = compressionAlgorithm; this.partitionList = partitionList; this.tableVersion = tableVersion; this.tentativeIndexList = Lists.newArrayList(); tentativeIndexList.add(tentativeIndex); }
Example #11
Source File: HFileGenerationFunction.java From spliceengine with GNU Affero General Public License v3.0 | 5 votes |
@Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { if(in.readBoolean()) { operationContext = (OperationContext) in.readObject(); } txnId = in.readLong(); heapConglom = in.readLong(); compressionAlgorithm = in.readUTF(); int n = in.readInt(); partitionList = new ArrayList<>(n); for (int i = 0; i < n; ++i) { BulkImportPartition partition = (BulkImportPartition) in.readObject(); partitionList.add(partition); } if (in.readBoolean()) { n = in.readInt(); pkCols = new int[n]; for (int i = 0; i < n; ++i) { pkCols[i] = in.readInt(); } } if (in.readBoolean()) { tableVersion = in.readUTF(); } operationType = OperationType.values()[in.readInt()]; if (in.readBoolean()) { tentativeIndexList = Lists.newArrayList(); n = in.readInt(); for (int i = 0; i < n; ++i) { byte[] message = new byte[in.readInt()]; in.readFully(message); tentativeIndexList.add(DDLMessage.TentativeIndex.parseFrom(message)); } } }
Example #12
Source File: ReplicationSystemProcedure.java From spliceengine with GNU Affero General Public License v3.0 | 5 votes |
private static List<TableDescriptor> getTableDescriptorsFromDatabase() throws Exception{ List<Pair<String, String>> tables = getTablesFromDatabase(); List<TableDescriptor> tableDescriptors = Lists.newArrayList(); for (Pair<String, String> table : tables) { TableDescriptor td = SpliceRegionAdmin.getTableDescriptor(table.getFirst(), table.getSecond()); tableDescriptors.add(td); } return tableDescriptors; }
Example #13
Source File: ReplicationSystemProcedure.java From spliceengine with GNU Affero General Public License v3.0 | 5 votes |
private static List<TableDescriptor> getTableDescriptorsFromSchema(String schemaName) throws Exception{ List<String> tables = getTablesFromSchema(schemaName); List<TableDescriptor> tableDescriptors = Lists.newArrayList(); for (String table : tables) { TableDescriptor td = SpliceRegionAdmin.getTableDescriptor(schemaName, table); tableDescriptors.add(td); } return tableDescriptors; }
Example #14
Source File: BackupRegionStatus.java From spliceengine with GNU Affero General Public License v3.0 | 5 votes |
@Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { startKey = ArrayUtil.readByteArray(in); endKey = ArrayUtil.readByteArray(in); status = ArrayUtil.readByteArray(in); int n = in.readInt(); backupFiles = Lists.newArrayList(); for (int i = 0; i < n; ++i) { backupFiles.add(in.readUTF()); } }
Example #15
Source File: SparkUHCDictionary.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
@Override public Tuple2<Integer, List<String>> call(String sequenceFilePath) throws Exception { Path path = new Path(sequenceFilePath); logger.info("Column absolute path is " + path.toString()); if (!HadoopUtil.getFileSystem(path).exists(path)) { return new Tuple2<>(-1, null); } String columnName = path.getParent().getName(); int index = -1; for (int i = 0;i < uhcColumns.size(); i++) { if (uhcColumns.get(i).getIdentity().equalsIgnoreCase(columnName)) { index = i; break; } } if (index == -1) { return new Tuple2<>(-1, null); } try (KylinConfig.SetAndUnsetThreadLocalConfig autoUnset = KylinConfig .setAndUnsetThreadLocalConfig(config)) { List<String> values = Lists.newArrayList(); values.addAll(HadoopUtil.readDistinctColumnValues(sequenceFilePath)); logger.info("UHC column " + columnName + " contains distinct values " + values); return new Tuple2<>(index, values); } }
Example #16
Source File: BulkImportFunction.java From spliceengine with GNU Affero General Public License v3.0 | 5 votes |
private void init(Iterator<BulkImportPartition> importPartitions) throws Exception { partitionMap = new HashMap<>(); while (importPartitions.hasNext()) { BulkImportPartition partition = importPartitions.next(); Long conglom = partition.getConglomerateId(); List<BulkImportPartition> partitionList = partitionMap.get(conglom); if (partitionList == null) { partitionList = Lists.newArrayList(); partitionMap.put(conglom, partitionList); } partitionList.add(partition); } }
Example #17
Source File: SpliceDefaultCompactor.java From spliceengine with GNU Affero General Public License v3.0 | 5 votes |
public List<StoreFileScanner> createFileScanners( final Collection<StoreFile> filesToCompact, long smallestReadPoint) throws IOException { List<HStoreFile> hStoreFileList = Lists.newArrayList(); for (StoreFile storeFile : filesToCompact) { hStoreFileList.add((HStoreFile) storeFile); } return StoreFileScanner.getScannersForStoreFiles(hStoreFileList, /* cache blocks = */ false, /* use pread = */ false, /* is compaction */ true, /* use Drop Behind */ false, smallestReadPoint); }
Example #18
Source File: BulkLoadIndexDataSetWriter.java From spliceengine with GNU Affero General Public License v3.0 | 5 votes |
@Override public DataSet<ExecRow> write() throws StandardException { List<Long> allCongloms = Lists.newArrayList(); allCongloms.add(heapConglom); if (sampling) { sampleAndSplitIndex(); } final List<BulkImportPartition> bulkLoadPartitions = getBulkImportPartitions(allCongloms, bulkLoadDirectory); String compressionAlgorithm = HConfiguration.getConfiguration().getCompressionAlgorithm(); // Write to HFile HFileGenerationFunction hfileGenerationFunction = new BulkLoadIndexHFileGenerationFunction(operationContext, txn.getTxnId(), heapConglom, compressionAlgorithm, bulkLoadPartitions, tableVersion, tentativeIndex); DataSet rowAndIndexes = dataSet .map(new IndexTransformFunction(tentativeIndex), null, false, true, String.format("Create Index %s: Generate HFiles", indexName)) .mapPartitions(new BulkLoadKVPairFunction(heapConglom), false, true, String.format("Create Index %s: Generate HFiles", indexName)); partitionUsingRDDSortUsingDataFrame(bulkLoadPartitions, rowAndIndexes, hfileGenerationFunction); bulkLoad(bulkLoadPartitions, bulkLoadDirectory, String.format("Create Index %s:", indexName)); ValueRow valueRow=new ValueRow(1); valueRow.setColumn(1,new SQLLongint(operationContext.getRecordsWritten())); return new SparkDataSet<>(SpliceSpark.getContext().parallelize(Collections.singletonList(valueRow), 1)); }
Example #19
Source File: BulkLoadIndexDataSetWriter.java From spliceengine with GNU Affero General Public License v3.0 | 5 votes |
private void sampleAndSplitIndex() throws StandardException { Activation activation = operationContext.getActivation(); LanguageConnectionContext lcc = activation.getLanguageConnectionContext(); double sampleFraction = BulkLoadUtils.getSampleFraction(lcc); DataSet sampledDataSet = dataSet.sampleWithoutReplacement(sampleFraction); DataSet sampleRowAndIndexes = sampledDataSet .map(new IndexTransformFunction(tentativeIndex), null, false, true, String.format("Create Index %s: Sample Data", indexName)) .mapPartitions(new BulkLoadKVPairFunction(heapConglom), false, true, String.format("Create Index %s: Sample Data", indexName)); // collect statistics for encoded key/value, include size and histgram RowKeyStatisticsFunction statisticsFunction = new RowKeyStatisticsFunction(heapConglom, Lists.newArrayList()); DataSet keyStatistics = sampleRowAndIndexes.mapPartitions(statisticsFunction); List<Tuple2<Long, Tuple2<Double, ColumnStatisticsImpl>>> result = keyStatistics.collect(); // Calculate cut points for main table and index tables List<Tuple2<Long, byte[][]>> cutPoints = BulkLoadUtils.getCutPoints(sampleFraction, result); // dump cut points to file system for reference ImportUtils.dumpCutPoints(cutPoints, bulkLoadDirectory); if (cutPoints != null && cutPoints.size() > 0) { BulkLoadUtils.splitTables(cutPoints); } }
Example #20
Source File: SparkTableSampler.java From spliceengine with GNU Affero General Public License v3.0 | 5 votes |
@Override public byte[][] sample() throws StandardException { if (sampleFraction == 0) { Activation activation = operationContext.getActivation(); LanguageConnectionContext lcc = activation.getLanguageConnectionContext(); sampleFraction = BulkLoadUtils.getSampleFraction(lcc); } DataSet sampledDataSet = dataSet.sampleWithoutReplacement(sampleFraction); DataSet sampleRowAndIndexes = sampledDataSet .map(new IndexTransformFunction(tentativeIndex), null, false, true, String.format("Create Index %s: Sample Data", indexName)) .mapPartitions(new BulkLoadKVPairFunction(-1), false, true, String.format("Create Index %s: Sample Data", indexName)); // collect statistics for encoded key/value, include size and histgram RowKeyStatisticsFunction statisticsFunction = new RowKeyStatisticsFunction(-1, Lists.newArrayList()); DataSet keyStatistics = sampleRowAndIndexes.mapPartitions(statisticsFunction); List<Tuple2<Long, Tuple2<Double, ColumnStatisticsImpl>>> result = keyStatistics.collect(); // Calculate cut points for main table and index tables List<Tuple2<Long, byte[][]>> cutPoints = BulkLoadUtils.getCutPoints(sampleFraction, result); if (cutPoints.size() == 1) return cutPoints.get(0)._2; else if (cutPoints.size() == 0) return null; else throw new RuntimeException("Unexpected cutpoints"); }
Example #21
Source File: ReplicationSystemProcedure.java From spliceengine with GNU Affero General Public License v3.0 | 5 votes |
private static List<String> getSchemas() throws SQLException { List<String> schemas = Lists.newArrayList(); String sql = "select schemaname from sys.sysschemas"; Connection conn = SpliceAdmin.getDefaultConn(); try (PreparedStatement ps = conn.prepareStatement(sql); ResultSet rs = ps.executeQuery()) { while (rs.next()) { schemas.add(rs.getString(1)); } return schemas; } }
Example #22
Source File: ReplicationSystemProcedure.java From spliceengine with GNU Affero General Public License v3.0 | 5 votes |
public static void LIST_PEERS(ResultSet[] resultSets) throws StandardException, SQLException { ReplicationManager replicationManager = EngineDriver.driver().manager().getReplicationManager(); List<ReplicationPeerDescription> replicationPeers = replicationManager.getReplicationPeers(); Connection conn = SpliceAdmin.getDefaultConn(); LanguageConnectionContext lcc = conn.unwrap(EmbedConnection.class).getLanguageConnection(); ResultColumnDescriptor[] rcds = { new GenericColumnDescriptor("PeerId", DataTypeDescriptor.getBuiltInDataTypeDescriptor(Types.VARCHAR)), new GenericColumnDescriptor("clusterKey", DataTypeDescriptor.getBuiltInDataTypeDescriptor(Types.VARCHAR)), new GenericColumnDescriptor("enabled", DataTypeDescriptor.getBuiltInDataTypeDescriptor(Types.BOOLEAN)), new GenericColumnDescriptor("serial", DataTypeDescriptor.getBuiltInDataTypeDescriptor(Types.BOOLEAN)) }; List<ExecRow> rows = Lists.newArrayList(); for (ReplicationPeerDescription replicationPeer : replicationPeers) { ExecRow template = new ValueRow(4); template.setRowArray(new DataValueDescriptor[]{new SQLVarchar(), new SQLVarchar(), new SQLBoolean(), new SQLBoolean()}); template.getColumn(1).setValue(replicationPeer.getId()); template.getColumn(2).setValue(replicationPeer.getClusterKey()); template.getColumn(3).setValue(replicationPeer.isEnabled()); template.getColumn(4).setValue(replicationPeer.isSerial()); rows.add(template); } IteratorNoPutResultSet inprs = new IteratorNoPutResultSet(rows, rcds, lcc.getLastActivation()); inprs.openCore(); resultSets[0] = new EmbedResultSet40(conn.unwrap(EmbedConnection.class), inprs, false, null, true); }
Example #23
Source File: SpliceTableAdmin.java From spliceengine with GNU Affero General Public License v3.0 | 5 votes |
private static List<String> getTables(String schemaName) throws SQLException { String sql = "select tablename from sys.systables t, sys.sysschemas s where s.schemaname=?" + " and s.schemaid=t.schemaid"; Connection connection = SpliceAdmin.getDefaultConn(); List<String> tables = Lists.newArrayList(); try(PreparedStatement ps = connection.prepareStatement(sql)) { ps.setString(1, schemaName); try(ResultSet rs = ps.executeQuery()) { while(rs.next()) { tables.add(rs.getString(1)); } } return tables; } }
Example #24
Source File: SpliceTableAdmin.java From spliceengine with GNU Affero General Public License v3.0 | 5 votes |
private static Map<String, List<String>> checkIndexesFast(String schema, String table, ConglomerateDescriptorList cdList, List<TentativeIndex> tentativeIndexList) throws IOException, SQLException { boolean consistent = true; Map<String, List<String>> errors = new HashMap<>(); long tableCount = countTable(schema, table, null); List<String> message = Lists.newArrayList(); message.add("count = " + tableCount); errors.put(table, message); for (TentativeIndex tentativeIndex:tentativeIndexList) { String indexName = getIndexName(cdList, tentativeIndex.getIndex().getConglomerate()); DDLMessage.Index index = tentativeIndex.getIndex(); message = Lists.newArrayList(); if (index.getExcludeDefaults() || index.getExcludeNulls()) { consistent = false; message.add("Index count and base table count are not expected to match because index excludes null or " + "default keys. The index should be checked at level 2"); } else { long indexCount = countTable(schema, table, indexName); if (indexCount != tableCount) { consistent = false; } message.add("count = " + indexCount); } errors.put(indexName, message); } if (consistent){ errors.clear(); } return errors; }
Example #25
Source File: MemstoreAwareObserverTest.java From spliceengine with GNU Affero General Public License v3.0 | 5 votes |
@Test public void compactionEarlyFailureWorks() throws Exception { SpliceCompactionRequest compactionRequest = new SpliceCompactionRequest(Lists.newArrayList()); // there's been an exception, preCompact() and preStorefilesRename() are not called // afterExecute() won't do anything compactionRequest.afterExecute(); }
Example #26
Source File: MergeNonDistinctAggregatesFunctionForMixedRows.java From spliceengine with GNU Affero General Public License v3.0 | 5 votes |
private void setup() { /** * With the rows split, the column positions recorded in aggregates are no longer valid, * so for multiple distinct aggregate case, we need to compose a new aggregates array with * column ids pointing to the new position in the split row. */ GenericAggregateOperation op = (GenericAggregateOperation) operationContext.getOperation(); SpliceGenericAggregator[] origAggregates = op.aggregates; int numOfGroupKeys = groupingKeys == null? 0 : groupingKeys.length; List<SpliceGenericAggregator> tmpAggregators = Lists.newArrayList(); int numOfNonDistinctAggregates = 0; ClassFactory cf = op.getActivation().getLanguageConnectionContext().getLanguageConnectionFactory().getClassFactory(); for (SpliceGenericAggregator aggregator : origAggregates) { AggregatorInfo aggInfo = aggregator.getAggregatorInfo(); if (aggregator.isDistinct()) continue; AggregatorInfo newAggInfo = new AggregatorInfo(aggInfo.getAggregateName() , aggInfo.getAggregatorClassName() , numOfGroupKeys + numOfNonDistinctAggregates * 3 + 2 , numOfGroupKeys + numOfNonDistinctAggregates * 3 + 1 , numOfGroupKeys + numOfNonDistinctAggregates * 3 + 3 , false , aggInfo.getResultDescription()); numOfNonDistinctAggregates++; tmpAggregators.add(new SpliceGenericAggregator(newAggInfo, cf)); } if (tmpAggregators.size() > 0) { nonDistinctAggregates = new SpliceGenericAggregator[tmpAggregators.size()]; tmpAggregators.toArray(nonDistinctAggregates); } distinctColumnId = numOfGroupKeys + 1; return; }
Example #27
Source File: DataStoreUtils.java From geowave with Apache License 2.0 | 5 votes |
public static List<Index> loadIndices(final IndexStore indexStore, final String indexNames) { List<Index> loadedIndices = Lists.newArrayList(); // Is there a comma? final String[] indices = indexNames.split(","); for (final String idxName : indices) { Index index = indexStore.getIndex(idxName); if (index == null) { throw new ParameterException("Unable to find index with name: " + idxName); } loadedIndices.add(index); } return Collections.unmodifiableList(loadedIndices); }
Example #28
Source File: SparkUHCDictionary.java From kylin with Apache License 2.0 | 5 votes |
@Override public Tuple2<Integer, List<String>> call(String sequenceFilePath) throws Exception { Path path = new Path(sequenceFilePath); logger.info("Column absolute path is " + path.toString()); if (!HadoopUtil.getFileSystem(path).exists(path)) { return new Tuple2<>(-1, null); } String columnName = path.getParent().getName(); int index = -1; for (int i = 0;i < uhcColumns.size(); i++) { if (uhcColumns.get(i).getIdentity().equalsIgnoreCase(columnName)) { index = i; break; } } if (index == -1) { return new Tuple2<>(-1, null); } try (KylinConfig.SetAndUnsetThreadLocalConfig autoUnset = KylinConfig .setAndUnsetThreadLocalConfig(config)) { List<String> values = Lists.newArrayList(); values.addAll(HadoopUtil.readDistinctColumnValues(sequenceFilePath)); logger.info("UHC column " + columnName + " contains distinct values " + values); return new Tuple2<>(index, values); } }
Example #29
Source File: TestDateTimeUtils.java From envelope with Apache License 2.0 | 5 votes |
@Test(expected = IllegalArgumentException.class) public void testParserInvalidInput() { DateTimeParser parser = new DateTimeParser(); parser.configureFormat(Lists.newArrayList( Arrays.asList("yyyy-MM-dd HH:mm:ss.SSSSS", "yyyy-MM-dd", "yyyy-MM-dd HH:mm:ss", "dd"))); parser.parse("09-19 23:29.92284"); }
Example #30
Source File: TestDateTimeUtils.java From envelope with Apache License 2.0 | 5 votes |
@Test public void testParserOneFormat() { DateTimeParser parser = new DateTimeParser(); parser.configureFormat(Lists.newArrayList( Arrays.asList("yyyy-MM-dd HH:mm:ss.SSSSS"))); assertEquals(new LocalDateTime(parser.parse("2018-09-19 23:49:29.92284")). toDateTime(DateTimeZone.UTC).toString(), "2018-09-19T23:49:29.922Z"); }