Intel-bigdata/OAP

OAP-master
- .github
  - PULL_REQUEST_TEMPLATE
- oap-native-sql
  - cpp
    - src
      - proto
        protobuf_utils.h
        Exprs.proto
        protobuf_utils.cc
      - utils
        macros.h
      - third_party
        arrow
        utils
        hashing.h
        vendored
        xxhash
        xxhash.h
        xxhash.c
        README.md
        xxh3.h
      - jni
        jni_wrapper.cc
        concurrent_map.h
        jni_common.h
      - data_source
        parquet
        adapter.cc
        adapter.h
      - benchmarks
        arrow_compute_benchmark_big_scale.cc
        arrow_compute_benchmark_join.cc
        arrow_compute_benchmark.cc
        CMakeLists.txt
        arrow_compute_benchmark_sort.cc
      - CMakeLists.txt
      - codegen
        code_generator_factory.h
        arrow_compute
        expr_visitor_impl.h
        expr_visitor.h
        code_generator.h
        ext
        conditioner.h
        sort_kernel.cc
        codegen_node_visitor.cc
        conditioned_shuffle_kernel.cc
        shuffle_v2_action.h
        actions_impl.h
        codegen_node_visitor_v2.cc
        kernels_ext.cc
        item_iterator.h
        codegen_common.h
        codegen_node_visitor_v2.h
        kernels_ext.h
        shuffle_v2_action.cc
        codegen_common.cc
        codegen_node_visitor.h
        conditioned_probe_kernel.cc
        array_item_index.h
        code_generator_base.h
        item_iterator.cc
        expr_visitor.cc
        expr_visitor.h
        common
        visitor_base.h
        result_iterator.h
        compute_ext
        code_generator.h
        code_generator.h
        gandiva
        code_generator.h
        expr_visitor.cc
      - tests
        arrow_compute_test_sort.cc
        arrow_compute_test_aggregate.cc
        test_utils.h
        arrow_compute_test_join.cc
        CMakeLists.txt
    - CMakeLists.txt
  - patch
    - pom.xml
    - arrow.patch
    - spark.patch
  - resource
    - ApacheArrowInstallation.md
    - SparkInstallation.md
    - InstallationNotes.md
  - pom.xml
  - LICENSE
  - README.md
  - core
    - src
      - main
        scala
        com
        intel
        sparkColumnarPlugin
        ColumnarPluginConfig.scala
        vectorized
        CloseableColumnBatchIterator.scala
        ArrowColumnarBatchSerializer.scala
        execution
        RowToArrowColumnarExec.scala
        ColumnarDataSourceRDD.scala
        ColumnarBatchScanExec.scala
        ColumnarHashAggregateExec.scala
        ColumnarShuffledHashJoinExec.scala
        ColumnarBasicPhysicalOperators.scala
        ColumnarSortExec.scala
        ColumnarPlugin.scala
        expression
        ColumnarLiterals.scala
        ColumnarSorter.scala
        ColumnarTernaryOperator.scala
        ColumnarShuffledHashJoin.scala
        ColumnarSubquery.scala
        ConverterUtils.scala
        ColumnarConditionProjector.scala
        ColumnarAggregateExpression.scala
        ColumnarInSetOperator.scala
        ColumnarIfOperator.scala
        ColumnarBoundAttribute.scala
        CodeGeneration.scala
        ColumnarNamedExpressions.scala
        ColumnarUnaryOperator.scala
        ColumnarInOperator.scala
        ColumnarExpressionConverter.scala
        ColumnarFilter.scala
        ColumnarAggregation.scala
        ColumnarCoalesceOperator.scala
        ColumnarCaseWhenOperator.scala
        ColumnarProjection.scala
        ColumnarExpression.scala
        ColumnarBinaryOperator.scala
        ColumnarArithmetic.scala
        org
        apache
        spark
        sql
        util
        ArrowUtils.scala
        execution
        ColumnarShuffleExchangeExec.scala
        datasources
        v2
        VectorizedFilePartitionReaderHandler.scala
        ShuffledColumnarBatchRDD.scala
        java
        test
        com
        intel
        sparkColumnarPlugin
        vectorized
        ExpressionEvaluatorJniWrapper.java
        JniUtils.java
        ArrowRecordBatchBuilder.java
        ArrowWritableColumnVector.java
        ArrowRecordBatchBuilderImpl.java
        ArrowBufBuilder.java
        BatchIterator.java
        ExpressionEvaluator.java
        AdaptorReferenceManager.java
        ArrowFieldNodeBuilder.java
        datasource
        VectorizedParquetArrowReader.java
        parquet
        ParquetWriterJniWrapper.java
        ParquetReaderJniWrapper.java
        ParquetWriter.java
        ParquetReader.java
        expression
        ArrowColumnarBatch.java
        ColumnarArithmeticWithGandiva.java
      - test
        scala
        com.intel.sparkColumnarPlugin
        ExtensionSuite.scala
        java
        com
        intel
        sparkColumnarPlugin
        datasource
        parquet
        part-00000-d648dd34-c9d2-4fe9-87f2-770ef3551442-c000.snappy.parquet
        ParquetReadWriteTest.java
        ParquetReadTest.java
    - pom.xml
- oap-cache
  - oap
    - src
      - main
        parquet1.10.1
        java
        org
        apache
        parquet
        hadoop
        ParquetFileReader.java
        resources
        oap
        static
        oap.js
        oap-template.html
        META-INF
        services
        org.apache.spark.sql.sources.DataSourceRegister
        spark3.0.0
        scala
        org
        apache
        spark
        sql
        execution
        datasources
        FileFormatWriter.scala
        OutputWriter.scala
        FileFormatDataWriter.scala
        DataSourceScanExec.scala
        scheduler
        DAGScheduler.scala
        status
        api
        v1
        OneApplicationResource.scala
        java
        org
        apache
        spark
        sql
        execution
        vectorized
        OnHeapColumnVector.java
        datasources
        parquet
        VectorizedColumnReader.java
        VectorizedRleValuesReader.java
        VectorizedPlainValuesReader.java
        antlr4
        org
        apache
        spark
        sql
        catalyst
        parser
        OapSqlBase.g4
        scala
        org
        apache
        spark
        sql
        OapExtensions.scala
        oap
        OapRuntime.scala
        ui
        OapTab.scala
        FiberCacheManagerPage.scala
        adapter
        TaskContextImplAdapter.scala
        InputFileNameHolderAdapter.scala
        AggregateFunctionAdapter.scala
        LogicalPlanAdapter.scala
        RpcEndpointRefAdapter.scala
        SqlConfAdapter.scala
        CapacityAdapter.scala
        FileIndexAdapter.scala
        ColumnVectorAdapter.scala
        FileSourceScanExecAdapter.scala
        rpc
        OapMessages.scala
        OapRpcManager.scala
        OapRpcManagerSlave.scala
        OapRpcManagerMaster.scala
        listener
        OapListener.scala
        internal
        oap
        OapConf.scala
        execution
        aggregate
        OapAggregationIterator.scala
        OapAggregateExec.scala
        OapAggUtils.scala
        datasources
        orc
        ReadOnlyNativeOrcFileFormat.scala
        package.scala
        oap
        statistics
        SampleBasedStatistics.scala
        MinMaxStatistics.scala
        BloomFilterStatistics.scala
        StatisticsManager.scala
        StatisticsType.scala
        PartByValueStatistics.scala
        Statistics.scala
        utils
        CaseInsensitiveMap.scala
        BTreeUtils.scala
        PersistentMemoryConfigUtils.scala
        NonNullKey.scala
        OapBitmapWrappedFiberCache.scala
        BitmapUtils.scala
        SerDe.scala
        FilterHelper.scala
        CacheStatusSerDe.scala
        OapUtils.scala
        index
        impl
        IndexFileReaderImpl.scala
        IndexFileWriterImpl.scala
        BTreeIndexRecordWriterV2.scala
        DummyIndexRecordWriter.scala
        IndexContext.scala
        IndexUtils.scala
        UnsafeIndexTree.scala
        BitmapReaderV1.scala
        BTreeIndexRecordWriterV1.scala
        indexPlans.scala
        BTreeIndexRecordReaderV2.scala
        BTreeIndexRecordReaderV1.scala
        IndexFileWriter.scala
        IndexColumn.scala
        BitmapReader.scala
        OapIndexOutputFormat.scala
        OapIndexFileFormat.scala
        BTreeIndexRecordReader.scala
        IndexScanner.scala
        OapIndexCommitProtocol.scala
        BitmapIndexRecordWriter.scala
        BTreeIndexRecordWriter.scala
        OapIndexOutputWriter.scala
        OapIndexProperties.scala
        OapIndexType.scala
        BitMapScanner.scala
        BloomFilter.scala
        IndexFileReader.scala
        BitmapReaderV2.scala
        BPlusTreeScanner.scala
        OapMetricsManager.scala
        package.scala
        OptimizedOrcFileFormat.scala
        OapFileFormat.scala
        DataSourceMeta.scala
        filecache
        FiberCache.scala
        CacheMemoryAllocator.scala
        FiberId.scala
        FiberBuilder.scala
        FiberSensor.scala
        MemoryManager.scala
        OapCache.scala
        CacheStats.scala
        FiberCacheManager.scala
        io
        IndexedOrcCacheReader.java
        OapDataReader.scala
        CodecFactory.scala
        OrcDataFileMeta.scala
        ParquetDataFiberCompressedReaderWriter.scala
        DataFile.scala
        ParquetDataFaultFiberReader.scala
        OapDataFile.scala
        ParquetDataFile.scala
        FiberParser.scala
        EncodedFiberBuilder.scala
        OapDataFileMeta.scala
        OapDataFileProperties.scala
        ParquetDataFileMeta.scala
        OapDataReaderWriter.scala
        VectorizedCacheReader.scala
        ParquetFiberDataLoader.scala
        ParquetDataFiberReaderWriter.scala
        IndexedVectorizedCacheReader.scala
        OrcDataFile.scala
        OrcCacheReader.java
        IndexFile.scala
        ColumnValues.scala
        OapStrategies.scala
        OapOutputWriter.scala
        OptimizedParquetFileFormat.scala
        HadoopFsRelationOptimizer.scala
        parquet
        ParquetFiltersWrapper.scala
        ReadOnlyParquetFileFormat.scala
        ParquetReadSupportWrapper.scala
        OapIndexWriteTaskStatsTracker.scala
        OapFileSourceStrategy.scala
        OapSparkSqlParser.scala
        hive
        orc
        ReadOnlyOrcFileFormat.scala
        thriftserver
        OapEnv.scala
        scheduler
        OapCacheLocations.scala
        util
        collection
        OapExternalSorter.scala
        OapBitSet.scala
        java
        org
        apache
        orc
        impl
        RecordReaderCacheImpl.java
        ColumnDiskRangeList.java
        parquet
        hadoop
        ParquetFiberDataReader.java
        utils
        Collections3.java
        IndexedVectorizedOapRecordReader.java
        metadata
        IndexedStripeMeta.java
        OrderedBlockMetaData.java
        ParquetFooter.java
        IndexedBlockMetaData.java
        LazyInitSeekableInputStream.java
        util
        LazyInitHadoopInputFile.java
        VectorizedOapRecordReader.java
        MrOapRecordReader.java
        InternalOapRecordReader.java
        ParquetReadMetrics.java
        OapParquetFileReader.java
        IndexedMrOapRecordReader.java
        SpecificOapRecordReaderBase.java
        ParquetCacheableFileReader.java
        io
        PositionableRecordReaderImpl.java
        RecordReaderFactory.java
        spark
        sql
        execution
        vectorized
        ColumnVectorAllocator.java
        datasources
        orc
        OrcColumnVectorAllocator.java
        oap
        orc
        IndexedOrcColumnarBatchReader.java
        OrcMapreduceRecordReader.java
        IndexedOrcMapreduceRecordReader.java
        OrcColumnarBatchReader.java
        index
        OapIndexFileOutputCommitter.java
        parquet
        SkippableVectorizedRleValuesReader.java
        SkippableVectorizedPlainValuesReader.java
        ParquetDictionaryWrapper.java
        SkippableVectorizedValuesReader.java
        SkippableVectorizedColumnReader.java
        RecordReader.java
        orc-1.5.9-nohive
        java
        org
        apache
        orc
        impl
        RecordReaderImpl.java
        RecordReaderBinaryCacheImpl.java
        RecordReaderBinaryCacheUtils.java
        RecordReaderUtils.java
      - test
        resources
        log4j.properties
        oap-perf-suite
        scala
        org
        apache
        spark
        sql
        BenchmarkConfig.scala
        OapPerfSuite.scala
        OapPerfSuiteContext.scala
        suites
        BtreeIndexSuite.scala
        LocalSparkMasterTestSuite.scala
        BitmapIndexSuite.scala
        TestUtil.scala
        OapBenchmarkDataBuilder.scala
        OapTestSuite.scala
        conf
        oap-benchmark-default.conf
        scala
        org
        apache
        spark
        sql
        TestOap.scala
        OapExtensionsSuite.scala
        oap
        OapRuntimeSuite.scala
        rpc
        OapRpcManagerSuite.scala
        test
        oap
        SharedOapContext.scala
        OapSharedSQLContext.scala
        parquet
        hadoop
        utils
        Collections3Suite.scala
        IndexedVectorizedOapReaderSuite.scala
        meta
        ParquetFooterSuite.scala
        execution
        vectorized
        VectorizedOapEncodingSuite.scala
        VectorizedOapIOSuite.scala
        datasources
        oap
        statistics
        BloomFilterStatisticsSuite.scala
        StatisticsSuite.scala
        MinMaxStatisticsSuite.scala
        StatisticsTest.scala
        StatisticsManagerSuite.scala
        PartByValueStatisticsSuite.scala
        SampleBasedStatisticsSuite.scala
        utils
        BitmapUtilsSuite.scala
        NonNullKeySuite.scala
        CacheStatusSerDeSuite.scala
        FilterHelperSuite.scala
        OapBitmapWrappedFiberCacheSuite.scala
        BTreeUtilsSuite.scala
        orc
        OapOrcQuerySuite.scala
        OapNativeOrcSuite.scala
        OapMetricsSuite.scala
        index
        BTreeRecordReaderWriterV1Suite.scala
        OapIndexCommitProtocolSuite.scala
        BloomFilterSuite.scala
        BitMapIndexSuite.scala
        IndexUtilsSuite.scala
        BitmapAnalyzeStatisticsSuite.scala
        BTreeIndexScannerSuite.scala
        DropIndexCommandSuite.scala
        BitmapMicroBenchmarkSuite.scala
        BitmapUsageSuite.scala
        BTreeRecordReaderWriterV2Suite.scala
        OapIndexFileOutputCommitterSuite.scala
        IndexSelectionSuite.scala
        OapCheckIndexSuite.scala
        FilterSuite.scala
        OapStrategySuite.scala
        filecache
        FiberSensorSuite.scala
        TestFiberCache.scala
        IndexDataCacheSeparationSuite.scala
        FiberSuite.scala
        BinaryFiberIdSuite.scala
        OapCacheSuite.scala
        MemoryManagerConfigSuite.scala
        FiberCacheManagerSuite.scala
        MemoryManagerSuite.scala
        DataTypeSupportSuite.scala
        OapSuite.scala
        adapter
        PropertiesAdapter.scala
        WholeStageCodeGenAdapter.scala
        CacheHotTablesSuite.scala
        OptimizedParquetFilterSuite.scala
        io
        DataFiberReaderWriterSuite.scala
        DataFileSuite.scala
        TestDataFile.scala
        OrcDataFileSuite.scala
        OapDataFileMetaV1Suite.scala
        ParquetDataFiberHeaderSuite.scala
        FileSkipSuite.scala
        ByteTypeDataFiberReaderWriterSuite.scala
        ShortTypeDataFiberReaderWriterSuite.scala
        IntegerTypeDataFiberReaderWriterSuite.scala
        LongTypeDataFiberReaderWriterSuite.scala
        BooleanTypeDataFiberReaderWriterSuite.scala
        CodecFactorySuite.scala
        FloatTypeDataFiberReaderWriterSuite.scala
        ParquetDataFileSuite.scala
        DictionaryBasedEncoderSuite.scala
        DeltaByteArrayEncoderSuite.scala
        DoubleTypeDataFiberReaderWriterSuite.scala
        OapDataReaderSuite.scala
        BinaryTypeDataFiberReaderWriterSuite.scala
        OapFileSourceStrategySuite.scala
        DataSourceMetaSuite.scala
        OapDDLSuite.scala
        ManualTestREADME.md
        cluster
        ClusteredFilterSuite.scala
        CombiningIndexSuite.scala
        OapIndexQuerySuite.scala
        parquet
        SkippableVectorizedPlainValuesReaderSuite.scala
        SkippableVectorizedColumnReaderSuite.scala
        SkipAndReadValueWithRleDefinitionLevelsSuite.scala
        SkippableVectorizedRleValuesReaderSuite.scala
        SkipAndReadValueWithPackedDefinitionLevelsSuite.scala
        hive
        execution
        OapQuerySuite.scala
        HiveOapIndexDDLSuite.scala
        assembly
        test-jar-with-dependencies.xml
    - scalastyle-config.xml
    - Spark-OAP-Integration-Guide.md
    - checkstyle.xml
    - pom.xml
    - checkstyle-suppressions.xml
    - LICENSE
    - README.md
    - .gitignore
    - docs
      - Developer-Guide.md
      - OAP-User-Guide.md
      - Spark.2.4.4.numa.patch
      - OAP-Architect-Overview.md
      - image
        OAP-Architecture.PNG
        OAP-overview.PNG
        OAPFileFormat.PNG
        Cache-Aware.PNG
        OAP-cache.PNG
        OAP-Introduction.PNG
    - conf
      - persistent-memory.xml.template
  - README.md
- pom.xml
- .clang-format
- oap-data-source
  - arrow
    - src
      - main
        resources
        META-INF
        services
        org.apache.spark.sql.sources.DataSourceRegister
        scala
        com
        intel
        oap
        spark
        sql
        DataFrameReaderImplicits.scala
        execution
        datasources
        arrow
        ArrowFileFormat.scala
        v2
        arrow
        ArrowScan.scala
        ArrowOptions.scala
        ArrowFilters.scala
        ArrowPartitionReaderFactory.scala
        ArrowDataSourceV2.scala
        ArrowSQLConf.scala
        ArrowScanBuilder.scala
        ArrowTable.scala
        org
        apache
        spark
        sql
        execution
        datasources
        v2
        arrow
        ArrowUtils.scala
      - test
        resources
        cars.csv
        scala
        com
        intel
        oap
        spark
        sql
        execution
        datasources
        arrow
        ArrowDataSourceTest.scala
        ArrowDataSourceTPCHBasedTest.scala
    - scalastyle-config.xml
    - pom.xml
    - README.md
    - .gitignore
  - README.md
- oap-shuffle
  - remote-shuffle
    - test-jar-with-dependencies.xml
    - src
      - main
        scala
        org
        apache
        spark
        network
        netty
        MyNettyBlockRpcServer.scala
        RemoteShuffleTransferService.scala
        scheduler
        DAGScheduler.scala
        util
        collection
        RWritablePartitionedPairCollection.scala
        RPartitionedAppendOnlyMap.scala
        RemoteSorter.scala
        RPartitionedPairBuffer.scala
        RemoteAppendOnlyMap.scala
        shuffle
        remote
        RemoteShuffleConf.scala
        RemoteShuffleReader.scala
        RemoteShuffleManager.scala
        RemoteShuffleWriter.scala
        RemoteShuffleBlockIterator.scala
        RemoteShuffleUtils.scala
        RemoteAggregator.scala
        RemoteShuffleBlockResolver.scala
        HadoopFileSegmentManagedBuffer.scala
        RemoteBlockObjectWriter.scala
        java
        org
        apache
        spark
        network
        shuffle
        MyOneForOneBlockFetcher.java
        shuffle
        sort
        RemoteUnsafeShuffleSorter.java
        RemoteUnsafeShuffleWriter.java
        RemoteBypassMergeSortShuffleWriter.java
        RemoteSpillInfo.java
      - test
        scala
        org
        apache
        spark
        util
        collection
        package.scala
        RemoteSorterSuite.scala
        RemoteAppendOnlyMapSuite.scala
        shuffle
        remote
        RemoteShuffleBlockObjectWriterSuite.scala
        RemoteShuffleManagerSuite.scala
        package.scala
        RemoteShuffleBlockIteratorSuite.scala
        RemoteShuffleBlockResolverSuite.scala
        java
        org
        apache
        spark
        shuffle
        sort
        RemoteUnsafeShuffleWriterSuite.java
    - scalastyle-config.xml
    - pom.xml
    - LICENSE
    - dev
      - post_results_to_PR.sh
      - checkstyle.xml
    - README.md
    - .gitignore
  - README.md
  - RPMem-shuffle
    - README.md
- dev
  - install_vmemcache.sh
  - make-distribution.sh
  - README.md
  - conda-release
    - conda-recipes
      - vmemcache
        meta.yaml
        build.sh
      - oap
        meta.yaml
        build.sh
      - intel-arrow
        build.sh
      - memkind
        meta.yaml
        configure.ac
        build.sh
    - README.md
    - make-conda-release.sh
  - prepare_oap_env.sh
  - install_memkind.sh
  - install_arrow.sh
  - build-native-sql.sh
- oap-common
  - src
    - main
      - java
        com
        intel
        oap
        common
        unsafe
        VMEMCacheJNI.java
        PersistentMemoryPlatform.java
        util
        NativeLibraryLoader.java
    - native
      - vmemcache
        compile.sh
        Makefile
        vmemcachejni.c
      - memkind
        compile.sh
        com_intel_oap_common_unsafe_PersistentMemoryPlatform.cpp
        CMakeLists.txt
        com_intel_oap_common_unsafe_PersistentMemoryPlatform.h
  - pom.xml
  - README.md
  - .gitignore
- .travis.yml
- README.md
- .scalafmt.conf
- .gitignore
- docs
  - Developer-Guide.md
- LICENSE.txt