haystack-traces-master
- Makefile
- backends
  - Makefile
  - pom.xml
  - cassandra
    - src
      - main
        resources
        logback.xml
        config
        base.conf
        scala
        com
        expedia
        www
        haystack
        trace
        storage
        backends
        cassandra
        services
        SpansPersistenceService.scala
        GrpcHandler.scala
        GrpcHealthService.scala
        client
        CassandraClusterFactory.scala
        AwsNodeDiscoverer.scala
        ClusterFactory.scala
        CassandraTableSchema.scala
        CassandraSession.scala
        store
        CassandraTraceRecordWriter.scala
        CassandraTraceRecordWriteResultListener.scala
        CassandraTraceRecordReadResultListener.scala
        CassandraTraceRecordReader.scala
        config
        ProjectConfiguration.scala
        entities
        SocketConfiguration.scala
        ServiceConfiguration.scala
        CredentialsConfiguration.scala
        ClientConfiguration.scala
        AwsNodeDiscoveryConfiguration.scala
        Service.scala
        metrics
        AppMetricNames.scala
      - test
        resources
        logback-test.xml
        config
        base.conf
        scala
        com
        expedia
        www
        haystack
        trace
        storage
        backends
        cassandra
        integration
        BaseIntegrationTestSpec.scala
        CassandraStorageBackendServiceIntegrationTestSpec.scala
        unit
        client
        AwsNodeDiscovererSpec.scala
        CassandraSessionSpec.scala
        CassandraTableSchemaSpec.scala
        config
        ConfigurationLoaderSpec.scala
        BaseUnitTestSpec.scala
    - Makefile
    - build
      - docker
        jmxtrans-agent.xml
        Dockerfile
        start-app.sh
      - integration-tests
        docker-compose.yml
    - pom.xml
    - README.md
  - memory
    - src
      - main
        resources
        logback.xml
        config
        base.conf
        scala
        com
        expedia
        www
        haystack
        trace
        storage
        backends
        memory
        services
        SpansPersistenceService.scala
        GrpcHealthService.scala
        store
        InMemoryTraceRecordStore.scala
        config
        ProjectConfiguration.scala
        entities
        ServiceConfiguration.scala
        Service.scala
      - test
        resources
        logback-test.xml
        config
        base.conf
        scala
        com
        expedia
        www
        haystack
        trace
        storage
        backends
        memory
        integration
        InMemoryTraceBackendServiceIntegrationTestSpec.scala
        BaseIntegrationTestSpec.scala
        unit
        config
        ConfigurationLoaderSpec.scala
        BaseUnitTestSpec.scala
    - Makefile
    - build
      - docker
        jmxtrans-agent.xml
        Dockerfile
        start-app.sh
    - pom.xml
    - README.md
- pom.xml
- .travis
  - settings.xml
  - deploy.sh
- mvnw
- LICENSE
- checkstyles
  - scalastyle_config.xml
- CONTRIBUTING.md
- commons
  - src
    - main
      - scala
        com
        expedia
        www
        haystack
        trace
        commons
        utils
        SpanUtils.scala
        config
        reload
        Reloadable.scala
        ConfigurationReloadProvider.scala
        ConfigurationReloadElasticSearchProvider.scala
        entities
        ReloadConfiguration.scala
        AWSRequestSigningConfiguration.scala
        TraceStoreBackends.scala
        WhitelistIndexFieldConfiguration.scala
        clients
        es
        AWSSigningJestClientFactory.scala
        document
        TraceIndexDoc.scala
        ServiceMetadataDoc.scala
        packer
        PackerFactory.scala
        Packer.scala
        PackedMessage.scala
        Unpacker.scala
    - test
      - resources
        logback-test.xml
      - scala
        com
        expedia
        www
        haystack
        trace
        commons
        unit
        WhitelistIndexFieldConfigurationSpec.scala
        PackerSpec.scala
        TraceIndexDocSpec.scala
        BaseUnitTestSpec.scala
  - pom.xml
- deployment
  - terraform
    - es-indices
      - curator-service-metadata
        templates
        curator-cron-job-yaml.tpl
        main.tf
        outputs.tf
        variables.tf
      - whitelisted-fields
        templates
        whitelisted-fields-pod-yaml.tpl
        main.tf
        outputs.tf
        variables.tf
      - curator-trace-index
        templates
        curator-cron-job-yaml.tpl
        main.tf
        outputs.tf
        variables.tf
      - main.tf
      - outputs.tf
      - variables.tf
    - main.tf
    - outputs.tf
    - variables.tf
    - trace-reader
      - templates
        deployment.yaml
        trace-reader.conf
      - main.tf
      - outputs.tf
      - variables.tf
    - trace-indexer
      - templates
        deployment.yaml
        trace-indexer.conf
      - main.tf
      - outputs.tf
      - variables.tf
  - scripts
    - publish-to-docker-hub.sh
- Release.md
- .mvn
  - wrapper
    - maven-wrapper.jar
    - MavenWrapperDownloader.java
    - maven-wrapper.properties
- indexer
  - src
    - main
      - resources
        logback.xml
        config
        base.conf
      - scala
        com
        expedia
        www
        haystack
        trace
        indexer
        StreamRunner.scala
        processors
        StreamTaskRunnable.scala
        supplier
        SpanIndexProcessorSupplier.scala
        StreamProcessorSupplier.scala
        StreamProcessor.scala
        StateListener.scala
        SpanIndexProcessor.scala
        store
        impl
        SpanBufferMemoryStore.scala
        StoreSupplier.scala
        traits
        CacheSizeObserver.scala
        EldestBufferedSpanEvictionListener.scala
        SpanBufferKeyValueStore.scala
        data
        model
        SpanBufferWithMetadata.scala
        DynamicCacheSizer.scala
        SpanBufferMemoryStoreSupplier.scala
        config
        ProjectConfiguration.scala
        entities
        ElasticSearchConfiguration.scala
        ServiceMetadataWriteConfiguration.scala
        KafkaConfiguration.scala
        SpanAccumulatorConfiguration.scala
        TraceBackendConfiguration.scala
        writers
        kafka
        KafkaWriter.scala
        grpc
        WriteSpansResponseObserver.scala
        GrpcTraceWriter.scala
        TraceWriter.scala
        es
        ElasticSearchResultHandler.scala
        ServiceMetadataDocumentGenerator.scala
        ServiceMetadataWriter.scala
        IndexDocumentGenerator.scala
        IndexTemplateHandler.scala
        ElasticSearchWriter.scala
        ThreadSafeBulkBuilder.scala
        App.scala
        metrics
        AppMetricNames.scala
        serde
        SpanDeserializer.scala
    - test
      - resources
        logback-test.xml
        config
        base.conf
      - scala
        com
        expedia
        www
        haystack
        trace
        indexer
        integration
        FailedTopologyRecoverySpec.scala
        serdes
        TestSerdes.scala
        EvictedSpanBufferSpec.scala
        ServiceMetadataIndexingTopologySpec.scala
        MultipleTraceIndexingTopologySpec.scala
        clients
        ElasticSearchTestClient.scala
        GrpcTestClient.scala
        KafkaTestClient.scala
        BaseIntegrationTestSpec.scala
        PartialTraceIndexingTopologySpec.scala
        unit
        IndexTemplateHandlerSpec.scala
        SpanIndexDocumentGeneratorSpec.scala
        ElasticSearchWriterUtilsSpec.scala
        SpanSerdeSpec.scala
        ThreadSafeBulkBuilderSpec.scala
        SpanIndexProcessorSpec.scala
        DynamicCacheSizerSpec.scala
        ConfigurationLoaderSpec.scala
        SpanBufferMemoryStoreSpec.scala
        ElasticSearchResultHandlerSpec.scala
  - Makefile
  - build
    - docker
      - jmxtrans-agent.xml
      - Dockerfile
      - start-app.sh
    - integration-tests
      - docker-compose.yml
  - pom.xml
  - README.md
- .travis.yml
- README.md
- reader
  - src
    - main
      - resources
        logback.xml
        config
        base.conf
      - scala
        com
        expedia
        www
        haystack
        trace
        reader
        services
        GrpcHandler.scala
        TraceService.scala
        GrpcHealthService.scala
        stores
        TraceStore.scala
        EsIndexedTraceStore.scala
        readers
        grpc
        GrpcTraceReaders.scala
        ReadSpansResponseListener.scala
        es
        ElasticSearchResultListener.scala
        query
        ServiceMetadataQueryGenerator.scala
        TraceSearchQueryGenerator.scala
        FieldValuesQueryGenerator.scala
        TraceCountsQueryGenerator.scala
        SpansIndexQueryGenerator.scala
        ESUtils.scala
        ElasticSearchReader.scala
        ElasticSearchReadResultListener.scala
        ElasticSearchCountResultListener.scala
        ResponseParser.scala
        config
        ProviderConfiguration.scala
        entities
        ServiceConfiguration.scala
        ElasticSearchConfiguration.scala
        TraceValidatorsConfiguration.scala
        TraceTransformersConfiguration.scala
        Service.scala
        exceptions
        InvalidTraceIdInDocument.scala
        TraceNotFoundException.scala
        SpanNotFoundException.scala
        InvalidTraceException.scala
        ElasticSearchClientError.scala
        readers
        transformers
        InfrastructureTagTransformer.scala
        SortSpanTransformer.scala
        ClockSkewFromParentTransformer.scala
        DeDuplicateSpanTransformer.scala
        ServerClientSpanMergeTransformer.scala
        InvalidParentTransformer.scala
        PartialSpanTransformer.scala
        ClockSkewTransformer.scala
        SpanTreeTransformer.scala
        TraceTransformationHandler.scala
        OrphanedTraceTransformer.scala
        ClientServerEventLogTransformer.scala
        InvalidRootTransformer.scala
        PostTraceTransformationHandler.scala
        TraceTransformer.scala
        TraceReader.scala
        utils
        TraceMerger.scala
        AuxiliaryTags.scala
        TagExtractors.scala
        SpanMerger.scala
        SpanTree.scala
        TagBuilders.scala
        TraceProcessor.scala
        validators
        TraceIdValidator.scala
        ParentIdValidator.scala
        TraceValidationHandler.scala
        RootValidator.scala
        TraceValidator.scala
        metrics
        AppMetricNames.scala
    - test
      - resources
        logback-test.xml
        raw_trace.json
        config
        base.conf
      - scala
        com
        expedia
        www
        haystack
        trace
        reader
        integration
        TraceServiceIntegrationTestSpec.scala
        BaseIntegrationTestSpec.scala
        unit
        stores
        ResponseParserSpec.scala
        readers
        grpc
        ReadSpansResponseListenerSpec.scala
        es
        query
        ElasticSearchReadResultListenerSpec.scala
        FieldValuesQueryGeneratorSpec.scala
        TraceCountsQueryGeneratorSpec.scala
        helper
        ExpressionTreeBuilder.scala
        TraceSearchQueryGeneratorSpec.scala
        ServiceMetadataQueryGeneratorSpec.scala
        config
        ConfigurationLoaderSpec.scala
        BaseUnitTestSpec.scala
        readers
        transformers
        ClientServerEventLogTransformerSpec.scala
        ClockSkewFromParentTransformerSpec.scala
        ServerClientSpanMergeTransformerSpec.scala
        SortSpanTransformerSpec.scala
        ClockSkewTransformerSpec.scala
        InfrastructureTagTransformerSpec.scala
        OrphanedTraceTransformerSpec.scala
        DeDuplicateSpanTransformerSpec.scala
        PartialSpanTransformerSpec.scala
        InvalidRootTransformerSpec.scala
        InvalidParentTransformerSpec.scala
        builders
        TraceBuilder.scala
        MultiServerSpanTraceBuilder.scala
        ClockSkewedTraceBuilder.scala
        MultiRootTraceBuilder.scala
        ValidTraceBuilder.scala
        validators
        ParentIdValidatorSpec.scala
        TraceIdValidatorSpec.scala
        RootValidatorSpec.scala
        TraceProcessorSpec.scala
        TraceMergerSpec.scala
  - Makefile
  - build
    - docker
      - jmxtrans-agent.xml
      - Dockerfile
      - start-app.sh
    - integration-tests
      - docker-app.conf
      - docker-compose.yml
  - pom.xml
  - README.md
- .gitignore
- mvnw.cmd

haystack-traces

This repo contains the haystack components that build the traces, store them in Cassandra and ElasticSearch(for indexing) and provide a grpc endpoint for accessing them

Building

Since this repo contains haystack-idl as the submodule, so use the following to clone the repo

git clone --recursive [email protected]:ExpediaDotCom/haystack-traces.git .

Prerequisite:

Make sure you have Java 1.8
Make sure you have maven 3.3.9 or higher
Make sure you have docker 1.13 or higher

Note : For mac users you can download docker for mac to set you up for the last two steps.

Build

For a full build, including unit tests and integration tests, docker image build, you can run -

make all

Integration Test

Prerequisite:

Install docker using Docker Tools or native docker if on mac
Verify if docker-compose is installed by running following command else install it.

docker-compose

Run the build and integration tests for individual components with

make indexer

make reader