Java Code Examples for org.apache.spark.broadcast.Broadcast#getValue()

private static FlatMapFunction<Shard<VariantContext>, VariantWalkerContext> getVariantsFunction(
        final String referenceFileName,
        final Broadcast<FeatureManager> bFeatureManager) {
    return (FlatMapFunction<Shard<VariantContext>, VariantWalkerContext>) shard -> {
        ReferenceDataSource reference = referenceFileName == null ? null : new ReferenceFileSource(IOUtils.getPath(SparkFiles.get(referenceFileName)));
        FeatureManager features = bFeatureManager == null ? null : bFeatureManager.getValue();

        return StreamSupport.stream(shard.spliterator(), false)
                .filter(v -> v.getStart() >= shard.getStart() && v.getStart() <= shard.getEnd()) // only include variants that start in the shard
                .map(v -> {
                    final SimpleInterval variantInterval = new SimpleInterval(v);
                    return new VariantWalkerContext(v,
                            new ReadsContext(), // empty
                            new ReferenceContext(reference, variantInterval),
                            new FeatureContext(features, variantInterval));
                }).iterator();
    };
}
 

@VisibleForTesting
static VariantContextBuilder annotateWithExternalCNVCalls(final String recordContig, final int pos, final int end,
                                                          final VariantContextBuilder inputBuilder,
                                                          final Broadcast<SAMSequenceDictionary> broadcastSequenceDictionary,
                                                          final Broadcast<SVIntervalTree<VariantContext>> broadcastCNVCalls,
                                                          final String sampleId) {
    if (broadcastCNVCalls == null)
        return inputBuilder;
    final SVInterval variantInterval = new SVInterval(broadcastSequenceDictionary.getValue().getSequenceIndex(recordContig), pos, end);
    final SVIntervalTree<VariantContext> cnvCallTree = broadcastCNVCalls.getValue();
    final String cnvCallAnnotation =
            Utils.stream(cnvCallTree.overlappers(variantInterval))
                    .map(overlapper -> formatExternalCNVCallAnnotation(overlapper.getValue(), sampleId))
                    .collect(Collectors.joining(VCFConstants.INFO_FIELD_ARRAY_SEPARATOR));
    if (!cnvCallAnnotation.isEmpty()) {
        return inputBuilder.attribute(GATKSVVCFConstants.EXTERNAL_CNV_CALLS, cnvCallAnnotation);
    } else
        return inputBuilder;
}
 

private static JavaRDD<GATKRead> getContigRawAlignments(final JavaSparkContext ctx,
                                                        final FindBreakpointEvidenceSpark.AssembledEvidenceResults assembledEvidenceResults,
                                                        final SvDiscoveryInputMetaData svDiscoveryInputMetaData) {
    final Broadcast<SAMSequenceDictionary> referenceSequenceDictionaryBroadcast =
            svDiscoveryInputMetaData.getReferenceData().getReferenceSequenceDictionaryBroadcast();
    final Broadcast<SAMFileHeader> headerBroadcast = svDiscoveryInputMetaData.getSampleSpecificData().getHeaderBroadcast();
    final SAMFileHeader headerForReads = headerBroadcast.getValue();
    final SAMReadGroupRecord contigAlignmentsReadGroup = new SAMReadGroupRecord(SVUtils.GATKSV_CONTIG_ALIGNMENTS_READ_GROUP_ID);
    final List<String> refNames = SequenceDictionaryUtils.getContigNamesList(referenceSequenceDictionaryBroadcast.getValue());

    return ctx.parallelize(
            assembledEvidenceResults
                    .getAlignedAssemblyOrExcuseList().stream()
                    .filter(AlignedAssemblyOrExcuse::isNotFailure)
                    .flatMap(aa -> aa.toSAMStreamForAlignmentsOfThisAssembly(headerForReads, refNames, contigAlignmentsReadGroup))
                    .map(SAMRecordToGATKReadAdapter::new)
                    .collect(Collectors.toList())
    );
}
 

private static FlatMapFunction<Iterator<AssemblyRegionWalkerContext>, VariantContext> assemblyFunction(final SAMFileHeader header,
                                                                                                       final String referenceFileName,
                                                                                                       final Broadcast<HaplotypeCallerArgumentCollection> hcArgsBroadcast,
                                                                                                       final Broadcast<AssemblyRegionArgumentCollection> assemblyRegionArgsBroadcast,
                                                                                                       final Broadcast<VariantAnnotatorEngine> annotatorEngineBroadcast) {
    return (FlatMapFunction<Iterator<AssemblyRegionWalkerContext>, VariantContext>) contexts -> {
        // HaplotypeCallerEngine isn't serializable but is expensive to instantiate, so construct and reuse one for every partition
        final ReferenceSequenceFile taskReferenceSequenceFile = taskReferenceSequenceFile(referenceFileName);
        final HaplotypeCallerEngine hcEngine = new HaplotypeCallerEngine(hcArgsBroadcast.value(), assemblyRegionArgsBroadcast.value(), false, false, header, taskReferenceSequenceFile, annotatorEngineBroadcast.getValue());
        Iterator<Iterator<VariantContext>> iterators = Utils.stream(contexts).map(context -> {
            AssemblyRegion region = context.getAssemblyRegion();
            FeatureContext featureContext = context.getFeatureContext();
            return hcEngine.callRegion(region, featureContext, context.getReferenceContext()).iterator();
        }).iterator();

        return Iterators.concat(iterators);
    };
}
 

@Override
@SuppressWarnings("unchecked")
public <T> Collection<T> getBroadcast(String name) {
    final Broadcast<?> broadcast = this.broadcasts.get(name);
    if (broadcast == null) {
        throw new RheemException("No such broadcast found: " + name);
    }

    return (Collection<T>) broadcast.getValue();
}
 

/**
 * Call variants from Tuples of AssemblyRegion and Simple Interval
 * The interval should be the non-padded shard boundary for the shard that the corresponding AssemblyRegion was
 * created in, it's used to eliminate redundant variant calls at the edge of shard boundaries.
 */
private static FlatMapFunction<Iterator<Tuple2<AssemblyRegion, SimpleInterval>>, VariantContext> callVariantsFromAssemblyRegions(
        final AuthHolder authHolder,
        final SAMFileHeader header,
        final Broadcast<ReferenceMultiSource> referenceBroadcast,
        final Broadcast<HaplotypeCallerArgumentCollection> hcArgsBroadcast) {
    return regionAndIntervals -> {
        //HaplotypeCallerEngine isn't serializable but is expensive to instantiate, so construct and reuse one for every partition
        final ReferenceMultiSourceAdapter referenceReader = new ReferenceMultiSourceAdapter(referenceBroadcast.getValue(), authHolder);
        final HaplotypeCallerEngine hcEngine = new HaplotypeCallerEngine(hcArgsBroadcast.value(), header, referenceReader);
        return iteratorToStream(regionAndIntervals).flatMap(regionToVariants(hcEngine)).iterator();
    };
}
 

public FirstIterationFunction(Broadcast<Map<String, Object>> word2vecVarMapBroadcast,
                                     Broadcast<double[]> expTableBroadcast, Broadcast<VocabCache<VocabWord>> vocabCacheBroadcast) {

    Map<String, Object> word2vecVarMap = word2vecVarMapBroadcast.getValue();
    this.expTable = expTableBroadcast.getValue();
    this.vectorLength = (int) word2vecVarMap.get("vectorLength");
    this.useAdaGrad = (boolean) word2vecVarMap.get("useAdaGrad");
    this.negative = (double) word2vecVarMap.get("negative");
    this.window = (int) word2vecVarMap.get("window");
    this.alpha = (double) word2vecVarMap.get("alpha");
    this.minAlpha = (double) word2vecVarMap.get("minAlpha");
    this.totalWordCount = (long) word2vecVarMap.get("totalWordCount");
    this.seed = (long) word2vecVarMap.get("seed");
    this.maxExp = (int) word2vecVarMap.get("maxExp");
    this.iterations = (int) word2vecVarMap.get("iterations");
    this.batchSize = (int) word2vecVarMap.get("batchSize");
    this.indexSyn0VecMap = new HashMap<>();
    this.pointSyn1VecMap = new HashMap<>();
    this.vocab = vocabCacheBroadcast.getValue();

    if (this.vocab == null)
        throw new RuntimeException("VocabCache is null");

    if (negative > 0) {
        negativeHolder = NegativeHolder.getInstance();
        negativeHolder.initHolder(vocab, expTable, this.vectorLength);
    }
}
 

private static FlatMapFunction<Iterator<Shard<GATKRead>>, AssemblyRegionWalkerContext> getAssemblyRegionsFunctionFast(
        final String referenceFileName,
        final Broadcast<FeatureManager> bFeatureManager,
        final SAMFileHeader header,
        final Broadcast<Supplier<AssemblyRegionEvaluator>> supplierBroadcast,
        final AssemblyRegionArgumentCollection assemblyRegionArgs) {
    return (FlatMapFunction<Iterator<Shard<GATKRead>>, AssemblyRegionWalkerContext>) shardedReadIterator -> {
        final ReferenceDataSource reference = referenceFileName == null ? null : new ReferenceFileSource(IOUtils.getPath(SparkFiles.get(referenceFileName)));
        final FeatureManager features = bFeatureManager == null ? null : bFeatureManager.getValue();
        final AssemblyRegionEvaluator assemblyRegionEvaluator = supplierBroadcast.getValue().get(); // one AssemblyRegionEvaluator instance per Spark partition
        final ReadsDownsampler readsDownsampler = assemblyRegionArgs.maxReadsPerAlignmentStart > 0 ?
                new PositionalDownsampler(assemblyRegionArgs.maxReadsPerAlignmentStart, header) : null;

        Iterator<Iterator<AssemblyRegionWalkerContext>> iterators = Utils.stream(shardedReadIterator)
                .map(shardedRead -> new ShardToMultiIntervalShardAdapter<>(
                        new DownsampleableSparkReadShard(
                                new ShardBoundary(shardedRead.getInterval(), shardedRead.getPaddedInterval()), shardedRead, readsDownsampler)))
                .map(downsampledShardedRead -> {
                    final Iterator<AssemblyRegion> assemblyRegionIter = new AssemblyRegionIterator(
                            new ShardToMultiIntervalShardAdapter<>(downsampledShardedRead),
                            header, reference, features, assemblyRegionEvaluator, assemblyRegionArgs);
                    return Utils.stream(assemblyRegionIter).map(assemblyRegion ->
                            new AssemblyRegionWalkerContext(assemblyRegion,
                                    new ReferenceContext(reference, assemblyRegion.getPaddedSpan()),
                                    new FeatureContext(features, assemblyRegion.getPaddedSpan()))).iterator();
                }).iterator();
        return Iterators.concat(iterators);
    };
}
 

private static FlatMapFunction<Iterator<Shard<GATKRead>>, ActivityProfileStateRange> getActivityProfileStatesFunction(
        final String referenceFileName,
        final Broadcast<FeatureManager> bFeatureManager,
        final SAMFileHeader header,
        final Broadcast<Supplier<AssemblyRegionEvaluator>> supplierBroadcast,
        final AssemblyRegionArgumentCollection assemblyRegionArgs) {
    return (FlatMapFunction<Iterator<Shard<GATKRead>>, ActivityProfileStateRange>) shardedReadIterator -> {
        final ReferenceDataSource reference = referenceFileName == null ? null : new ReferenceFileSource(IOUtils.getPath(SparkFiles.get(referenceFileName)));
        final FeatureManager features = bFeatureManager == null ? null : bFeatureManager.getValue();
        final AssemblyRegionEvaluator assemblyRegionEvaluator = supplierBroadcast.getValue().get(); // one AssemblyRegionEvaluator instance per Spark partition
        
        return Utils.stream(shardedReadIterator)
                .map(shardedRead -> {
                    final ReadsDownsampler readsDownsampler = assemblyRegionArgs.maxReadsPerAlignmentStart > 0 ?
                            new PositionalDownsampler(assemblyRegionArgs.maxReadsPerAlignmentStart, header) : null;
                    return new ShardToMultiIntervalShardAdapter<>(
                            new DownsampleableSparkReadShard(
                                    new ShardBoundary(shardedRead.getInterval(), shardedRead.getPaddedInterval()), shardedRead, readsDownsampler));
                })
                .map(shardedRead -> {
                    final Iterator<ActivityProfileState> activityProfileStateIter = new ActivityProfileStateIterator(
                            new ShardToMultiIntervalShardAdapter<>(shardedRead),
                            header, reference, features, assemblyRegionEvaluator
                    );
                    return new ActivityProfileStateRange(shardedRead, activityProfileStateIter);
                }).iterator();
    };
}
 

private static FlatMapFunction<Iterator<AssemblyRegion>, AssemblyRegionWalkerContext> getAssemblyRegionWalkerContextFunction(
        final String referenceFileName,
        final Broadcast<FeatureManager> bFeatureManager) {

    return (FlatMapFunction<Iterator<AssemblyRegion>, AssemblyRegionWalkerContext>) assemblyRegionIter -> {
        final ReferenceDataSource reference = referenceFileName == null ? null : new ReferenceFileSource(IOUtils.getPath(SparkFiles.get(referenceFileName)));
        final FeatureManager features = bFeatureManager == null ? null : bFeatureManager.getValue();
        return Utils.stream(assemblyRegionIter).map(assemblyRegion ->
                new AssemblyRegionWalkerContext(assemblyRegion,
                        new ReferenceContext(reference, assemblyRegion.getPaddedSpan()),
                        new FeatureContext(features, assemblyRegion.getPaddedSpan()))).iterator();
    };
}
 

private static FlatMapFunction<Iterator<GATKRead>, ReadWalkerContext> getReadsFunction(
        String referenceFileName, Broadcast<FeatureManager> bFeatureManager) {
    return readIterator -> {
        ReferenceDataSource reference = referenceFileName == null ? null : new ReferenceFileSource(IOUtils.getPath(SparkFiles.get(referenceFileName)));
        FeatureManager features = bFeatureManager == null ? null : bFeatureManager.getValue();
        return Iterators.transform(readIterator, new Function<GATKRead, ReadWalkerContext>() {
            @Nullable
            @Override
            public ReadWalkerContext apply(@Nullable GATKRead r) {
                final SimpleInterval readInterval = getReadInterval(r);
                return new ReadWalkerContext(r, new ReferenceContext(reference, readInterval), new FeatureContext(features, readInterval));
            }
        });
    };
}
 

private static FlatMapFunction<Iterator<LocusWalkerContext>, AllelicCountCollector> distributedCount(final Broadcast<SampleLocatableMetadata> sampleMetadataBroadcast,
                                                                                                     final int minimumBaseQuality) {
    return (FlatMapFunction<Iterator<LocusWalkerContext>, AllelicCountCollector>) contextIterator -> {
        final AllelicCountCollector result = new AllelicCountCollector(sampleMetadataBroadcast.getValue());

        contextIterator.forEachRemaining( ctx -> {
            final byte refAsByte = ctx.getReferenceContext().getBase();
            result.collectAtLocus(Nucleotide.decode(refAsByte), ctx.getAlignmentContext().getBasePileup(),
                    ctx.getAlignmentContext().getLocation(), minimumBaseQuality);
            }
        );
        return Collections.singletonList(result).iterator();
    };
}
 

private static List<SVInterval> findHighCoverageSubintervalsAndLog(
        final FindBreakpointEvidenceSparkArgumentCollection params,
        final JavaSparkContext ctx,
        final Broadcast<ReadMetadata> broadcastMetadata,
        final List<SVInterval> intervals,
        final JavaRDD<GATKRead> unfilteredReads,
        final SVReadFilter filter,
        final Logger logger) {

    final int minFlankingHighCovFactor = params.highDepthCoverageFactor;
    final int minPeakHighCovFactor = params.highDepthCoveragePeakFactor;

    final ReadMetadata shortReadMetadata = broadcastMetadata.getValue();
    int minFlankingHighCoverageValue = (int) (minFlankingHighCovFactor * shortReadMetadata.getCoverage());
    int minPeakHighCoverageValue = (int) (minPeakHighCovFactor * shortReadMetadata.getCoverage());
    final List<SVInterval> result =
            findHighCoverageSubIntervals(ctx, broadcastMetadata, intervals, unfilteredReads,
                    filter,
                    minFlankingHighCoverageValue,
                    minPeakHighCoverageValue);
    log("Found " + result.size() + " sub-intervals with coverage over " + minFlankingHighCoverageValue +
            " and a peak coverage of over " + minPeakHighCoverageValue + ".", logger);

    final String intervalFile = params.highCoverageIntervalsFile;
    if (intervalFile != null) {
        try (final OutputStreamWriter writer =
                     new OutputStreamWriter(new BufferedOutputStream(BucketUtils.createFile(intervalFile)))) {
            for (final SVInterval svInterval : result) {
                final String bedLine = shortReadMetadata.getContigName(svInterval.getContig()) + "\t" + (svInterval.getStart() - 1) + "\t" + svInterval.getEnd() + "\n";
                writer.write(bedLine);
            }
        } catch (final IOException ioe) {
            throw new UserException.CouldNotCreateOutputFile("Can't write high coverage intervals file " + intervalFile, ioe);
        }
    }
    return result;
}
 

Reader(Table table, Broadcast<FileIO> io, Broadcast<EncryptionManager> encryptionManager,
    boolean caseSensitive, DataSourceOptions options) {
  this.table = table;
  this.snapshotId = options.get("snapshot-id").map(Long::parseLong).orElse(null);
  this.asOfTimestamp = options.get("as-of-timestamp").map(Long::parseLong).orElse(null);
  if (snapshotId != null && asOfTimestamp != null) {
    throw new IllegalArgumentException(
        "Cannot scan using both snapshot-id and as-of-timestamp to select the table snapshot");
  }

  this.startSnapshotId = options.get("start-snapshot-id").map(Long::parseLong).orElse(null);
  this.endSnapshotId = options.get("end-snapshot-id").map(Long::parseLong).orElse(null);
  if (snapshotId != null || asOfTimestamp != null) {
    if (startSnapshotId != null || endSnapshotId != null) {
      throw new IllegalArgumentException(
          "Cannot specify start-snapshot-id and end-snapshot-id to do incremental scan when either snapshot-id or " +
              "as-of-timestamp is specified");
    }
  } else {
    if (startSnapshotId == null && endSnapshotId != null) {
      throw new IllegalArgumentException("Cannot only specify option end-snapshot-id to do incremental scan");
    }
  }

  // look for split behavior overrides in options
  this.splitSize = options.get("split-size").map(Long::parseLong).orElse(null);
  this.splitLookback = options.get("lookback").map(Integer::parseInt).orElse(null);
  this.splitOpenFileCost = options.get("file-open-cost").map(Long::parseLong).orElse(null);

  if (io.getValue() instanceof HadoopFileIO) {
    String scheme = "no_exist";
    try {
      Configuration conf = new Configuration(SparkSession.active().sparkContext().hadoopConfiguration());
      // merge hadoop config set on table
      mergeIcebergHadoopConfs(conf, table.properties());
      // merge hadoop config passed as options and overwrite the one on table
      mergeIcebergHadoopConfs(conf, options.asMap());
      FileSystem fs = new Path(table.location()).getFileSystem(conf);
      scheme = fs.getScheme().toLowerCase(Locale.ENGLISH);
    } catch (IOException ioe) {
      LOG.warn("Failed to get Hadoop Filesystem", ioe);
    }
    this.localityPreferred = options.get("locality").map(Boolean::parseBoolean)
        .orElse(LOCALITY_WHITELIST_FS.contains(scheme));
  } else {
    this.localityPreferred = false;
  }

  this.schema = table.schema();
  this.io = io;
  this.encryptionManager = encryptionManager;
  this.caseSensitive = caseSensitive;

  this.batchReadsEnabled = options.get("vectorization-enabled").map(Boolean::parseBoolean).orElse(
      PropertyUtil.propertyAsBoolean(table.properties(),
          TableProperties.PARQUET_VECTORIZATION_ENABLED, TableProperties.PARQUET_VECTORIZATION_ENABLED_DEFAULT));
  this.batchSize = options.get("batch-size").map(Integer::parseInt).orElse(
      PropertyUtil.propertyAsInt(table.properties(),
        TableProperties.PARQUET_BATCH_SIZE, TableProperties.PARQUET_BATCH_SIZE_DEFAULT));
}
 

@Test
public void testBroadcastHcArgs() {
    Broadcast<HaplotypeCallerArgumentCollection> broadcast = SparkContextFactory.getTestSparkContext().broadcast(new HaplotypeCallerArgumentCollection());
    broadcast.getValue();
}
 

public ChunkProcessingTask(Broadcast<String> transform,
                           Broadcast<String> header, Broadcast<Properties> exporterProperties) {
    fHeader = header.getValue();
    fTansform = transform.getValue();
    fProperites = exporterProperties.getValue();
}
 

@Override
protected void runTool(JavaSparkContext ctx) {
    Broadcast<SAMFileHeader> headerBroadcast = ctx.broadcast(getHeaderForReads());
    JavaRDD<GATKRead> reads = getReads();

    ////////////////////////////////////////////////////////////////////////////
    // Grab the input header and remap values where appropriate
    ////////////////////////////////////////////////////////////////////////////
    SAMFileHeader localHeader = headerBroadcast.getValue();
    validateHeaderOverrides(localHeader, sampleAlias, libraryName);
    if (sampleAlias != null) {
        localHeader.getReadGroups().forEach(rg -> rg.setSample(sampleAlias));
    }
    if (libraryName != null) {
        localHeader.getReadGroups().forEach(rg -> rg.setLibrary(libraryName));
    }

    ////////////////////////////////////////////////////////////////////////////
    // Map the readgroups in the header to appropriate
    ////////////////////////////////////////////////////////////////////////////
    Map<String, Path> writerMap = getOutputMap(outputMap,
                                              output,
                                              getDefaultExtension(readArguments.getReadPathSpecifiers().get(0), outputByReadgroupFileFormat),
                                              localHeader.getReadGroups(),
                                              outputByReadGroup);

    ////////////////////////////////////////////////////////////////////////////
    // Construct appropriate headers for the output files
    ////////////////////////////////////////////////////////////////////////////
    final Map<String, SAMFileHeader> headerMap = getReadGroupHeaderMap(localHeader, writerMap);

    // Revert the reads based on the given attributes
    List<String> attributesToRevert = removeDefaults ? DEFAULT_ATTRIBUTES_TO_CLEAR : new ArrayList<>();
    attributesToRevert.addAll(attributesToClear);
    JavaRDD<GATKRead> readsReverted = revertReads(reads, attributesToRevert);

    ////////////////////////////////////////////////////////////////////////////
    // Sanitize the reads, sorting them into appropriate order if necessary
    ////////////////////////////////////////////////////////////////////////////
    if (sanitize) {
        Map<String, FastqQualityFormat> readGroupFormatMap = createReadGroupFormatMap(readsReverted, headerBroadcast, !dontRestoreOriginalQualities);

        readsReverted = sanitize(readGroupFormatMap, readsReverted, localHeader, keepFirstDuplicate);
    }

    // Write the one or many read output files
    for (Map.Entry<String, Path> rmap: writerMap.entrySet()) {
        //TODO what to do if the readgroup isn't present
        final String key = rmap.getKey();
        JavaRDD<GATKRead> filteredreads = rmap.getKey()==null? readsReverted :
                                                                readsReverted.filter(r -> r.getReadGroup().equals(key));
        writeReads(ctx, rmap.getValue().toString(), filteredreads, headerMap.get(rmap.getKey()), false); //TODO proper header map
    }
}
 

@Test(groups = "sv")
public void testMiscCases() {
    final JavaSparkContext testSparkContext = SparkContextFactory.getTestSparkContext();
    Broadcast<ReferenceMultiSparkSource> referenceBroadcast = testSparkContext.broadcast(TestUtilsForAssemblyBasedSVDiscovery.b38_reference_chr20_chr21);
    Broadcast<SAMSequenceDictionary> refSeqDictBroadcast = testSparkContext.broadcast(TestUtilsForAssemblyBasedSVDiscovery.b38_seqDict_chr20_chr21);

    // the following works for: multiple evidence contigs, insertion sequence mapping available, good non-canonical chromosome mapping available
    final AlignmentInterval asm018485_tig00004_1 = TestUtilsForAssemblyBasedSVDiscovery.fromSAMRecordString("asm018485:tig00004\t0\tchr20\t28831147\t38\t275M2I114M229S\t*\t0\t0\tTATCTTCACATAAAAACTACACAGTATCATTCTGAGAAACTTGTTTGTGATGTGTGCATTCATCTCACAGATTTGAACCCTTCCATCTTTTGAGCAGTTTGTACACCTTCTTTTTGTAAAATCTACAAGTGGATATATGGAGCGCTTTGAGGCCTATTGTGGAAAAGGAAATACCTTCACATAAAAACTACACAGAAGCATTCTGAGAAACTTCTTTTTGACGTGTGCATTCATCTCACAGAGTTGAACATTTCATGTGATTGAGCAGCTTTGAAACACTCTTTTTGTAAAATCTGCAAGTGGGTATTTGCAGCACTTTGAGGCCTATTTTGGAAAAGGAAATATCTTCCCATAAAAACTACATAGAAACATTCTCAGAAACTTCTTTGTGTTATCTGCATGTATGTAGAGAGTTGAACCTTTCATTTGATTTAGCAGTTTGCAAACACTCTTTTAGTAGAATCTGCAAGTAGATATTTGAAGCCCTTGGGGCCTATTATGGAAAAGGAAATATCTTCACATAAAAACTATGCAAAAGCGTTCTGAGAAACTTCATTGTGATGTGTGCATTCACTTAACAGAGTTGAACCTTTCTTTGAATTAAGCAGTTTTGAAACACT\t*\tSA:Z:chr3,90319741,-,121M499S,0,11;chr20,29212084,+,409S71M140S,38,4;chrUn_KN707904v1_decoy,742,+,620M,60,8;\tMD:Z:69A12T1G22C43T21T24A18G3T6C5T8A1A3C0G5T11T18G13A6G3C5C0A5C0G0G6C12A13C4G6G3C11\tRG:Z:GATKSVContigAlignments\tNM:i:34\tAS:i:211\tXS:i:181", true);
    final AlignmentInterval asm018485_tig00004_2 = TestUtilsForAssemblyBasedSVDiscovery.fromSAMRecordString("asm018485:tig00004\t2048\tchr20\t29212084\t38\t409H71M140H\t*\t0\t0\tAGAGTTGAACCTTTCATTTGATTTAGCAGTTTGCAAACACTCTTTTAGTAGAATCTGCAAGTAGATATTTG\t*SA:Z:chr20,28831147,+,275M2I114M229S,38,34;chr3,90319741,-,121M499S,0,11;chrUn_KN707904v1_decoy,742,+,620M,60,8;\tMD:Z:32T0G12T15G8\tRG:Z:GATKSVContigAlignments\tNM:i:4\tAS:i:51\tXS:i:33", true);
    final AlignmentInterval asm018485_tig00004_nonCanonical = TestUtilsForAssemblyBasedSVDiscovery.fromSAMRecordString("asm018485:tig00004\t2048\tchrUn_KN707904v1_decoy\t742\t60\t620M\t*\t0\t0\tTATCTTCACATAAAAACTACACAGTATCATTCTGAGAAACTTGTTTGTGATGTGTGCATTCATCTCACAGATTTGAACCCTTCCATCTTTTGAGCAGTTTGTACACCTTCTTTTTGTAAAATCTACAAGTGGATATATGGAGCGCTTTGAGGCCTATTGTGGAAAAGGAAATACCTTCACATAAAAACTACACAGAAGCATTCTGAGAAACTTCTTTTTGACGTGTGCATTCATCTCACAGAGTTGAACATTTCATGTGATTGAGCAGCTTTGAAACACTCTTTTTGTAAAATCTGCAAGTGGGTATTTGCAGCACTTTGAGGCCTATTTTGGAAAAGGAAATATCTTCCCATAAAAACTACATAGAAACATTCTCAGAAACTTCTTTGTGTTATCTGCATGTATGTAGAGAGTTGAACCTTTCATTTGATTTAGCAGTTTGCAAACACTCTTTTAGTAGAATCTGCAAGTAGATATTTGAAGCCCTTGGGGCCTATTATGGAAAAGGAAATATCTTCACATAAAAACTATGCAAAAGCGTTCTGAGAAACTTCATTGTGATGTGTGCATTCACTTAACAGAGTTGAACCTTTCTTTGAATTAAGCAGTTTTGAAACACT\t*\tSA:Z:chr20,28831147,+,275M2I114M229S,38,34;chr3,90319741,-,121M499S,0,11;chr20,29212084,+,409S71M140S,38,4;\tMD:Z:107C34T55T22T81A25G52C12G224\tRG:Z:GATKSVContigAlignments\tNM:i:8\tAS:i:580\tXS:i:293", true);
    final AlignmentInterval asm018485_tig00004_insmapping = TestUtilsForAssemblyBasedSVDiscovery.fromSAMRecordString("asm018485:tig00004\t2064\tchr3\t90319741\t0\t121M499H\t*\t0\t0\tAGTGTTTCAAAACTGCTTAATTCAAAGAAAGGTTCAACTCTGTTAAGTGAATGCACACATCACAATGAAGTTTCTCAGAACGCTTTTGCATAGTTTTTATGTGAAGATATTTCCTTTTCCA\t*\tSA:Z:chr20,28831147,+,275M2I114M229S,38,34;chr20,29212084,+,409S71M140S,38,4;chrUn_KN707904v1_decoy,742,+,620M,60,8;\tMD:Z:5A11C3C0A23T8G24T4C2T0C17C13\tRG:Z:GATKSVContigAlignments\tNM:i:11\tAS:i:66\tXS:i:62", true);
    final AlignmentInterval asm028167_tig00007_1 = TestUtilsForAssemblyBasedSVDiscovery.fromSAMRecordString("asm028167:tig00007\t0\tchr20\t28831011\t60\t72M2I339M2I114M104S\t*\t0\t0\tGAGAAACTTCTTTGTGATGTGTGCATTCATCTCACAGAGATGAACCTATCTTTTCATAGAGCAGTTTTGAAACTCTCTTTCTGTAGAATCTGCGACTGGATATTTGGAGCCCTTAGCGGCCTATGGTGGAAACGGAATTATCTTCACATAAAAACTACACAGTATCATTCTGAGAAACTTGTTTGTGATGTGTGCATTCATCTCACAGATTTGAACCCTTCCATCTTTTGAGCAGTTTGTACACCTTCTTTTTGTAAAATCTACAAGTGGATATATGGAGCGCTTTGAGGCCTATTGTGGAAAAGGAAATACCTTCACATAAAAACTACACAGAAGCATTCTGAGAAACTTCTTTTTGACGTGTGCATTCATCTCACAGAGTTGAACATTTCATGTGATTGAGCAGCTTTGAAACACTCTTTTTGTAAAATCTGCAAGTGGGTATTTGCAGCACTTTGAGGCCTATTTTGGAAAAGGAAATATCTTCCCATAAAAACTACATAGAAACATTCTCAGAAACTTCTTTGTGTTATCTGCATGTATGTAGAGAGTTGAACCTTTCATTTGATTTAGCAGTTTGCAAACACTCTTTTAGTAGAATCTGCAAGTAGATATTTGAAGCCCTTGGGGCCT\t*\tSA:Z:chr20,29212084,+,547S71M15S,21,4;chrUn_KN707904v1_decoy,604,+,633M,60,10;\tMD:Z:24G29G36A1A36A74A12T1G22C43T21T24A18G3T6C5T8A1A3C0G5T11T18G13A6G3C5C0A5C0G0G6C12A13C4G6G3C11\tRG:Z:GATKSVContigAlignments\tNM:i:41\tAS:i:304\tXS:i:179", true);
    final AlignmentInterval asm028167_tig00007_2 = TestUtilsForAssemblyBasedSVDiscovery.fromSAMRecordString("asm028167:tig00007\t2048\tchr20\t29212084\t21\t547H71M15H\t*\t0\t0\tAGAGTTGAACCTTTCATTTGATTTAGCAGTTTGCAAACACTCTTTTAGTAGAATCTGCAAGTAGATATTTG\t*SA:Z:chr20,28831011,+,72M2I339M2I114M104S,60,41;chrUn_KN707904v1_decoy,604,+,633M,60,10;\tMD:Z:32T0G12T15G8\tRG:Z:GATKSVContigAlignments\tNM:i:4\tAS:i:51\tXS:i:41", true);
    final AlignmentInterval asm028167_tig00007_nonCanonical = TestUtilsForAssemblyBasedSVDiscovery.fromSAMRecordString("asm028167:tig00007\t2048\tchrUn_KN707904v1_decoy\t604\t60\t633M\t*\t0\t0\tGAGAAACTTCTTTGTGATGTGTGCATTCATCTCACAGAGATGAACCTATCTTTTCATAGAGCAGTTTTGAAACTCTCTTTCTGTAGAATCTGCGACTGGATATTTGGAGCCCTTAGCGGCCTATGGTGGAAACGGAATTATCTTCACATAAAAACTACACAGTATCATTCTGAGAAACTTGTTTGTGATGTGTGCATTCATCTCACAGATTTGAACCCTTCCATCTTTTGAGCAGTTTGTACACCTTCTTTTTGTAAAATCTACAAGTGGATATATGGAGCGCTTTGAGGCCTATTGTGGAAAAGGAAATACCTTCACATAAAAACTACACAGAAGCATTCTGAGAAACTTCTTTTTGACGTGTGCATTCATCTCACAGAGTTGAACATTTCATGTGATTGAGCAGCTTTGAAACACTCTTTTTGTAAAATCTGCAAGTGGGTATTTGCAGCACTTTGAGGCCTATTTTGGAAAAGGAAATATCTTCCCATAAAAACTACATAGAAACATTCTCAGAAACTTCTTTGTGTTATCTGCATGTATGTAGAGAGTTGAACCTTTCATTTGATTTAGCAGTTTGCAAACACTCTTTTAGTAGAATCTGCAAGTAGATATTTGAAGCCCTTGGGGCCT\t*\tSA:Z:chr20,28831011,+,72M2I339M2I114M104S,60,41;chr20,29212084,+,547S71M15S,21,4;\tMD:Z:108A23A112C34T55T22T81A25G52C12G99\tRG:Z:GATKSVContigAlignments\tNM:i:10\tAS:i:583\tXS:i:304", true);

    final SimpleChimera asm018485_tig00004_chimera = new SimpleChimera("asm018485:tig00004", asm018485_tig00004_1, asm018485_tig00004_2, StrandSwitch.NO_SWITCH,
            true, Collections.singletonList(asm018485_tig00004_insmapping.toPackedString()), asm018485_tig00004_nonCanonical.toSATagString());
    final SimpleChimera asm028167_tig00007_chimera = new SimpleChimera("asm028167:tig00007", asm028167_tig00007_1, asm028167_tig00007_2, StrandSwitch.NO_SWITCH,
            true, Collections.emptyList(), asm028167_tig00007_nonCanonical.toSATagString());
    List<SimpleChimera> simpleChimeras = Arrays.asList(asm018485_tig00004_chimera, asm028167_tig00007_chimera);
    SimpleInterval leftBreakpoint = new SimpleInterval("chr20:28831535-28831535");
    SimpleInterval rightBreakpoint = new SimpleInterval("chr20:29212083-29212083");
    BreakpointComplications complications = new BreakpointComplications.SimpleInsDelOrReplacementBreakpointComplications("", "TTATCTGCATGTATGTAG");
    NovelAdjacencyAndAltHaplotype novelAdjacencyAndAltHaplotype = new NovelAdjacencyAndAltHaplotype(leftBreakpoint, rightBreakpoint, StrandSwitch.NO_SWITCH, complications, TypeInferredFromSimpleChimera.RPL, "TTATCTGCATGTATGTAG".getBytes());
    SimpleNovelAdjacencyAndChimericAlignmentEvidence simpleNovelAdjacencyAndChimericAlignmentEvidence = new SimpleNovelAdjacencyAndChimericAlignmentEvidence(novelAdjacencyAndAltHaplotype, simpleChimeras);
    Allele refAllele = Allele.create("G", true);
    final SimpleSVType.Deletion del_chr20_28831535_29212083 = new SimpleSVType.Deletion("chr20", 28831535, 29212083, "DEL_chr20_28831535_29212083", refAllele, Allele.create("<DEL>"), -380548, Collections.emptyMap());
    VariantContext expected = new VariantContextBuilder().chr("chr20").start(28831535).stop(29212083).id("DEL_chr20_28831535_29212083")
            .alleles(Arrays.asList(refAllele, Allele.create("<DEL>")))
            .attribute(VCFConstants.END_KEY, 29212083).attribute(SVLEN, -380548).attribute(SVTYPE, "DEL").attribute(CONTIG_NAMES, "asm018485:tig00004,asm028167:tig00007")
            .attribute(TOTAL_MAPPINGS, 2).attribute(HQ_MAPPINGS, 0).attribute(MAPPING_QUALITIES, "38,21").attribute(ALIGN_LENGTHS, "71,71").attribute(MAX_ALIGN_LENGTH, 71)
            .attribute(SEQ_ALT_HAPLOTYPE, "TTATCTGCATGTATGTAG").attribute(INSERTED_SEQUENCE, "TTATCTGCATGTATGTAG").attribute(INSERTED_SEQUENCE_LENGTH, 18)
            .attribute(INSERTED_SEQUENCE_MAPPINGS, "500_620_chr3:90319741-90319861_-_499H121M_0_11_66_O").attribute(CTG_GOOD_NONCANONICAL_MAPPING, "chrUn_KN707904v1_decoy,742,+,620M,60,8,580,chrUn_KN707904v1_decoy,604,+,633M,60,10,583").make();
    VariantContext actual = AnnotatedVariantProducer.produceAnnotatedVcFromAssemblyEvidence(del_chr20_28831535_29212083, simpleNovelAdjacencyAndChimericAlignmentEvidence, referenceBroadcast, refSeqDictBroadcast, null, "testSample").make();
    VariantContextTestUtils.assertVariantContextsAreEqual(actual, expected, Collections.singletonList(HQ_MAPPINGS), Collections.emptyList());

    // cnv call available
    referenceBroadcast = testSparkContext.broadcast(TestUtilsForAssemblyBasedSVDiscovery.b37_reference);
    refSeqDictBroadcast = testSparkContext.broadcast(TestUtilsForAssemblyBasedSVDiscovery.b37_seqDict);
    final SAMFileHeader samFileHeader = new SAMFileHeader(refSeqDictBroadcast.getValue());
    SAMReadGroupRecord test = new SAMReadGroupRecord("test");
    test.setSample("sample");
    samFileHeader.addReadGroup(test);
    final Broadcast<SVIntervalTree<VariantContext>> cnvCallsBroadcast =
            StructuralVariationDiscoveryPipelineSpark.broadcastCNVCalls(testSparkContext, samFileHeader, SVIntegrationTestDataProvider.EXTERNAL_CNV_CALLS);
    final AlignmentInterval asm000000_tig00006_1 = TestUtilsForAssemblyBasedSVDiscovery.fromSAMRecordString("asm000000:tig00006\t16\t21\t43349675\t60\t448M229S\t*\t0\t0\tACTAGGTGGGTTATAACTTTTATTTAAAACTTTCAGTTCCAGCTGATGGTTATACCATTGGGAGCCTCCATTTACTTAGAAATGAAACTGAAAACAGACAACTAAAGCATGTCCAGGACTCCTGGCTCCACACCATGCCAGGCGACATCACTCAAGTCTCCAAAGATCACCAAGTGTCCAGCTCAGCTCCTGCCCTCATCAGCAAGTTTTCCAAATGAAAGTTACGTTGAAAGCCACAGTTACCATACTGTAACCAGAATTCAGGCAGTGGCTGCTAGCAGAGTATGATGAACAAGAGCAGGTCTGGTATAAAGACAGTGACTTTGCATTCCAAAGCTTAGCTTAGGGGAAGAACAGGCTTCTGCCTTAAGGGTACCCCTTTGCTTTCGGGGCAGAAAGCAGGCACTTTCAAAAGGGGGCTTGGCATGAATGTCATGAAAGGGAGGAACACCACTGTGAACCCGCTGCCCTACACGGCAGTTCTAGGGCTGAACTCACCGAACAGTGTTAACAAAAAGAGGCCTTGCTGTCTTATCATTTTTATTTAACGCACGAACATTAAGCAGTGTCTCACCCTGGACATTTTACAAGAGATTAAGCTGGCTGGATGCCTTTGCAAAAACAGTGCCCTAAAAATGTGTCATGTTTGGCCAAGATGCTCATCCAAGAATGGAAAA\t*\tSA:Z:21,43353486,-,442S235M,60,0;\tMD:Z:387T60\tRG:Z:GATKSVContigAlignments\tNM:i:1\tAS:i:443\tXS:i:0", true);
    final AlignmentInterval asm000000_tig00006_2 = TestUtilsForAssemblyBasedSVDiscovery.fromSAMRecordString("asm000000:tig00006\t2064\t21\t43353486\t60\t442H235M\t*\t0\t0\tGAGGAACACCACTGTGAACCCGCTGCCCTACACGGCAGTTCTAGGGCTGAACTCACCGAACAGTGTTAACAAAAAGAGGCCTTGCTGTCTTATCATTTTTATTTAACGCACGAACATTAAGCAGTGTCTCACCCTGGACATTTTACAAGAGATTAAGCTGGCTGGATGCCTTTGCAAAAACAGTGCCCTAAAAATGTGTCATGTTTGGCCAAGATGCTCATCCAAGAATGGAAAA\t*\tSA:Z:21,43349675,-,448M229S,60,1;\tMD:Z:235\tRG:Z:GATKSVContigAlignments\tNM:i:0\tAS:i:235\tXS:i:0", true);
    final AlignmentInterval asm000001_tig00001_1 = TestUtilsForAssemblyBasedSVDiscovery.fromSAMRecordString("asm000001:tig00001\t16\t21\t43349641\t60\t25M1D456M417S\t*\t0\t0\tGTCTCCCTGGCTTCTGAGATGGGCCTTCCCCCGACTAGGTGGGTTATAACTTTTATTTAAAACTTTCAGTTCCAGCTGATGGTTATACCATTGGGAGCCTCCATTTACTTAGAAATGAAACTGAAAACAGACAACTAAAGCATGTCCAGGACTCCTGGCTCCACACCATGCCAGGCGACATCACTCAAGTCTCCAAAGATCACCAAGTGTCCAGCTCAGCTCCTGCCCTCATCAGCAAGTTTTCCAAATGAAAGTTACGTTGAAAGCCACAGTTACCATACTGTAACCAGAATTCAGGCAGTGGCTGCTAGCAGAGTATGATGAACAAGAGCAGGTCTGGTATAAAGACAGTGACTTTGCATTCCAAAGCTTAGCTTAGGGGAAGAACAGGCTTCTGCCTTAAGGGTACCCCTTTGCTTTCGGGGCAGAAAGCAGGCACTTTCAAAAGGGGGCTTGGCATGAATGTCATGAAAGGGAGGAACACCACTGTGAACCCGCTGCCCTACACGGCAGTTCTAGGGCTGAACTCACCGAACAGTGTTAACAAAAAGAGGCCTTGCTGTCTTATCATTTTTATTTAACGCACGAACATTAAGCAGTGTCTCACCCTGGACATTTTACAAGAGATTAAGCTGGCTGGATGCCTTTGCAAAAACAGTGCCCTAAAAATGTGTCATGTTTGGCCAAGATGCTCATCCAAGAATGGAAAAGGCCATGTACACAATCCAAGCACCCGAGGGTGTTCTACTCCCAACTGACCCTTCCCAGGAGCCCGGGCAGATCCCAACAGGACTTCCTCCTTGTGGGTATGCATAGGATCCAGGCTGGCAAGAGCGACCAGGCTCCTCCTCCCGCACTCACAGCCCCGTGAAAGGGGAGGGGAGGGGAGGGAACCCGT\t*\tSA:Z:21,43353486,-,475S423M,60,0;\tMD:Z:25^T6A388T60\tRG:Z:GATKSVContigAlignments\tNM:i:3\tAS:i:454\tXS:i:0", true);
    final AlignmentInterval asm000001_tig00001_2 = TestUtilsForAssemblyBasedSVDiscovery.fromSAMRecordString("asm000001:tig00001\t2064\t21\t43353486\t60\t475H423M\t*\t0\t0\tGAGGAACACCACTGTGAACCCGCTGCCCTACACGGCAGTTCTAGGGCTGAACTCACCGAACAGTGTTAACAAAAAGAGGCCTTGCTGTCTTATCATTTTTATTTAACGCACGAACATTAAGCAGTGTCTCACCCTGGACATTTTACAAGAGATTAAGCTGGCTGGATGCCTTTGCAAAAACAGTGCCCTAAAAATGTGTCATGTTTGGCCAAGATGCTCATCCAAGAATGGAAAAGGCCATGTACACAATCCAAGCACCCGAGGGTGTTCTACTCCCAACTGACCCTTCCCAGGAGCCCGGGCAGATCCCAACAGGACTTCCTCCTTGTGGGTATGCATAGGATCCAGGCTGGCAAGAGCGACCAGGCTCCTCCTCCCGCACTCACAGCCCCGTGAAAGGGGAGGGGAGGGGAGGGAACCCGT\t*SA:Z:21,43349641,-,25M1D456M417S,60,3;\tMD:Z:423\tRG:Z:GATKSVContigAlignments\tNM:i:0\tAS:i:423\tXS:i:0", true);
    final SimpleChimera asm000000_tig00006_chimera = new SimpleChimera("asm000000:tig00006", asm000000_tig00006_1, asm000000_tig00006_2, StrandSwitch.NO_SWITCH,
            false, Collections.emptyList(), NO_GOOD_MAPPING_TO_NON_CANONICAL_CHROMOSOME);
    final SimpleChimera asm000001_tig00001_chimera = new SimpleChimera("asm000001:tig00001", asm000001_tig00001_1, asm000001_tig00001_2, StrandSwitch.NO_SWITCH,
            false, Collections.emptyList(), NO_GOOD_MAPPING_TO_NON_CANONICAL_CHROMOSOME);
    simpleChimeras = Arrays.asList(asm000001_tig00001_chimera, asm000000_tig00006_chimera);
    leftBreakpoint = new SimpleInterval("21:43350116-43350116");
    rightBreakpoint = new SimpleInterval("21:43353485-43353485");
    complications = new BreakpointComplications.SimpleInsDelOrReplacementBreakpointComplications("GAGGAA", "");
    novelAdjacencyAndAltHaplotype = new NovelAdjacencyAndAltHaplotype(leftBreakpoint, rightBreakpoint, StrandSwitch.NO_SWITCH, complications, TypeInferredFromSimpleChimera.SIMPLE_DEL, new byte[]{});
    simpleNovelAdjacencyAndChimericAlignmentEvidence = new SimpleNovelAdjacencyAndChimericAlignmentEvidence(novelAdjacencyAndAltHaplotype, simpleChimeras);
    final SimpleSVType.Deletion del_21_43350116_43353485 = new SimpleSVType.Deletion("21", 43350116, 43353485, "DEL_21_43350116_43353485", refAllele, Allele.create("<DEL>"), -3369, Collections.emptyMap());
    expected = new VariantContextBuilder().chr("21").start(43350116).stop(43353485).id("DEL_21_43350116_43353485")
            .alleles(Arrays.asList(refAllele, Allele.create("<DEL>"))).attribute(VCFConstants.END_KEY, 43353485)
            .attribute(SVLEN, -3369).attribute(SVTYPE, "DEL").attribute(CONTIG_NAMES, "asm000000:tig00006,asm000001:tig00001").attribute(TOTAL_MAPPINGS, 2)
            .attribute(HQ_MAPPINGS, 2).attribute(MAPPING_QUALITIES, "60,60").attribute(ALIGN_LENGTHS, "229,417").attribute(MAX_ALIGN_LENGTH, 417).attribute(HOMOLOGY, "GAGGAA")
            .attribute(HOMOLOGY_LENGTH, 6).attribute(EXTERNAL_CNV_CALLS, "CNV_21_43350200_43353400:1:80").make();
    actual = AnnotatedVariantProducer.produceAnnotatedVcFromAssemblyEvidence(del_21_43350116_43353485, simpleNovelAdjacencyAndChimericAlignmentEvidence, referenceBroadcast, refSeqDictBroadcast, cnvCallsBroadcast, "sample").make();
    VariantContextTestUtils.assertVariantContextsAreEqual(actual, expected, Collections.singletonList(HQ_MAPPINGS), Collections.emptyList());
}
 

public SelectNearestCentroid(Broadcast<List<Tuple2<Integer, Point>>> brCenters) {
	this.brCenters = brCenters.getValue();
}
 

/**
 * This constructor specifies to calculate the interaction of 
 * a specific group and a specific atom,.e.g. the atom "O" in
 * the group "HOH".
 * @param filter Specifies the conditions for calculating interactions
 * @param pairwise If true, results as one row per pair interaction, otherwise
 * the interactions of one atom with all other atoms are returned as a single row.
 */
public StructureToAtomInteractions(Broadcast<InteractionFilter> bfilter, boolean pairwise) {
	this.filter = bfilter.getValue();
	this.pairwise = pairwise;
}