• Example Search
• Project Search

• Java
• Python
• JavaScript
• TypeScript
• C++
• Scala
• Blog

• sddf-master
  • src
    • main
      • resources
        • log4j.properties.template
        • project-default.properties
      • scala
        • de
          • unihamburg
            • vsis
              • sddf
                • print
                  • PipePrintHead.scala
                  • PipePrintHeadTuple.scala
                • writing
                  • PipeWriterTuple.scala
                  • PipeWriterTuplePairs.scala
                  • PipeWriterTupleClusterActualDate.scala
                  • ClusterWriterCsvFile.scala
                  • ResultWriterFile.scala
                  • TupleWriterFile.scala
                  • ClusterWriterCsvFileActualDate.scala
                  • PipeWriterTupleCluster.scala
                • sparkextensions
                  • RddSequenceFunctions.scala
                  • RddUtils.scala
                • shell
                  • ShellPrerequesites.scala
                  • MusicbrainzTemplate.scash
                  • BtwDemo.scash
                  • thesisExample.scash
                • examples
                  • PipeWordcount.scala
                • classification
                  • PipeClassificationSvm.scala
                  • PipePrintHeadFalsePositives.scala
                  • AbstractPipeClassification.scala
                  • AbstractPipePrintFalseTuples.scala
                  • PipeClassificationTrainingDataGenerator.scala
                  • PipePrintSampleFalseNegatives.scala
                  • PipeClassificationNaiveBayes.scala
                  • PipeClassificationDecisionTree.scala
                  • PipeAnalyseClassification.scala
                  • PipePrintSampleFalsePositives.scala
                  • PipePrintHeadFalseNegatives.scala
                  • PipeClassificationThreshold.scala
                  • PipeAnalyseClassificationTraining.scala
                • visualisation
                  • ModelRouterSilent.scala
                  • ModelRouter.scala
                  • Table.scala
                  • PipeSampler.scala
                  • model
                    • ClassificationModel.scala
                    • TrainingSetModel.scala
                    • RecallAndPrecision.scala
                    • GoldstandardModel.scala
                    • BlockingModel.scala
                    • BasicAnalysable.scala
                    • IndexingModelExtended.scala
                    • ReadingModel.scala
                    • GoldstandardClusterModel.scala
                    • Analysable.scala
                    • ClusterModel.scala
                    • AlgoAnalysable.scala
                    • IndexingModel.scala
                  • logger
                    • ClassificationOutputter.scala
                    • ReadingOutputter.scala
                    • GoldstandardOutputter.scala
                    • BlockingOutputter.scala
                    • BasicOutputter.scala
                    • AlgoOutputter.scala
                    • GoldstandardClusterOutputter.scala
                    • ModelRouterLogging.scala
                    • ClusterOutputter.scala
                    • Outputter.scala
                    • TrainingSetOutputter.scala
                    • IndexingOutputter.scala
                    • IndexingOutputterExtended.scala
                • clustering
                  • PipeAnalyseClustering.scala
                  • PipeClusteringTransitiveClosure.scala
                  • PipeClusteringStrongestPath.scala
                  • AbstractPipeClusteringGraph.scala
                • config
                  • Config.scala
                • indexing
                  • PipeIndexerSuffixArray.scala
                  • IndexingPipe.scala
                  • PipeIndexerSortedNeighborhood.scala
                  • PipeAnalyseIndexerExtended.scala
                  • PipeIndexerStandard.scala
                  • blocking
                    • PipeAnalyseBlocker.scala
                    • PipeBlockerStandard.scala
                    • PipeBlockerSuffixArray.scala
                    • keygeneration
                      • BlockingKeyBuilderBasic.scala
                      • BlockingKeyBuilder.scala
                    • PipeBlockerSortedNeighborhood.scala
                    • PipeBlockerPrefixArray.scala
                    • BlockingPipe.scala
                  • DummyIndexer.scala
                  • PipeAnalyseIndexer.scala
                • SddfContext.scala
                • convert
                  • PipeConvertClusterToPair.scala
                  • PipeConvertSetToPair.scala
                • Parameterized.scala
                • analyze
                  • PipeAnalyzeCount.scala
                • reading
                  • corpus
                    • AbstractPipeReaderTuple.scala
                    • PipePrintSampleCorpus.scala
                    • PipeStoreInContextCorpus.scala
                    • PipeAnalyseCorpus.scala
                    • PipePrintHeadCorpus.scala
                    • PipeReaderTupleFixedWidth.scala
                    • PipeContextReadCorpus.scala
                    • PipeReaderTupleCsv.scala
                  • TupleHashMap.scala
                  • TupleArray.scala
                  • PipeReaderOmitTail.scala
                  • IdConverterHex.scala
                  • SymPair.scala
                  • goldstandard
                    • PipePrintHeadGoldstandard.scala
                    • PipeReaderGoldstandardClusterOutput.scala
                    • PipeReaderGoldstandardIdsPairs.scala
                    • PipeAnalyseGoldstandard.scala
                    • PipeReaderGoldstandardIdToTuple.scala
                    • PipeStoreInContextGoldstandard.scala
                    • PipeReaderGoldstandard.scala
                    • PipePrintSampleGoldstandard.scala
                    • PipeReaderGoldstandardIdsCluster.scala
                    • PipeAnalyseGoldstandardCluster.scala
                  • PipeReaderOmitHead.scala
                  • FeatureIdNameMapping.scala
                  • IdConverter.scala
                  • IdConverterBasic.scala
                  • Tuple.scala
                • tuple
                  • TupleSymPairFunctions.scala
                • preprocessing
                  • PipePreprocessorReplaceRegex.scala
                  • TraitPipePreprocessor.scala
                  • PipePreprocessorMerger.scala
                  • PipePreprocessorSplitter.scala
                  • PipePreprocessorToLower.scala
                  • PipePreprocessorTrim.scala
                  • PipePreprocessorRemoveRegex.scala
                • tools
                  • ExactDuplicateFilter.scala
                  • ConvertGoldStandardPairFormat.scala
                • pipe
                  • Pipe.scala
                  • context
                    • ResultContext.scala
                    • AbstractPipeContext.scala
                    • GoldstandardContext.scala
                    • SddfPipeContext.scala
                    • CorpusContext.scala
                  • PipeOptimizer.scala
                  • PipeElementPassthrough.scala
                  • PipeMetaParallel.scala
                  • PipeElement.scala
                  • PipePassthrough.scala
                  • Pipeline.scala
                  • PipeFilterRDD.scala
                  • optimize
                    • PipeOptimizePersistAndName.scala
                    • PipeOptimizeUnpersist.scala
                    • PipeOptimizeCheckpoint.scala
                • filter
                  • PipeFilterDistinct.scala
                • SddfApp.scala
                • logging
                  • EachRunNewFileAppender.scala
                  • Logging.scala
                • similarity
                  • PipeSimilarity.scala
                  • SimilarityCalculator.scala
                  • aggregator
                    • Mean.scala
                    • SimilarityAggregator.scala
                    • Median.scala
                  • measures
                    • MeasureWrapperToLower.scala
                    • MeasureNumeric.scala
                    • MeasureEquality.scala
    • test
      • resources
        • log4j.properties.template
        • musicbrainz-1000.csv.dup
        • musicbrainz-10.csv.dup
        • project-default.properties
      • scala
        • de
          • unihamburg
            • vsis
              • sddf
                • test
                  • sparkextensions
                    • RddSequenceFunctionsTest.scala
                  • examples
                    • WordcountTest.scala
                  • classification
                    • PipeDecisionTest.scala
                    • PipeTrainingDataGeneratorTest.scala
                  • util
                    • FixtureHelper.scala
                    • MusicbrainzSchema.scala
                    • TestSddfPipeContext.scala
                    • LocalSparkContext.scala
                  • clustering
                    • StrongestPathClusteringTest.scala
                  • evaluation
                    • ClusterAnalyserTest.scala
                  • reading
                    • SymPairTest.scala
                    • PipeReaderOmitTailTest.scala
                    • goldstandard
                      • PipeGoldstandardReaderClusterTest.scala
                    • parser
                      • CsvTupleParserTest.scala
                      • FixedWidthTupleParserTest.scala
                  • blocking
                    • BlockingTest.scala
                    • keygeneration
                      • BlockingKeyBuilderBasicTest.scala
                    • SuffixArrayBlockingTest.scala
                    • SortedNeighbourhoodBlockerTest.scala
                  • logging
                    • LoggingTest.scala
                  • SparkApiTest.scala
  • launch.sh
  • LICENSE
  • project
    • build.properties
    • plugins.sbt
  • README.md
  • build.sbt
  • .gitignore

package de.unihamburg.vsis.sddf.writing

import java.io.File

import org.apache.spark.rdd.RDD

import de.unihamburg.vsis.sddf.pipe.PipeElementPassthrough
import de.unihamburg.vsis.sddf.pipe.context.AbstractPipeContext
import de.unihamburg.vsis.sddf.reading.Tuple
import de.unihamburg.vsis.sddf.visualisation.model.BasicAnalysable

class PipeWriterTupleCluster(file: File, separator: Char = ',')
  extends PipeElementPassthrough[RDD[Set[Tuple]]] {

  def substep(input: RDD[Set[Tuple]])(implicit pipeContext: AbstractPipeContext): Unit = {
    val writer = new TupleWriterFile(file, separator)
    // TODO write tuples to hdfs in parallel and merge them afterwards
    val collected = input.collect()
    collected.foreach(set => {
      set.foreach(tuple => {
        writer.writeTuple(tuple)
      })
      writer.blankLine()
    })
    writer.close()
  }

}

object PipeWriterTupleCluster {

  def apply(file: File, separator: Char = ',') = {
    new PipeWriterTupleCluster(file, separator)
  }

}