• Example Search
• Project Search

• Java
• Python
• JavaScript
• TypeScript
• C++
• Scala
• Blog

• sddf-master
  • src
    • main
      • resources
        • log4j.properties.template
        • project-default.properties
      • scala
        • de
          • unihamburg
            • vsis
              • sddf
                • print
                  • PipePrintHead.scala
                  • PipePrintHeadTuple.scala
                • writing
                  • PipeWriterTuple.scala
                  • PipeWriterTuplePairs.scala
                  • PipeWriterTupleClusterActualDate.scala
                  • ClusterWriterCsvFile.scala
                  • ResultWriterFile.scala
                  • TupleWriterFile.scala
                  • ClusterWriterCsvFileActualDate.scala
                  • PipeWriterTupleCluster.scala
                • sparkextensions
                  • RddSequenceFunctions.scala
                  • RddUtils.scala
                • shell
                  • ShellPrerequesites.scala
                  • MusicbrainzTemplate.scash
                  • BtwDemo.scash
                  • thesisExample.scash
                • examples
                  • PipeWordcount.scala
                • classification
                  • PipeClassificationSvm.scala
                  • PipePrintHeadFalsePositives.scala
                  • AbstractPipeClassification.scala
                  • AbstractPipePrintFalseTuples.scala
                  • PipeClassificationTrainingDataGenerator.scala
                  • PipePrintSampleFalseNegatives.scala
                  • PipeClassificationNaiveBayes.scala
                  • PipeClassificationDecisionTree.scala
                  • PipeAnalyseClassification.scala
                  • PipePrintSampleFalsePositives.scala
                  • PipePrintHeadFalseNegatives.scala
                  • PipeClassificationThreshold.scala
                  • PipeAnalyseClassificationTraining.scala
                • visualisation
                  • ModelRouterSilent.scala
                  • ModelRouter.scala
                  • Table.scala
                  • PipeSampler.scala
                  • model
                    • ClassificationModel.scala
                    • TrainingSetModel.scala
                    • RecallAndPrecision.scala
                    • GoldstandardModel.scala
                    • BlockingModel.scala
                    • BasicAnalysable.scala
                    • IndexingModelExtended.scala
                    • ReadingModel.scala
                    • GoldstandardClusterModel.scala
                    • Analysable.scala
                    • ClusterModel.scala
                    • AlgoAnalysable.scala
                    • IndexingModel.scala
                  • logger
                    • ClassificationOutputter.scala
                    • ReadingOutputter.scala
                    • GoldstandardOutputter.scala
                    • BlockingOutputter.scala
                    • BasicOutputter.scala
                    • AlgoOutputter.scala
                    • GoldstandardClusterOutputter.scala
                    • ModelRouterLogging.scala
                    • ClusterOutputter.scala
                    • Outputter.scala
                    • TrainingSetOutputter.scala
                    • IndexingOutputter.scala
                    • IndexingOutputterExtended.scala
                • clustering
                  • PipeAnalyseClustering.scala
                  • PipeClusteringTransitiveClosure.scala
                  • PipeClusteringStrongestPath.scala
                  • AbstractPipeClusteringGraph.scala
                • config
                  • Config.scala
                • indexing
                  • PipeIndexerSuffixArray.scala
                  • IndexingPipe.scala
                  • PipeIndexerSortedNeighborhood.scala
                  • PipeAnalyseIndexerExtended.scala
                  • PipeIndexerStandard.scala
                  • blocking
                    • PipeAnalyseBlocker.scala
                    • PipeBlockerStandard.scala
                    • PipeBlockerSuffixArray.scala
                    • keygeneration
                      • BlockingKeyBuilderBasic.scala
                      • BlockingKeyBuilder.scala
                    • PipeBlockerSortedNeighborhood.scala
                    • PipeBlockerPrefixArray.scala
                    • BlockingPipe.scala
                  • DummyIndexer.scala
                  • PipeAnalyseIndexer.scala
                • SddfContext.scala
                • convert
                  • PipeConvertClusterToPair.scala
                  • PipeConvertSetToPair.scala
                • Parameterized.scala
                • analyze
                  • PipeAnalyzeCount.scala
                • reading
                  • corpus
                    • AbstractPipeReaderTuple.scala
                    • PipePrintSampleCorpus.scala
                    • PipeStoreInContextCorpus.scala
                    • PipeAnalyseCorpus.scala
                    • PipePrintHeadCorpus.scala
                    • PipeReaderTupleFixedWidth.scala
                    • PipeContextReadCorpus.scala
                    • PipeReaderTupleCsv.scala
                  • TupleHashMap.scala
                  • TupleArray.scala
                  • PipeReaderOmitTail.scala
                  • IdConverterHex.scala
                  • SymPair.scala
                  • goldstandard
                    • PipePrintHeadGoldstandard.scala
                    • PipeReaderGoldstandardClusterOutput.scala
                    • PipeReaderGoldstandardIdsPairs.scala
                    • PipeAnalyseGoldstandard.scala
                    • PipeReaderGoldstandardIdToTuple.scala
                    • PipeStoreInContextGoldstandard.scala
                    • PipeReaderGoldstandard.scala
                    • PipePrintSampleGoldstandard.scala
                    • PipeReaderGoldstandardIdsCluster.scala
                    • PipeAnalyseGoldstandardCluster.scala
                  • PipeReaderOmitHead.scala
                  • FeatureIdNameMapping.scala
                  • IdConverter.scala
                  • IdConverterBasic.scala
                  • Tuple.scala
                • tuple
                  • TupleSymPairFunctions.scala
                • preprocessing
                  • PipePreprocessorReplaceRegex.scala
                  • TraitPipePreprocessor.scala
                  • PipePreprocessorMerger.scala
                  • PipePreprocessorSplitter.scala
                  • PipePreprocessorToLower.scala
                  • PipePreprocessorTrim.scala
                  • PipePreprocessorRemoveRegex.scala
                • tools
                  • ExactDuplicateFilter.scala
                  • ConvertGoldStandardPairFormat.scala
                • pipe
                  • Pipe.scala
                  • context
                    • ResultContext.scala
                    • AbstractPipeContext.scala
                    • GoldstandardContext.scala
                    • SddfPipeContext.scala
                    • CorpusContext.scala
                  • PipeOptimizer.scala
                  • PipeElementPassthrough.scala
                  • PipeMetaParallel.scala
                  • PipeElement.scala
                  • PipePassthrough.scala
                  • Pipeline.scala
                  • PipeFilterRDD.scala
                  • optimize
                    • PipeOptimizePersistAndName.scala
                    • PipeOptimizeUnpersist.scala
                    • PipeOptimizeCheckpoint.scala
                • filter
                  • PipeFilterDistinct.scala
                • SddfApp.scala
                • logging
                  • EachRunNewFileAppender.scala
                  • Logging.scala
                • similarity
                  • PipeSimilarity.scala
                  • SimilarityCalculator.scala
                  • aggregator
                    • Mean.scala
                    • SimilarityAggregator.scala
                    • Median.scala
                  • measures
                    • MeasureWrapperToLower.scala
                    • MeasureNumeric.scala
                    • MeasureEquality.scala
    • test
      • resources
        • log4j.properties.template
        • musicbrainz-1000.csv.dup
        • musicbrainz-10.csv.dup
        • project-default.properties
      • scala
        • de
          • unihamburg
            • vsis
              • sddf
                • test
                  • sparkextensions
                    • RddSequenceFunctionsTest.scala
                  • examples
                    • WordcountTest.scala
                  • classification
                    • PipeDecisionTest.scala
                    • PipeTrainingDataGeneratorTest.scala
                  • util
                    • FixtureHelper.scala
                    • MusicbrainzSchema.scala
                    • TestSddfPipeContext.scala
                    • LocalSparkContext.scala
                  • clustering
                    • StrongestPathClusteringTest.scala
                  • evaluation
                    • ClusterAnalyserTest.scala
                  • reading
                    • SymPairTest.scala
                    • PipeReaderOmitTailTest.scala
                    • goldstandard
                      • PipeGoldstandardReaderClusterTest.scala
                    • parser
                      • CsvTupleParserTest.scala
                      • FixedWidthTupleParserTest.scala
                  • blocking
                    • BlockingTest.scala
                    • keygeneration
                      • BlockingKeyBuilderBasicTest.scala
                    • SuffixArrayBlockingTest.scala
                    • SortedNeighbourhoodBlockerTest.scala
                  • logging
                    • LoggingTest.scala
                  • SparkApiTest.scala
  • launch.sh
  • LICENSE
  • project
    • build.properties
    • plugins.sbt
  • README.md
  • build.sbt
  • .gitignore

package de.unihamburg.vsis.sddf.tools

import java.io.File

import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
import org.apache.spark.rdd.RDD

import de.unihamburg.vsis.sddf.logging.Logging
import de.unihamburg.vsis.sddf.pipe.context.SddfPipeContext
import de.unihamburg.vsis.sddf.reading.FeatureIdNameMapping
import de.unihamburg.vsis.sddf.reading.FeatureIdNameMapping.Id
import de.unihamburg.vsis.sddf.reading.corpus.PipeStoreInContextCorpus
import de.unihamburg.vsis.sddf.reading.corpus.PipePrintSampleCorpus
import de.unihamburg.vsis.sddf.reading.Tuple
import de.unihamburg.vsis.sddf.reading.corpus.PipeReaderTupleCsv
import de.unihamburg.vsis.sddf.writing.TupleWriterFile

/**
*	Does not work at the moment.
*	I used gnu sort to achieve uniquenes.
*/
object ExactDuplicateFilter extends App with Logging {

  if (args.size == 1 && (new File(args(0))).exists()) {
    val conf = new SparkConf().setAppName("ExactDuplicateFilter")
    conf.setMaster("local")
    val sc = new SparkContext(conf)

    implicit val pipeContext = new SddfPipeContext
      
    val Content: (Int, String) = (0, "content")

    val featureMapping: Map[Int, String] = Map(Content)

    implicit val featureIdNameMapper = new FeatureIdNameMapping(featureMapping)

    val inputFileKey = "musicbrainz"

    // Parse Tuples
    val allFields: Seq[Int] = Seq(Content._1)
    val allFieldsWithId: Seq[Int] = Id +: allFields

    val parserPipe = new PipeTupleParserCsvIdContent(allFieldsWithId)
    val pipe = parserPipe.append(PipeStoreInContextCorpus()).append(PipePrintSampleCorpus())
    pipe.start(sc.textFile(args(0)))
    val result: RDD[Tuple] = parserPipe.output.get
    val resultCount = result.count
    log.info("Lines parsed: " + resultCount)
    
    val distinct = result.distinct()
    val distinctCount = distinct.count
    log.info("Distinct Lines Count: " + distinctCount)
    log.info("Lines removed: " + (resultCount - distinctCount))
    
    val tupleWriter = new TupleWriterFile(new File(args(0) + ".distinct"))
    tupleWriter.writeTuple(distinct)

  } else {
    println("Please provide a valid file path.")
  }

}

class PipeTupleParserCsvIdContent(featureIds: Seq[Int]) extends PipeReaderTupleCsv(featureIds) {
  override def extractValues(line: String): Seq[String] = {
    val splitted = parser.parseLine(line)
    Seq(splitted.head, splitted.tail.mkString(","))
  }
}