Java Code Examples for org.apache.log4j.BasicConfigurator#configure()

@BeforeSuite
public void setProps() {
    BasicConfigurator.resetConfiguration();
    BasicConfigurator.configure(new ConsoleAppender(new PatternLayout("%d{HH:mm:ss,SSS} [%t] %p %c %x - %m%n")));
    Logger.getRootLogger().setLevel(Level.INFO);
    Logger.getLogger(SimpleEVCacheTest.class).setLevel(Level.DEBUG);
    Logger.getLogger(Base.class).setLevel(Level.DEBUG);
    Logger.getLogger(EVCacheImpl.class).setLevel(Level.DEBUG);
    Logger.getLogger(EVCacheClient.class).setLevel(Level.DEBUG);
    Logger.getLogger(EVCacheClientPool.class).setLevel(Level.DEBUG);

    final Properties props = getProps();
    props.setProperty("EVCACHE_TEST.use.simple.node.list.provider", "true");
    props.setProperty("EVCACHE_TEST.EVCacheClientPool.readTimeout", "1000");
    props.setProperty("EVCACHE_TEST.EVCacheClientPool.bulkReadTimeout", "1000");
    props.setProperty("EVCACHE_TEST.max.read.queue.length", "100");
    props.setProperty("EVCACHE_TEST.operation.timeout", "10000");
    props.setProperty("EVCACHE_TEST.throw.exception", "false");

    int maxThreads = 2;
    final BlockingQueue<Runnable> queue = new LinkedBlockingQueue<Runnable>(100000);
    pool = new ThreadPoolExecutor(maxThreads * 4, maxThreads * 4, 30, TimeUnit.SECONDS, queue);
    pool.prestartAllCoreThreads();
}
 

public static void main(String[] args) throws InterruptedException {
    // Configure console logging
    BasicConfigurator.configure();

    String topic = "publishing_to_topic";
    if (args.length > 0) {
        topic = args[0];
    }
    System.out.println("Subscribing to topic " + topic);

    // Subscribe to the topic
    PubSub.subscribe(topic, new SubscribeToolSubscriber());

    // Wait for the client to be interrupted
    PubSub.client().join();
}
 

public static void main(String[] args) {
        BasicConfigurator.configure();

//        String mainDir = "/home/gpapadakis/data/ccerDatasets/allDatasets/";
        String mainDir = "/home/gpapadakis/data/datasets/";
        String datasetsD1 = "cleanDBPedia1";
        String datasetsD2 = "cleanDBPedia2";
        String groundtruthDirs = "newDBPediaMatches";

        final IEntityReader eReader1 = new EntitySerializationReader(mainDir + datasetsD1);
        final List<EntityProfile> profiles1 = eReader1.getEntityProfiles();
        System.out.println("\n\nInput Entity Profiles D1\t:\t" + profiles1.size());

        final IEntityReader eReader2 = new EntitySerializationReader(mainDir + datasetsD2);
        final List<EntityProfile> profiles2 = eReader2.getEntityProfiles();
        System.out.println("\n\nInput Entity Profiles D2\t:\t" + profiles2.size());

        IGroundTruthReader gtReader = new GtSerializationReader(mainDir + groundtruthDirs);
        final AbstractDuplicatePropagation duplicatePropagation = new BilateralDuplicatePropagation(gtReader.getDuplicatePairs(null));
        System.out.println("Existing Duplicates\t:\t" + duplicatePropagation.getDuplicates().size());

        for (float simThreshold = 0.7f; simThreshold < 0.99; simThreshold += 0.1) {
            long time1 = System.currentTimeMillis();
//            AllPairs join = new AllPairs(simThreshold[datasetId]);
            PPJoin join = new PPJoin(simThreshold);
            SimilarityPairs simPairs = join.executeFiltering("name", "name", profiles1, profiles2);
            System.out.println(simPairs.getNoOfComparisons());

            final IEntityClustering ec = new UniqueMappingClustering(simThreshold);
            final EquivalenceCluster[] clusters = ec.getDuplicates(simPairs);

            long time2 = System.currentTimeMillis();

            final ClustersPerformance clp = new ClustersPerformance(clusters, duplicatePropagation);
            clp.setStatistics();
            clp.printStatistics(0, "", "");

            System.out.println("Running time\t:\t" + (time2 - time1));
        }
    }
 

public static void main(String[] args) throws Exception {
    System.setProperty("org.apache.commons.logging.Log", "org.apache.commons.logging.impl.NoOpLog");
    org.apache.log4j.Logger logger4j = org.apache.log4j.Logger.getRootLogger();
    logger4j.setLevel(org.apache.log4j.Level.toLevel("INFO"));

    BasicConfigurator.configure();

    SimpleTests test = new SimpleTests(args);

    if (test.help) {
        test.jCommander.usage();
        return;
    }

    test.validateOperation(test.operation);

    runId = System.nanoTime();

    switch ( test.operationName ) {
        case READ_THROUGHPUT:
            test.readThroughput();
            break;
        case WRITE_THROUGHPUT:
            test.writeThroughput();
            break;
        case READ_IDS:
            test.readIds();
            break;
        default:
            System.err.println("Operation name (-o) must be createTable, deleteTable, listTables, or loadTable");
            break;
    }

    reporter.report();
    reporter.close();
    if (client != null) {
        client.close();
    }
}
 

@BeforeClass
public static void initLog4j() {
    BasicConfigurator.resetConfiguration();
    BasicConfigurator.configure();

    final Layout LAYOUT = new PatternLayout("[ %d{ISO8601} %-5p ] [%c{1}] %m%n");

    LevelRangeFilter ERROR_FILTER = new LevelRangeFilter();
    ERROR_FILTER.setLevelMin(Level.ERROR);
    ERROR_FILTER.setLevelMax(Level.FATAL);

    // everything including ERROR
    final Appender STDOUT = new ConsoleAppender(LAYOUT, ConsoleAppender.SYSTEM_OUT);

    // just things <= ERROR
    final Appender STDERR = new ConsoleAppender(LAYOUT, ConsoleAppender.SYSTEM_ERR);
    STDERR.addFilter(ERROR_FILTER);

    final Logger ROOT_LOGGER = Logger.getRootLogger();

    ROOT_LOGGER.removeAllAppenders();

    ROOT_LOGGER.setLevel(Level.WARN); // don't care about higher

    ROOT_LOGGER.addAppender(STDOUT);
    ROOT_LOGGER.addAppender(STDERR);
}
 

@Test
public void retriedRequest_AppendsCorrectRetryCountInUserAgent() throws Exception {
    BasicConfigurator.configure();
    stubFor(get(urlEqualTo(RESOURCE_PATH)).willReturn(aResponse().withStatus(500)));

    executeRequest();

    verify(1, getRequestedFor(urlEqualTo(RESOURCE_PATH)).withHeader(HEADER_SDK_RETRY_INFO, containing("0/0/")));
    verify(1, getRequestedFor(urlEqualTo(RESOURCE_PATH)).withHeader(HEADER_SDK_RETRY_INFO, containing("1/0/")));
    verify(1, getRequestedFor(urlEqualTo(RESOURCE_PATH)).withHeader(HEADER_SDK_RETRY_INFO, containing("2/10/")));
    verify(1, getRequestedFor(urlEqualTo(RESOURCE_PATH)).withHeader(HEADER_SDK_RETRY_INFO, containing("3/20/")));
}
 

@Before
public void setUp() throws Exception {
    BasicConfigurator.configure();

    Injector injector = Guice.createInjector(new SwaggerApiImporterTestModule());
    client = injector.getInstance(ApiGatewaySdkApiImporter.class);
}
 

public static void main(String[] args) {
	try {
		BasicConfigurator.configure();

		ApplicationProperties.getDefaultProperties().setProperty(
				ApplicationProperty.CustomizationExternalTerm.key(),
				BannerTermProvider.class.getName());
		ApplicationProperties.getDefaultProperties().setProperty(
				ApplicationProperty.CustomizationDefaultCourseUrl.key(),
				"https://selfservice.mypurdue.purdue.edu/prod/bzwsrch.p_catalog_detail?term=:xterm&subject=:xsubject&cnbr=:xcourseNbr&enhanced=Y");
		ApplicationProperties.getDefaultProperties().setProperty(
				ApplicationProperty.CustomizationDefaultCourseDetailsDownload.key(),
				"true");
		ApplicationProperties.getDefaultProperties().setProperty(
				ApplicationProperty.CustomizationDefaultCourseDetailsContent.key(),
				"(?idm)(<table [ ]*class=\"[a-z]*\" summary=\"This table lists the course detail for the selected term.\" .*)<table [ ]*class=\"[a-z]*\" summary=\"This is table displays line separator at end of the page.\"");
		ApplicationProperties.getDefaultProperties().setProperty(
				ApplicationProperty.CustomizationDefaultCourseDetailsModifiers.key(),
				"(?i)<a href=\"[^>]*\">\n<b>\n"+
				"(?i)</a>\n</b>\n"+
				"(?i)<span class=[\"]?fieldlabeltext[\"]?>\n<b>\n"+
				"(?i)</span>\n</b>\n"+
				"(?i) class=\"nttitle\" \n class=\"unitime-MainTableHeader\" \n"+
				"(?i) class=\"datadisplaytable\" \n class=\"unitime-MainTable\" ");
		
		System.out.println(
				"URL:" + new DefaultCourseDetailsProvider().getCourseUrl(new AcademicSessionInfo(-1l, "2010", "Spring", "PWL"), "AAE", "20300A")
				);
		
		System.out.println(
				"Details:\n" +
				new DefaultCourseDetailsProvider().getDetails(new AcademicSessionInfo(-1l, "2010", "Spring", "PWL"), "AAE", "20300A")
				);
		
	} catch (Exception e) {
		e.printStackTrace();
	}
}
 

/** Sets up the log to ease debugging. */
protected void setUpLog() {
    // Useful if you need to get some info when debugging
    BasicConfigurator.configure();
    ConsoleAppender ca = new ConsoleAppender();
    ca.setWriter(new OutputStreamWriter(System.out));
    ca.setLayout(new PatternLayout("%-5p [%t]: %m%n"));
    Logger.getRootLogger().addAppender(ca);
    Logger.getRootLogger().setLevel(Level.DEBUG);
}
 

@BeforeClass
public static void classInit() {
    CentralPAPropertyRepository.PA_CLASSLOADING_USEHTTP.setValue(false);
    if (System.getProperty("log4j.configuration") == null) {
        // While logger is not configured and it not set with sys properties, use Console logger
        Logger.getRootLogger().getLoggerRepository().resetConfiguration();
        BasicConfigurator.configure(new ConsoleAppender(new PatternLayout("%m%n")));
        Logger.getRootLogger().setLevel(Level.INFO);
    }
    Logger.getLogger(AOSynchronization.class).setLevel(Level.TRACE);
}
 

@Override
public void start(BundleContext context) throws Exception {
	BasicConfigurator.configure();
	Saludo saludo = new Saludo();
	saludo.setSaludo("Hola");
	saludo.saluda();
}
 

public static void main(String[] args) {
        BasicConfigurator.configure();

        String fileName = "/home/gap2/data/JedAIdata/blocks/dblpAcm/sb";
        float bpRatio = 0.004f;
        float bfRatio = 0.5000000000000003f;

        final EntitySerializationReader inReader = new EntitySerializationReader(null);
        final List<AbstractBlock> blocks = (List<AbstractBlock>) inReader.loadSerializedObject(fileName);

        final SizeBasedBlockPurging sbbp = new SizeBasedBlockPurging(bpRatio);
        final List<AbstractBlock> purgedBlocks = sbbp.refineBlocks(blocks);

        final BlockFiltering bf = new BlockFiltering(bfRatio);
        final List<AbstractBlock> filteredBlocks = bf.refineBlocks(purgedBlocks);

        final IGroundTruthReader gtReader = new GtSerializationReader("/home/gap2/data/JedAIdata/datasets/cleanCleanErDatasets/dblpAcmIdDuplicates");
        final Set<IdDuplicates> duplicatePairs = gtReader.getDuplicatePairs(null);
        final AbstractDuplicatePropagation adp = new BilateralDuplicatePropagation(duplicatePairs);

        BlocksPerformance blockStats = new BlocksPerformance(filteredBlocks, adp);
        blockStats.setStatistics();
        blockStats.printStatistics(0, "", "");
        final float originalPC = blockStats.getPc();
        System.out.println("Original PC\t:\t" + originalPC);

//        final BLAST blast = new BLAST();
        final WeightedNodePruning blast = new WeightedNodePruning();
        final List<AbstractBlock> newBlocks = blast.refineBlocks(filteredBlocks);
        blockStats = new BlocksPerformance(newBlocks, adp);
        blockStats.setStatistics();
        blockStats.printStatistics(0, "", "");
        final float newPC = blockStats.getPc();
        System.out.println("New PC\t:\t" + newPC);
    }
 

/** Run statistics, then upload to the server */
public void run() {
	final long start_time = System.nanoTime();
	
       BasicConfigurator.configure();
       Logger.getRootLogger().setLevel(Level.ERROR);
	
	System.out.println("Performing train/test session\n"
			+ "    #=top    o=top3    .=recall    ' '=missing");
	ConcurrentHashMap<Long, DefaultPipeline> pipes =
			new ConcurrentHashMap<>();
	
	int[] all_ranks = questionsource.parallelStream().mapToInt(q -> {
		long tid = Thread.currentThread().getId();
		DefaultPipeline pipe = pipes.computeIfAbsent(tid, (i) -> new DefaultPipeline());
		
		List<Answer> answers;
		try{
			answers = pipe.ask(q, message -> {});
		} catch (Exception e) {
			log.fatal(e, e);
			return 99;
		}
		
		int tq = total_questions.incrementAndGet();
		if (tq % 50 == 0) {
			System.out.println(
				String.format(
					"[%d]: %d (%.02f%%) accurate",
					total_questions.get(),
					total_correct.get(),
					total_correct.get() * 100.0 / total_questions.get()));
		}
		
		int correct_rank = 99;
		
		if (answers.size() == 0) {
			System.out.print('!');
			return 99;
		}
		
		for (int rank=0; rank<answers.size(); rank++) {
			Answer candidate = answers.get(rank);
			if (candidate.scores.get("CORRECT") > 0.99) {
				total_inverse_rank.addAndGet(1 / ((double)rank + 1));
				available.incrementAndGet();
				if (rank < 100) correct_rank = rank;
				break;
			}
		}
		if (correct_rank == 0) {
			total_correct.incrementAndGet();
			System.out.print('#');
		} else if (correct_rank < 3) {
			System.out.print('o');
		} else if (correct_rank < 99) {
			System.out.print('.');
		} else {
			System.out.print(' ');
		}
		
		total_answers += answers.size();
		//System.out.println("Q: " + text.question + "\n" +
		//		"A[Guessed: " + top_answer.getScore() + "]: " + top_answer.getTitle() + "\n" +
		//		"A[Actual:" + correct_answer_score + "]: "  + text.answer);
		return correct_rank;
	}).mapToObj(x -> {int[] xs = new int[100]; xs[x] = 1; return xs;}).reduce(new int[100], StatsGenerator::add);

	// Only count the rank of questions that were actually there
	// This is not atomic but by now only one is running
	total_inverse_rank.set(total_inverse_rank.doubleValue() / available.doubleValue());
	// Finish the timing
	runtime = System.nanoTime() - start_time;
	runtime /= 1e9;
	report(all_ranks);
}
 

public static void main(String[] args) throws FileNotFoundException {
    BasicConfigurator.configure();
    
    String mainDirectory = "data/cleanCleanErDatasets/";

    EntityRDFReader rdfEntityReader = new EntityRDFReader(mainDirectory + "DBLP2toRdf.xml");
    List<EntityProfile> rdfDBLP = rdfEntityReader.getEntityProfiles();
    System.out.println("RDF DBLP Entity Profiles\t:\t" + rdfDBLP.size());
    
    rdfEntityReader = new EntityRDFReader(mainDirectory + "ACMtoRdf.xml");
    List<EntityProfile> rdfACM = rdfEntityReader.getEntityProfiles();
    System.out.println("RDF ACM Entity Profiles\t:\t" + rdfACM.size());

    GtRdfCsvReader csvGtReader = new GtRdfCsvReader(mainDirectory + "DBLP-ACM" + File.separator + "DBLP-ACM_perfectMapping.csv");
    csvGtReader.setIgnoreFirstRow(true);
    csvGtReader.setSeparator(",");
    csvGtReader.getDuplicatePairs(rdfDBLP, rdfACM);
    final AbstractDuplicatePropagation duplicatePropagation = new BilateralDuplicatePropagation(csvGtReader.getDuplicatePairs(rdfDBLP, rdfACM));
    System.out.println("Existing Duplicates\t:\t" + duplicatePropagation.getDuplicates().size());

    StringBuilder workflowConf = new StringBuilder();
    StringBuilder workflowName = new StringBuilder();

    float time1 = System.currentTimeMillis();

    IBlockBuilding tokenBlocking = new StandardBlocking();
    List<AbstractBlock> blocks = tokenBlocking.getBlocks(rdfDBLP, rdfACM);
    workflowConf.append(tokenBlocking.getMethodConfiguration());
    workflowName.append(tokenBlocking.getMethodName());

    IBlockProcessing blockCleaning = new BlockFiltering();
    List<AbstractBlock> bcBlocks = blockCleaning.refineBlocks(blocks);
    workflowConf.append("\n").append(blockCleaning.getMethodConfiguration());
    workflowName.append("->").append(blockCleaning.getMethodName());
    
    IBlockProcessing comparisonCleaning = new CardinalityNodePruning();
    List<AbstractBlock> ccBlocks = comparisonCleaning.refineBlocks(bcBlocks);
    workflowConf.append("\n").append(comparisonCleaning.getMethodConfiguration());
    workflowName.append("->").append(comparisonCleaning.getMethodName());
            
    float time2 = System.currentTimeMillis();

    BlocksPerformance blStats = new BlocksPerformance(ccBlocks, duplicatePropagation);
    blStats.setStatistics();
    blStats.printStatistics(time2 - time1, workflowConf.toString(), workflowName.toString());
    
    float time3 = System.currentTimeMillis();
    
    IEntityMatching entityMatching = new ProfileMatcher(rdfDBLP, rdfACM);
    SimilarityPairs simPairs = entityMatching.executeComparisons(blocks);
    workflowConf.append("\n").append(entityMatching.getMethodConfiguration());
    workflowName.append("->").append(entityMatching.getMethodName());

    IEntityClustering entityClusttering = new ConnectedComponentsClustering();
    EquivalenceCluster[] entityClusters = entityClusttering.getDuplicates(simPairs);
    workflowConf.append("\n").append(entityClusttering.getMethodConfiguration());
    workflowName.append("->").append(entityClusttering.getMethodName());

    float time4 = System.currentTimeMillis();

    ClustersPerformance clp = new ClustersPerformance(entityClusters, duplicatePropagation);
    clp.setStatistics();
    clp.printStatistics(time4 - time3, workflowName.toString(), workflowConf.toString());
    
    PrintToFile.toCSV(rdfDBLP, rdfACM, entityClusters, mainDirectory + "foundMatchesRDF.csv");
}
 

@Test
public void testProcess() {
    BasicConfigurator.configure();

    AdvancedParams params = new AdvancedParams();
    ConnectOptions connectOptions = new ConnectOptions();
    connectOptions.setConnectTimeout(3000);
    connectOptions.setAcceptTimeout(5000);
    // Concurrent DICOM operations
    connectOptions.setMaxOpsInvoked(15);
    connectOptions.setMaxOpsPerformed(15);
    params.setConnectOptions(connectOptions);

    DicomNode calling = new DicomNode("WEASIS-SCU");
    DicomNode called = new DicomNode("DICOMSERVER", "dicomserver.co.uk", 11112);
    DicomNode scpNode = new DicomNode("DICOMLISTENER", "localhost", 11113);
    DicomNode destination = new DicomNode("FWD-AET", "localhost", 11113);

    Map<ForwardDicomNode, List<ForwardDestination>> destinations = new HashMap<>();
    Attributes attrs = new Attributes();
    attrs.setString(Tag.PatientName, VR.PN, "Override^Patient^Name");
    attrs.setString(Tag.PatientID, VR.LO, "ModifiedPatientID");
    DefaultAttributeEditor editor = new DefaultAttributeEditor(true, attrs);

    DicomProgress progress = new DicomProgress();
    progress.addProgressListener(p -> {
        if (p.isLastFailed()) {
            System.out.println("Failed: DICOM Status:" + p.getStatus());
        }
    });
    List<ForwardDestination> list = new ArrayList<>();
    ForwardDicomNode fwdSrcNode = new ForwardDicomNode(destination.getAet());
    fwdSrcNode.addAcceptedSourceNode(calling.getAet(), "localhost");
    WebForwardDestination web = new WebForwardDestination(fwdSrcNode,
        "http://localhost:8080/dcm4chee-arc/aets/DCM4CHEE/rs/studies", progress, editor);
    list.add(web);
    destinations.put(fwdSrcNode, list);

    GatewayParams gparams = new GatewayParams(params, false, null, GatewayParams.getAcceptedCallingAETitles(destinations));

    DicomGateway gateway;
    try {
        gateway = new DicomGateway(destinations);
        gateway.start(scpNode, gparams);
    } catch (Exception e) {
        e.printStackTrace();
    }

    DicomProgress progress2 = new DicomProgress();
    progress2.addProgressListener(progress1 -> {
        System.out.println("DICOM Status:" + progress1.getStatus());
        System.out.println("NumberOfRemainingSuboperations:" + progress1.getNumberOfRemainingSuboperations());
        System.out.println("NumberOfCompletedSuboperations:" + progress1.getNumberOfCompletedSuboperations());
        System.out.println("NumberOfFailedSuboperations:" + progress1.getNumberOfFailedSuboperations());
        System.out.println("NumberOfWarningSuboperations:" + progress1.getNumberOfWarningSuboperations());
        if (progress1.isLastFailed()) {
            System.out.println("Last file has failed:" + progress1.getProcessedFile());
        }
    });

    String studyUID = "1.2.826.0.1.3680043.11.111";
    DicomState state = CGetForward.processStudy(params, params, calling, called, destination, progress2, studyUID);
    // String seriesUID = "1.2.528.1.1001.100.3.3865.6101.93503564261.20070711142700388";
    // DicomState state = CGetForward.processSeries(params, params, calling, called, destination, progress2, seriesUID);

    // Force to write endmarks and stop the connection
    web.stop();

    // Should never happen
    Assert.assertNotNull(state);

    System.out.println("DICOM Status for retrieving:" + state.getStatus());
    // see org.dcm4che3.net.Status
    // See server log at http://dicomserver.co.uk/logs/
    Assert.assertThat(state.getMessage(), state.getStatus(), IsEqual.equalTo(Status.Pending));
    
    System.out.println("DICOM Status for forwarding:" + web.getState().getStatus());
    Assert.assertThat(web.getState().getMessage(), web.getState().getStatus(), IsEqual.equalTo(Status.Pending));
    
}
 

public static void main(String[] args) {
        BasicConfigurator.configure();

        float[] bestThresholds = {0.75f, 0.45f};
        RepresentationModel[] bestModels = {RepresentationModel.CHARACTER_BIGRAM_GRAPHS, RepresentationModel.CHARACTER_BIGRAMS_TF_IDF};
        SimilarityMetric[] bestMetrics = {SimilarityMetric.GRAPH_OVERALL_SIMILARITY, SimilarityMetric.GENERALIZED_JACCARD_SIMILARITY};

        String mainDir = "/home/gap2/data/JedAIdata/datasets/dirtyErDatasets/";
        String[] profilesFile = {"cddbProfiles", "coraProfiles"};
        String[] groundtruthFile = {"cddbIdDuplicates", "coraIdDuplicates"};

        for (int i = 0; i < groundtruthFile.length; i++) {
            IEntityReader eReader = new EntitySerializationReader(mainDir + profilesFile[i]);
            List<EntityProfile> profiles = eReader.getEntityProfiles();
            System.out.println("Input Entity Profiles\t:\t" + profiles.size());

            IGroundTruthReader gtReader = new GtSerializationReader(mainDir + groundtruthFile[i]);
            final AbstractDuplicatePropagation duplicatePropagation = new UnilateralDuplicatePropagation(gtReader.getDuplicatePairs(null));
            System.out.println("Existing Duplicates\t:\t" + duplicatePropagation.getDuplicates().size());

            IBlockBuilding blockBuildingMethod = new StandardBlocking();
            List<AbstractBlock> blocks = blockBuildingMethod.getBlocks(profiles);
            System.out.println("Original blocks\t:\t" + blocks.size());

            IBlockProcessing blockCleaningMethod1 = new ComparisonsBasedBlockPurging(false);
            blocks = blockCleaningMethod1.refineBlocks(blocks);

            IBlockProcessing blockCleaningMethod2 = new BlockFiltering();
            blocks = blockCleaningMethod2.refineBlocks(blocks);

            IBlockProcessing comparisonCleaningMethod = new CardinalityNodePruning(WeightingScheme.ARCS);
            List<AbstractBlock> cnpBlocks = comparisonCleaningMethod.refineBlocks(blocks);

            List<Comparison> allComparisons = new ArrayList<>();
            for (AbstractBlock block : cnpBlocks) {
                final ComparisonIterator cIterator = block.getComparisonIterator();
                while (cIterator.hasNext()) {
                    allComparisons.add(cIterator.next());
                }
            }
            System.out.println("Total comparisons\t:\t" + allComparisons.size());

            final IEntityMatching emOr = new ProfileMatcher(profiles, bestModels[i], bestMetrics[i]);
            SimilarityPairs originalSims = emOr.executeComparisons(cnpBlocks);
            System.out.println("Executed comparisons\t:\t" + originalSims.getNoOfComparisons());

//            Map<Comparison, Double> comparisonWeight = new HashMap<>();
//            PairIterator pi = originalSims.getPairIterator();
//            while (pi.hasNext()) {
//                Comparison c = pi.next();
//                comparisonWeight.put(c, c.getUtilityMeasure());
//            }
            IEntityClustering ec = new ConnectedComponentsClustering(bestThresholds[i]);
            EquivalenceCluster[] clusters = ec.getDuplicates(originalSims);

            ClustersPerformance clp = new ClustersPerformance(clusters, duplicatePropagation);
            clp.setStatistics();
            clp.printStatistics(0, "", "");
            float originalRecall = clp.getRecall();

            final IEntityMatching em = new ProfileMatcher(profiles, bestModels[i], bestMetrics[i]);
            SimilarityPairs sims = new SimilarityPairs(false, (int) allComparisons.size());

            int counter = 0;
//            int missingComparisons = 0;
            Collections.shuffle(allComparisons);
            for (Comparison comparison : allComparisons) {
                counter++;

                float similarity = em.executeComparison(comparison);
                comparison.setUtilityMeasure(similarity);
//                Double originalSim = comparisonWeight.get(comparison);
//                if (originalSim == null) {
//                    missingComparisons++;
//                } else {
//                    if (!originalSim.equals(similarity)) {
//                        System.out.println(similarity + "\t" + originalSim);
//                    }
//                }
                sims.addComparison(comparison);
//            }
//            System.out.println("Misssing comparisons\t:\t" + missingComparisons);

                ec = new ConnectedComponentsClustering(bestThresholds[i]);
                clusters = ec.getDuplicates(sims);

                clp = new ClustersPerformance(clusters, duplicatePropagation);
                clp.setStatistics();
                float currentRecall = clp.getRecall();
                System.out.println("Current recall\t:\t" + counter + "\t" + currentRecall);
                if (originalRecall <= currentRecall) {
                    clp.printStatistics(0, "", "");
                    break;
                }
            }
        }
    }
 

@BeforeClass
public static void beforeClass() {

    BasicConfigurator.configure();
}
 

public static void main(String[] args) {
  BasicConfigurator.configure();
  Driver driver = new Driver();
  ParameterOverrides overrides = new ParameterOverrides();
  JCommander jCommander =
      JCommander.newBuilder()
          .addObject(driver)
          .addObject(overrides)
          .addObject(KafkaFlags.getInstance())
          .build();
  jCommander.parse(args);
  if (driver.help) {
    jCommander.usage();
    return;
  }
  if (driver.testParameterProvider == null) {
    log.error("You must provide a value to the --testParameters flag to run a loadtest.");
  }
  List<TestParameters> testParameters = driver.testParameterProvider.parameters();
  HashMap<TestParameters, Map<ClientType, LatencyTracker>> results = new HashMap<>();
  testParameters.forEach(
      parameters -> {
        TestParameters updated = overrides.apply(parameters);
        Map<ClientType, LatencyTracker> result;
        if (overrides.local) {
          result =
              driver.run(
                  updated,
                  (project, clientParamsMap) ->
                      LocalController.newLocalController(
                          project, clientParamsMap, Executors.newScheduledThreadPool(500)));
        } else if (overrides.inProcess) {
          result =
              driver.run(
                  updated,
                  (project, clientParamsMap) ->
                      InprocessController.newController(
                          project,
                          ImmutableList.copyOf(clientParamsMap.keySet()),
                          Executors.newScheduledThreadPool(20)));
        } else {
          result =
              driver.run(
                  updated,
                  (project, clientParamsMap) ->
                      GCEController.newGCEController(
                          project, clientParamsMap, Executors.newScheduledThreadPool(500)));
        }
        driver.printStats(updated, result);
        results.put(updated, result);
      });
  List<ResultsOutput.TrackedResult> trackedResults =
      results.entrySet().stream()
          .flatMap(
              entry ->
                  entry.getValue().entrySet().stream()
                      .map(
                          typeAndTracker ->
                              new ResultsOutput.TrackedResult(
                                  entry.getKey(),
                                  typeAndTracker.getKey(),
                                  typeAndTracker.getValue())))
          .collect(Collectors.toList());
  driver.outputStats(trackedResults);
  System.exit(0);
}
 

@BeforeClass
public static void initClass() {
    BasicConfigurator.configure();
    Logger.getRootLogger().setLevel(Level.ERROR);
    Logger.getLogger("com.indeed").setLevel(Level.ERROR);
}