Java Code Examples for org.apache.commons.io.FileUtils#lineIterator()

private void getSents(final Tokens alphabet) {

		LineIterator iterator = null;
		try {
			iterator = FileUtils.lineIterator(docLoc);
		} catch (final IOException e) {
			e.printStackTrace();
		}

		final ArrayList<Sentence> sents = new ArrayList<Sentence>();
		while (iterator.hasNext()) {
			final String in = iterator.nextLine().trim();
			// !! Include empty sentences so nodeID indexing is consistent !!
			// if (!in.equals("")) {
			sents.add(new Sentence(in, nsents, this, alphabet));
			nsents++;
			// }
		}

		this.sents = new Sentence[nsents];
		sents.toArray(this.sents);

		LineIterator.closeQuietly(iterator);
	}
 

private void waitForStartupCompleted() throws IOException {
    for (int i = 0; i < 100; i++) {
        if (!this.process.isAlive())
            throw new IOException("Unable to start Cassandra process, more details here: " + this.logFile.getAbsolutePath());

        LineIterator lines = FileUtils.lineIterator(this.logFile, UTF8Charset.get().name());

        while (lines.hasNext()) {
            String line = lines.next();

            if (line.contains("Starting listening for CQL clients"))
                return;
        }

        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            throw new IOException("Unexpected interruption", e);
        }
    }

    throw new IOException("Cassandra process startup timeout, more details here: " + this.logFile.getAbsolutePath());
}
 

/**
 * @return the original document text
 */
public String getOriginal() {

	final StringBuffer doc = new StringBuffer();
	LineIterator iterator = null;
	try {
		iterator = FileUtils.lineIterator(docLoc);
	} catch (final IOException e) {
		e.printStackTrace();
	}

	while (iterator.hasNext())
		doc.append(iterator.nextLine().trim() + "\n");
	LineIterator.closeQuietly(iterator);

	return doc.toString();
}
 

/** Read in GoKrimp sequences (sorted by compression benefit) */
public static LinkedHashMap<Sequence, Double> readGoKrimpSequences(final File output) throws IOException {
	final LinkedHashMap<Sequence, Double> sequences = new LinkedHashMap<>();

	final LineIterator it = FileUtils.lineIterator(output);
	while (it.hasNext()) {
		final String line = it.nextLine();
		if (!line.trim().isEmpty() && line.charAt(0) == '[') {
			final String[] splitLine = line.split(" ");
			final double worth = Double.parseDouble(splitLine[splitLine.length - 1]);
			final Sequence seq = new Sequence();
			for (int i = 1; i < splitLine.length - 2; i++)
				seq.add(Integer.parseInt(splitLine[i]));
			sequences.put(seq, worth);
		}
	}

	return sequences;
}
 

/** Read in SQS sequences (sorted by worth) */
public static LinkedHashMap<Sequence, Double> readSQSSequences(final File output) throws IOException {
	final LinkedHashMap<Sequence, Double> sequences = new LinkedHashMap<>();

	final LineIterator it = FileUtils.lineIterator(output);
	while (it.hasNext()) {
		final String line = it.nextLine();
		if (!line.trim().isEmpty()) {
			final String[] splitLine = line.split("  ");
			final String[] items = splitLine[0].split(" ");
			final Sequence seq = new Sequence();
			for (final String item : items)
				seq.add(Integer.parseInt(item));
			final double worth = Double.parseDouble(splitLine[1].split(" ")[1]);
			sequences.put(seq, worth);
		}
	}

	return sequences;
}
 

private static void _CheckReload(){
    if(wordfilter.lastModified() > lastModified){
        synchronized(SensitiveWord.class){
            try{
                lastModified = wordfilter.lastModified();
                LineIterator lines = FileUtils.lineIterator(wordfilter, "utf-8");
                while(lines.hasNext()){
                    String line = lines.nextLine();
                    if(StringUtils.isNotBlank(line))
                        words.add(StringUtils.trim(line).toLowerCase());
                }
            }catch(IOException e){
                e.printStackTrace();
            }
        }
    }
}
 

@Test
public void deconvolutionResultRoundDosageTest() throws Exception {
	String[] args = {"-o",outputDir+"deconvolutionRoundedDosage","-c",counts,
					 "-e",expression, "-g", genotypes,
					 "-sn", geneSnpList, "-r"};
	commandLineOptions.parseCommandLine(args);
	Deconvolution deconvolution = new Deconvolution(commandLineOptions);
	deconvolution.readInputData();
	List<DeconvolutionResult> deconvolutionResults = deconvolution.runDeconPerGeneSnpPair();
	deconvolution.writeDeconvolutionResults(deconvolutionResults);

	LineIterator deconResults = FileUtils.lineIterator(new File(outputDir+"deconvolutionRoundedDosage/deconvolutionResults.csv"), "UTF-8");
	LineIterator deconExpected = FileUtils.lineIterator(new File("src/test/resources/expected_results/deconRoundDosageExpected.txt"), "UTF-8");
	//test if header is same
	assertEquals("File header the same",deconExpected.next(),deconResults.next());
	while (deconResults.hasNext() && deconExpected.hasNext()){
		ArrayList<String> deconResultsStringVector = new ArrayList<String>(Arrays.asList(deconResults.next().split("\t")));
		ArrayList<String> deconExpectedStringVector = new ArrayList<String>(Arrays.asList(deconExpected.next().split("\t")));
		assertEquals("Deconresult same as expected", deconExpectedStringVector, deconResultsStringVector);
		assertEquals("QTL name the same", deconExpectedStringVector.remove(0), deconResultsStringVector.remove(0));
	}
}
 

/**
 * Read in GOKRIMP sequences (sorted by compression benefit)
 *
 * @deprecated gives slightly different results to reference implementation
 */
@Deprecated
public static LinkedHashMap<Sequence, Double> readGoKrimpSequencesSPMF(final File output) throws IOException {
	final LinkedHashMap<Sequence, Double> sequences = new LinkedHashMap<>();

	final LineIterator it = FileUtils.lineIterator(output);
	while (it.hasNext()) {
		final String line = it.nextLine();
		if (!line.trim().isEmpty()) {
			final String[] splitLine = line.split("#SUP:");
			final String[] items = splitLine[0].trim().split(" ");
			final Sequence seq = new Sequence();
			for (final String item : items)
				seq.add(Integer.parseInt(item.trim()));
			final double compressionBenefit = Double.parseDouble(splitLine[1].trim());
			sequences.put(seq, compressionBenefit);
		}
	}

	return sequences;
}
 

@Test
public final void givenUsingApacheIo_whenStreamingThroughAFile_thenCorrect() throws IOException {
    final String path = "G:\\full\\train\\input\\" + "trainDataNegative.csv";
    // final String path = "G:\\full\\train\\input\\" + "trainDataPositive.csv";

    logMemory();

    final LineIterator it = FileUtils.lineIterator(new File(path), "UTF-8");
    try {
        while (it.hasNext()) {
            final String line = it.nextLine();
            // do something with line
        }
    } finally {
        LineIterator.closeQuietly(it);
    }

    logMemory();
}
 

public boolean isSetConfigChanges(File androidManifest) throws IOException {
    Pattern regex = Pattern.compile("(.*)android:configChanges(\\s*)=", Pattern.MULTILINE);
    LineIterator iter = FileUtils.lineIterator(androidManifest);
    while (iter.hasNext()) {
        String row = iter.next();
        Matcher regexMatcher = regex.matcher(row);
        if (regexMatcher.find()) {
            return true;
        }
    }
    return false;
}
 

@Override
protected CustomBatchExecutor getCustomBatchExecutor() {
    return new CustomBatchExecutor() {
        public boolean execute() {
            StopWatch stopWatch = new StopWatch();
            stopWatch.start();

            String inputFile = batchFileDirectoryName + File.separator + GeneralLedgerConstants.BatchFileSystem.BACKUP_FILE + GeneralLedgerConstants.BatchFileSystem.EXTENSION;
            String outputFile = batchFileDirectoryName + File.separator + GeneralLedgerConstants.BatchFileSystem.PRE_SCRUBBER_FILE + GeneralLedgerConstants.BatchFileSystem.EXTENSION;
            
            PreScrubberReportData preScrubberReportData = null;
            LineIterator oeIterator = null;
            try {
                oeIterator = FileUtils.lineIterator(new File(inputFile));
                preScrubberReportData = preScrubberService.preprocessOriginEntries(oeIterator, outputFile);
            }
            catch (IOException e) {
                LOG.error("IO exception occurred during pre scrubbing.", e);
                throw new RuntimeException("IO exception occurred during pre scrubbing.", e);
            }
            finally {
                LineIterator.closeQuietly(oeIterator);
            }

            if (preScrubberReportData != null) {
                new PreScrubberReport().generateReport(preScrubberReportData, preScrubberReportWriterService);
            }
            
            stopWatch.stop();
            if (LOG.isDebugEnabled()) {
                LOG.debug("scrubber step of took " + (stopWatch.getTotalTimeSeconds() / 60.0) + " minutes to complete");
            }
            return true;
        }
    };
}
 

public static final List<String> readlines(File file)
{
    if (file == null || !file.exists())
    {
        throw new IllegalArgumentException("file object must not be null and must exist: " + file.getAbsolutePath());
    }
    if (!file.isFile() || !(file.isFile() && file.canRead()))
    {
        throw new IllegalArgumentException("file object must be a readable file: " + file.getAbsolutePath());
    }
    LineIterator it;
    try
    {
        it = FileUtils.lineIterator(file, "UTF-8");
    }
    catch (IOException e)
    {
        throw new BenchmarkingException("Unable to read lines from file: " + file.getAbsolutePath(), e);
    }
    List<String> result = new LinkedList<String>();
    try
    {
        while (it.hasNext())
        {
            result.add(it.nextLine());
        }
    }
    finally
    {
        LineIterator.closeQuietly(it);
    }

    return result;
}
 

private static Set<String> loadIdsInputFile(String input) {
	if (input != null) {
		File inputFile = new File(input);
		if (inputFile.exists() && inputFile.isFile() && inputFile.canRead()) {
			Set<String> ids = new HashSet<String>();
			LineIterator iterator = null;
			try {
				iterator = FileUtils.lineIterator(inputFile);
				while (iterator.hasNext()) {
					String line = iterator.next();
					line = StringUtils.trimToNull(line);
					if (line != null) {
						ids.add(line);
					}
				}
				logger.info("Finished loading input file: "+input+"\n id count: "+ids.size());
				return ids;
			} catch (IOException e) {
				logger.warn("Could not load ids file: "+input, e);
			}
			finally {
				LineIterator.closeQuietly(iterator);
			}
		}
		else {
			logger.warn("Could not load ids file: "+input);
		}
	}
	return null;
}
 

public static String getLineContent(String file) throws IOException
{

    if (fileLineIterator == null)
    {
        fileLineIterator = FileUtils.lineIterator(new File(file), "UTF-8");
    }

    if (fileLineIterator.hasNext())
    {
        return fileLineIterator.nextLine();
    }

    return null;
}
 

public ObservableList<Message> getFilteredMessages(Event event, String[] words, int operator){
    ObservableList<Message> messages = FXCollections.observableArrayList();
    String[] interval = event.getTemporalDescription().split(",");
    int timeSliceA = convertDayToTimeSlice(Double.parseDouble(interval[0]));
    int timeSliceB = convertDayToTimeSlice(Double.parseDouble(interval[1]));
    String term = event.getTextualDescription().split(" ")[0];
    NumberFormat formatter = new DecimalFormat("00000000");
    for(int i = timeSliceA; i <= timeSliceB; i++){
        try {
            File textFile = new File(path+File.separator+preprocessing+File.separator+formatter.format(i)+".text");
            File timeFile = new File(path+File.separator+preprocessing+File.separator+formatter.format(i)+".time");
            File authorFile = new File(path+File.separator+preprocessing+File.separator+formatter.format(i)+".author");
            LineIterator textIter = FileUtils.lineIterator(textFile);
            LineIterator timeIter = FileUtils.lineIterator(timeFile);
            LineIterator authorIter = FileUtils.lineIterator(authorFile);
            while(textIter.hasNext()){
                String text = textIter.nextLine();
                short[] test = new short[words.length];
                for(int j = 0; j < words.length; j++){
                    if(StringUtils.containsIgnoreCase(text,words[j])){
                        test[j] = 1;
                    }else{
                        test[j] = 0;
                    }
                }
                if(StringUtils.containsIgnoreCase(text,term)){
                    int testSum = ArrayUtils.sum(test, 0, test.length-1);
                    String author = authorIter.nextLine();
                    String time = timeIter.nextLine();
                    if(operator==0 && testSum == test.length){
                        messages.add(new Message(author,time,text));
                    }
                    if(operator==1 && testSum > 0){
                        messages.add(new Message(author,time,text));
                    }
                }
            }
        } catch (IOException ex) {
            Logger.getLogger(Corpus.class.getName()).log(Level.SEVERE, null, ex);
        }
    }
    return messages;
}
 

private static void processRawContactInformation(String fileToRead, double minValue, ArrayList<DesiredChrContact> contactsToCheck, boolean intra, boolean probeIsSmaller) throws IOException {

        //Check if sorted version is available
        //If not make sorted available.
        if (!Gpio.exists(fileToRead + ".sorted")) {
            if (intra) {
                umcg.genetica.io.chrContacts.SortIntraChrContacts.readNonSortedWriteSorted(fileToRead, fileToRead + ".sorted");
            } else {
                umcg.genetica.io.chrContacts.SortInterChrContacts.readNonSortedWriteSorted(fileToRead, fileToRead + ".sorted");
            }

        }

        int numberToBeMatched = 0;

        LineIterator it = FileUtils.lineIterator(new File(fileToRead + ".sorted"), "UTF-8");

        try {
            while (it.hasNext()) {
                String[] parts = StringUtils.split(it.nextLine(), '\t');

                int posChr1;

                if (probeIsSmaller) {
                    posChr1 = org.apache.commons.lang.math.NumberUtils.createInteger(parts[0]);
                } else {
                    posChr1 = org.apache.commons.lang.math.NumberUtils.createInteger(parts[1]);
                }

                while (numberToBeMatched < contactsToCheck.size()) {
                    if (posChr1 < contactsToCheck.get(numberToBeMatched).getChrLocationSmaller()) {
                        break;
                    } else if (posChr1 == contactsToCheck.get(numberToBeMatched).getChrLocationSmaller()) {
                        double contact = org.apache.commons.lang.math.NumberUtils.createDouble(parts[2]);
                        if (contact >= minValue) {
                            contactsToCheck.get(numberToBeMatched).setNormalizedContactValue(contactsToCheck.get(numberToBeMatched).getNormalizedContactValue() + 1);
                        }
                        numberToBeMatched++;

                    } else if (posChr1 > contactsToCheck.get(numberToBeMatched).getChrLocationSmaller()) {
                        numberToBeMatched++;
                    }
                }
            }
        } finally {
            LineIterator.closeQuietly(it);
        }

    }
 

/** Convert dataset from SPMF format to GoKrimp format */
private static void convertDatasetGoKrimpFormat(final File inputDB, final File outputDB) throws IOException {

	// Output DB
	final BufferedWriter db = new BufferedWriter(new FileWriter(outputDB));

	// for each line (transaction) until the end of file
	boolean newSeq = false;
	final LineIterator it = FileUtils.lineIterator(inputDB, "UTF-8");
	while (it.hasNext()) {

		final String line = it.nextLine();
		// if the line is a comment, is empty or is a
		// kind of metadata
		if (line.isEmpty() == true || line.charAt(0) == '#' || line.charAt(0) == '%' || line.charAt(0) == '@') {
			continue;
		}

		// sequence separator
		if (newSeq)
			db.write("\n");

		// split the transaction into items
		final String[] lineSplited = line.split(" ");

		for (int i = 0; i < lineSplited.length; i++) {
			if (lineSplited[i].equals("-1")) { // end of item

			} else if (lineSplited[i].equals("-2")) { // end of sequence
				newSeq = true;
			} else { // extract the value for an item
				db.write(lineSplited[i] + " ");
			}
		}

	}
	db.newLine();
	db.close();

	// close the input file
	LineIterator.closeQuietly(it);

}
 

private static void processRawContactInformation(String fileToRead, double minValue, ArrayList<DesiredChrContact> contactsToCheck, TextFile outWriter, boolean intra) throws IOException {

        //Check if sorted version is available
        //If not make sorted available.
        if (!Gpio.exists(fileToRead + ".sorted")) {
            if (intra) {
                umcg.genetica.io.chrContacts.SortIntraChrContacts.readNonSortedWriteSorted(fileToRead, fileToRead + ".sorted");
            } else {
                umcg.genetica.io.chrContacts.SortInterChrContacts.readNonSortedWriteSorted(fileToRead, fileToRead + ".sorted");
            }

        }

        int numberToBeMatched = 0;

        LineIterator it = FileUtils.lineIterator(new File(fileToRead + ".sorted"), "UTF-8");

        try {
            while (it.hasNext()) {
                String[] parts = StringUtils.split(it.nextLine(), '\t');

                int posChr1 = org.apache.commons.lang.math.NumberUtils.createInteger(parts[0]);
                int posChr2 = org.apache.commons.lang.math.NumberUtils.createInteger(parts[1]);

                while (numberToBeMatched < contactsToCheck.size()) {
                    if (posChr1 < contactsToCheck.get(numberToBeMatched).getChrLocationSmaller()) {
                        break;
                    } else if (posChr1 == contactsToCheck.get(numberToBeMatched).getChrLocationSmaller()) {
                        if (posChr2 < contactsToCheck.get(numberToBeMatched).getChrLocationLarger()) {
                            break;
                        }
                        if (posChr2 == contactsToCheck.get(numberToBeMatched).getChrLocationLarger()) {
                            double contact = org.apache.commons.lang.math.NumberUtils.createDouble(parts[2]);
                            if (contact >= minValue) {
                                outWriter.writeln(contactsToCheck.get(numberToBeMatched).getSnpName() + "\t" + contactsToCheck.get(numberToBeMatched).getProbeName() + "\t" + posChr1 + "\t" + posChr2 + "\tContact\t" + contact);
                                numberToBeMatched++;
                            } else {
                                outWriter.writeln(contactsToCheck.get(numberToBeMatched).getSnpName() + "\t" + contactsToCheck.get(numberToBeMatched).getProbeName() + "\t" + posChr1 + "\t" + posChr2 + "\t-\t-");
                                numberToBeMatched++;
                            }
                        } else if (posChr2 > contactsToCheck.get(numberToBeMatched).getChrLocationLarger()) {
                            outWriter.writeln(contactsToCheck.get(numberToBeMatched).getSnpName() + "\t" + contactsToCheck.get(numberToBeMatched).getProbeName() + "\t" + posChr1 + "\t" + posChr2 + "\t-\t-");
                            numberToBeMatched++;
                        }
                    } else if (posChr1 > contactsToCheck.get(numberToBeMatched).getChrLocationSmaller()) {
                        outWriter.writeln(contactsToCheck.get(numberToBeMatched).getSnpName() + "\t" + contactsToCheck.get(numberToBeMatched).getProbeName() + "\t" + posChr1 + "\t" + posChr2 + "\t-\t-");
                        numberToBeMatched++;
                    }
                }
            }
        } finally {
            LineIterator.closeQuietly(it);
        }

    }
 

private static void processRawContactInformation(String fileToRead, double minValue, ArrayList<DesiredChrContact> contactsToCheck, boolean intra) throws IOException {

        //Check if sorted version is available
        //If not make sorted available.
        if (!Gpio.exists(fileToRead + ".sorted")) {
            if (intra) {
                umcg.genetica.io.chrContacts.SortIntraChrContacts.readNonSortedWriteSorted(fileToRead, fileToRead + ".sorted");
            } else {
                umcg.genetica.io.chrContacts.SortInterChrContacts.readNonSortedWriteSorted(fileToRead, fileToRead + ".sorted");
            }

        }

        int numberToBeMatched = 0;

        LineIterator it = FileUtils.lineIterator(new File(fileToRead + ".sorted"), "UTF-8");

        try {
            while (it.hasNext()) {
                String[] parts = StringUtils.split(it.nextLine(), '\t');

                int posChr1 = org.apache.commons.lang.math.NumberUtils.createInteger(parts[0]);
                int posChr2 = org.apache.commons.lang.math.NumberUtils.createInteger(parts[1]);

                while (numberToBeMatched < contactsToCheck.size()) {
                    if (posChr1 < contactsToCheck.get(numberToBeMatched).getChrLocationSmaller()) {
                        break;
                    } else if (posChr1 == contactsToCheck.get(numberToBeMatched).getChrLocationSmaller()) {
                        if (posChr2 < contactsToCheck.get(numberToBeMatched).getChrLocationLarger()) {
                            break;
                        }
                        if (posChr2 == contactsToCheck.get(numberToBeMatched).getChrLocationLarger()) {
                            double contact = org.apache.commons.lang.math.NumberUtils.createDouble(parts[2]);
                            if (contact >= minValue) {
                                contactsToCheck.get(numberToBeMatched).setContact();
                                numberToBeMatched++;
                            } else {
                                numberToBeMatched++;
                            }
                        } else if (posChr2 > contactsToCheck.get(numberToBeMatched).getChrLocationLarger()) {
                            numberToBeMatched++;
                        }
                    } else if (posChr1 > contactsToCheck.get(numberToBeMatched).getChrLocationSmaller()) {
                        numberToBeMatched++;
                    }
                }
            }
        } finally {
            LineIterator.closeQuietly(it);
        }

    }
 

public static void main(String[] args) throws IOException {
    Properties p = CommandLineUtils.getProperties(args, helpMessage, new String[]{"-i", "-l"});
    File in = new File(p.getProperty("-i"));
    int split = 1000;
    try {
        split = Integer.parseInt(p.getProperty("-l"));
    } catch (NumberFormatException e) {
        System.exit(1);
        System.err.printf("Number of lines as given is not a number: %s\n", p.getProperty("-l"));
    }
    if (!in.exists()) {
        System.exit(1);
        System.err.printf("File %s does not exist.\n", in.getName());
    }

    // ok, let's split
    int count = 0;
    int filenum = 0;
    String filePattern = FilenameUtils.getBaseName(p.getProperty("-i")) + "%03d." + FilenameUtils.getExtension(p.getProperty("-i"));
    if (p.get("-d")!=null) {
        filePattern = p.getProperty("-d") + '/' + filePattern;
    }
    String fileExec = FilenameUtils.getBaseName(p.getProperty("-i")) + "-index.sh";
    String currentFileName = String.format(filePattern, filenum);
    System.out.printf("Writing file %s ...\n", currentFileName);
    BufferedWriter bw = new BufferedWriter(new FileWriter(currentFileName), charBufferSize);
    BufferedWriter bwExec = null;
    if (p.get("-x") != null) bwExec = new BufferedWriter(new FileWriter(fileExec));
    addToShellFile(bwExec, currentFileName);
    LineIterator lineIterator = FileUtils.lineIterator(in);
    bw.write("<add>\n");
    while (lineIterator.hasNext()) {
        String currentLine = lineIterator.next();
        if (!(currentLine.startsWith("<add>") || currentLine.startsWith("#") || currentLine.startsWith("</add>"))) {
            // check if the old file is full ...
            if (count >= split) {
                bw.write("</add>\n");
                bw.close();
                currentFileName = String.format(filePattern, ++filenum);
                System.out.printf("Writing file %s ...\n", currentFileName);
                bw = new BufferedWriter(new FileWriter(currentFileName), charBufferSize);
                bw.write("<add>\n");
                count = 0;
                addToShellFile(bwExec, currentFileName);
            }
            // write to the current file ...
            bw.write(currentLine);
            bw.write('\n');
            count++;
        }
    }
    bw.write("</add>\n");
    bw.close();
    if (bwExec != null) bwExec.close();
    System.out.println("Finished.");
}