Java Code Examples for org.apache.commons.io.IOUtils#lineIterator()

The following examples show how to use org.apache.commons.io.IOUtils#lineIterator() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hiped2   File: Main.java    License: Apache License 2.0 6 votes vote down vote up
public static int createInputFile(Path file, Path targetFile)
    throws IOException {
  Configuration conf = new Configuration();
  FileSystem fs = file.getFileSystem(conf);

  int numNodes = getNumNodes(file);
  double initialPageRank = 1.0 / (double) numNodes;

  OutputStream os = fs.create(targetFile);
  LineIterator iter = IOUtils
      .lineIterator(fs.open(file), "UTF8");

  while (iter.hasNext()) {
    String line = iter.nextLine();

    String[] parts = StringUtils.split(line);

    Node node = new Node()
        .setPageRank(initialPageRank)
        .setAdjacentNodeNames(
            Arrays.copyOfRange(parts, 1, parts.length));
    IOUtils.write(parts[0] + '\t' + node.toString() + '\n', os);
  }
  os.close();
  return numNodes;
}
 
Example 2
public static Table<String, String, Long> loadTable(InputStream stream)
        throws IOException
{
    Table<String, String, Long> result = TreeBasedTable.create();

    LineIterator lineIterator = IOUtils.lineIterator(stream, "utf-8");
    while (lineIterator.hasNext()) {
        String line = lineIterator.next();

        System.out.println(line);

        String[] split = line.split("\t");
        String language = split[0];
        String license = split[1];
        Long documents = Long.valueOf(split[2]);
        Long tokens = Long.valueOf(split[3]);

        result.put(language, "docs " + license, documents);
        result.put(language, "tokens " + license, tokens);
    }

    return result;
}
 
Example 3
/**
 * Reads headers from the batch starting from the given position.
 * <p>
 * Retrieved headers will be added to the map given by target parameter.
 *
 * @param iterator batch iterator.
 * @param target destination of the retrieved headers.
 */
public static void readHeaders(
        final ODataBatchLineIterator iterator, final Map<String, Collection<String>> target) {

  try {
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    readBatchPart(new ODataBatchController(iterator, null), baos, true);

    final LineIterator headers = IOUtils.lineIterator(new ByteArrayInputStream(baos.toByteArray()), Constants.UTF8);
    while (headers.hasNext()) {
      final String line = headers.nextLine().trim();
      if (StringUtils.isNotBlank(line)) {
        addHeaderLine(line, target);
      }
    }
  } catch (Exception e) {
    LOG.error("Error retrieving headers", e);
    throw new IllegalStateException(e);
  }
}
 
Example 4
Source Project: spring-boot-cookbook   File: IOHelper.java    License: Apache License 2.0 6 votes vote down vote up
public static void printTarGzFile(File tarFile) throws IOException {
    BufferedInputStream bin = new BufferedInputStream(FileUtils.openInputStream(tarFile));
    CompressorInputStream cis = new GzipCompressorInputStream(bin);

    try (TarArchiveInputStream tais = new TarArchiveInputStream(cis)) {
        TarArchiveEntry entry;
        while ((entry = tais.getNextTarEntry()) != null) {
            if (entry.isDirectory()) {
                LOGGER.warn("dir:{}", entry.getName());
            } else {
                int size = (int) entry.getSize();
                byte[] content = new byte[size];
                int readCount = tais.read(content, 0, size);
                LOGGER.info("fileName:{}", entry.getName());
                ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(content, 0, readCount);
                try (LineIterator iterator = IOUtils.lineIterator(byteArrayInputStream, Charset.forName("utf-8"))) {
                    while (iterator.hasNext()) {
                        LOGGER.info("line:{}", iterator.nextLine());
                    }
                }
            }
        }
        LOGGER.info("===============finish===============");
    }
}
 
Example 5
Source Project: data-prep   File: FileComparator.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Compare two files in order to check if they contain the same lines (but maybe not in the same order).
 *
 * @param expected the file expected to match the actual one
 * @param actual the actual file
 * @return <code>true</code> if both files contains the same lines, <code>false</code> else.
 * @throws IOException in case of reading file problem.
 */
public static boolean doesFilesContainSameLines(@NotNull InputStream expected, @NotNull InputStream actual)
        throws IOException {
    LineIterator actualLI = IOUtils.lineIterator(actual, StandardCharsets.UTF_8);
    LineIterator expectedLI = IOUtils.lineIterator(expected, StandardCharsets.UTF_8);
    Map<String, Integer> unmatchedLinesFromActualFile = new HashMap<>();
    Map<String, Integer> unmatchedLinesFromExpectedFile = new HashMap<>();

    while (actualLI.hasNext() || expectedLI.hasNext()) {
        String actualLine = actualLI.hasNext() ? actualLI.nextLine() : null;
        String expectedLine = expectedLI.hasNext() ? expectedLI.nextLine() : null;
        if (actualLine != null && expectedLine != null) {
            if (!actualLine.equals(expectedLine)) {
                if (!removeOrDecrement(unmatchedLinesFromExpectedFile, actualLine)) {
                    putOrIncrement(unmatchedLinesFromActualFile, actualLine);
                }
                if (!removeOrDecrement(unmatchedLinesFromActualFile, expectedLine)) {
                    putOrIncrement(unmatchedLinesFromExpectedFile, expectedLine);
                }
            }
        } else if (actualLine != null) {
            putOrIncrement(unmatchedLinesFromActualFile, actualLine);
        } else {
            putOrIncrement(unmatchedLinesFromExpectedFile, expectedLine);
        }
    }
    if (unmatchedLinesFromActualFile.isEmpty() && unmatchedLinesFromExpectedFile.isEmpty()) {
        return true;
    }
    if (!unmatchedLinesFromActualFile.isEmpty()) {
        LOGGER.warn("Lines present only in actual file :\n" + getKeys(unmatchedLinesFromActualFile));
    }
    if (!unmatchedLinesFromExpectedFile.isEmpty()) {
        LOGGER.warn("Lines present only in expected file :\n" + getKeys(unmatchedLinesFromExpectedFile));
    }
    return false;
}
 
Example 6
private Iterator<List<Writable>> lineIterator(InputStreamReader reader) {
 	lineIter = IOUtils.lineIterator(reader);
 	
 	return new Iterator<List<Writable>>() {
@Override
public boolean hasNext() {
	return lineIter.hasNext();
}

@Override
public List<Writable> next() {
	return parseLine(lineIter.next());
}
 	};
 }
 
Example 7
Source Project: tez   File: ATSImportTool.java    License: Apache License 2.0 5 votes vote down vote up
private void logErrorMessage(ClientResponse response) throws IOException {
  LOG.error("Response status={}", response.getClientResponseStatus().toString());
  LineIterator it = null;
  try {
    it = IOUtils.lineIterator(response.getEntityInputStream(), UTF8);
    while (it.hasNext()) {
      String line = it.nextLine();
      LOG.error(line);
    }
  } finally {
    if (it != null) {
      it.close();
    }
  }
}
 
Example 8
Source Project: hbase-operator-tools   File: HBCK2.java    License: Apache License 2.0 5 votes vote down vote up
List<Long> assigns(Hbck hbck, String[] args) throws IOException {
  Options options = new Options();
  Option override = Option.builder("o").longOpt("override").build();
  Option inputFile = Option.builder("i").longOpt("inputFiles").build();
  options.addOption(override);
  options.addOption(inputFile);
  // Parse command-line.
  CommandLineParser parser = new DefaultParser();
  CommandLine commandLine;
  try {
    commandLine = parser.parse(options, args, false);
  } catch (ParseException e) {
    showErrorMessage(e.getMessage());
    return null;
  }
  boolean overrideFlag = commandLine.hasOption(override.getOpt());

  List<String> argList = commandLine.getArgList();
  if (!commandLine.hasOption(inputFile.getOpt())) {
    return hbck.assigns(argList, overrideFlag);
  }
  List<String> assignmentList = new ArrayList<>();
  for (String filePath : argList) {
    try (InputStream fileStream = new FileInputStream(filePath)){
      LineIterator it = IOUtils.lineIterator(fileStream, "UTF-8");
      while (it.hasNext()) {
        assignmentList.add(it.nextLine().trim());
      }
    }
  }
  return hbck.assigns(assignmentList, overrideFlag);
}
 
Example 9
Source Project: webanno   File: AutomationUtil.java    License: Apache License 2.0 5 votes vote down vote up
private static void buildPredictFile(File apredFt, File aPredFile,
        List<List<String>> aPredictions, AnnotationFeature aFeature)
    throws IOException
{
    LineIterator it = IOUtils.lineIterator(new FileReader(apredFt));
    StringBuilder predBuffer = new StringBuilder();
    int i = 0;
    while (it.hasNext()) {
        String line = it.next();
        if (line.trim().equals("")) {
            predBuffer.append("\n");
            continue;
        }
        StringTokenizer st = new StringTokenizer(line, " ");
        // if the target feature is on multiple token, we do not need the morphological features
        // in the prediction file
        switch (aFeature.getLayer().getAnchoringMode()) {
        case TOKENS:
            predBuffer.append(st.nextToken()).append(" ");
            break;
        case SINGLE_TOKEN:
            while (st.hasMoreTokens()) {
                predBuffer.append(st.nextToken()).append(" ");
            }
            break;
        default:
            throw new IllegalStateException("Unsupported anchoring mode: ["
                    + aFeature.getLayer().getAnchoringMode() + "]");
        }
        
        for (List<String> prediction : aPredictions) {
            predBuffer.append(prediction.get(i)).append(" ");
        }
        // add its
        predBuffer.append("\n");
        i++;
    }
    
    IOUtils.write(predBuffer.toString(), new FileOutputStream(aPredFile));
}
 
Example 10
@Override
public Boolean call() throws IOException, InterruptedException {
    try (Reader reader = new InputStreamReader(in)) {
        LineIterator it = IOUtils.lineIterator(reader);
        while (it.hasNext()) {
            String line = it.nextLine();
            if (outputObserver.onReadLine(line)) {
                return true; // interrupted by OutputObserver
            }
        }
    }
    return false;
}
 
Example 11
Source Project: gocd   File: StreamPumper.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void run() {
    try (LineIterator lineIterator = IOUtils.lineIterator(in)) {
        while (lineIterator.hasNext()) {
            consumeLine(lineIterator.nextLine());
        }
    } catch (Exception ignore) {
    } finally {
        completed = true;
    }
}
 
Example 12
Source Project: OpenModsLib   File: ShaderProgramBuilder.java    License: MIT License 5 votes vote down vote up
private static String readShaderSource(ResourceLocation source) {
	try {
		final InputStream is = Minecraft.getMinecraft().getResourceManager().getResource(source).getInputStream();
		final Iterator<String> lines = IOUtils.lineIterator(is, StandardCharsets.UTF_8);
		final StringBuilder out = new StringBuilder();
		Joiner.on('\n').appendTo(out, lines);
		return out.toString();
	} catch (IOException e) {
		throw new OpenGLException("Failed to read resource " + source, e);
	}

}
 
Example 13
Source Project: DataVec   File: CSVSequenceRecordReader.java    License: Apache License 2.0 5 votes vote down vote up
private List<List<Writable>> loadAndClose(InputStream inputStream) {
    LineIterator lineIter = null;
    try {
        lineIter = IOUtils.lineIterator(new BufferedReader(new InputStreamReader(inputStream)));
        return load(lineIter);
    } finally {
        if (lineIter != null) {
            lineIter.close();
        }
        IOUtils.closeQuietly(inputStream);
    }
}
 
Example 14
Source Project: docker-java   File: DockerClientTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testNginxDockerfileBuilder() throws DockerException,
		IOException {
	File baseDir = new File(Thread.currentThread().getContextClassLoader()
			.getResource("nginx").getFile());

	ClientResponse response = dockerClient.build(baseDir);

	StringWriter logwriter = new StringWriter();

	try {
		LineIterator itr = IOUtils.lineIterator(
				response.getEntityInputStream(), "UTF-8");
		while (itr.hasNext()) {
			String line = itr.next();
			logwriter.write(line + "\n");
			LOG.info(line);
		}
	} finally {
		IOUtils.closeQuietly(response.getEntityInputStream());
	}

	String fullLog = logwriter.toString();
	assertThat(fullLog, containsString("Successfully built"));

	String imageId = StringUtils.substringBetween(fullLog,
			"Successfully built ", "\\n\"}").trim();

	ImageInspectResponse imageInspectResponse = dockerClient
			.inspectImage(imageId);
	assertThat(imageInspectResponse, not(nullValue()));
	LOG.info("Image Inspect: {}", imageInspectResponse.toString());
	tmpImgs.add(imageInspectResponse.getId());

	assertThat(imageInspectResponse.getAuthor(),
			equalTo("Guillaume J. Charmes \"[email protected]\""));
}
 
Example 15
private List<List<Writable>> loadAndClose(InputStream inputStream) {
    LineIterator lineIter = null;
    try {
        lineIter = IOUtils.lineIterator(new BufferedReader(new InputStreamReader(inputStream)));
        return load(lineIter);
    } finally {
        if (lineIter != null) {
            lineIter.close();
        }
        IOUtils.closeQuietly(inputStream);
    }
}
 
Example 16
@Override
public void getNext(JCas aJCas) throws IOException, CollectionException
{
    Resource res = nextFile();
    initCas(aJCas, res);

    StringBuilder text = new StringBuilder();
    
    try (InputStream is = new BufferedInputStream(
            CompressionUtils.getInputStream(res.getLocation(), res.getInputStream()))) {
        LineIterator i = IOUtils.lineIterator(is, "UTF-8");
        try {
            while (i.hasNext()) {
                String line = i.next();
                
                if (line.startsWith("#")) {
                    continue;
                }
                
                String[] fields = line.split("\\s", 2);
                
                if (text.length() > 0) {
                    text.append("\n");
                }
                
                int sentenceBegin = text.length();
                text.append(fields[1]);
                
                NamedEntity ne = new NamedEntity(aJCas,  sentenceBegin, text.length());
                ne.setValue(fields[0]);
                ne.addToIndexes();
                
                new Sentence(aJCas, sentenceBegin, text.length()).addToIndexes();
            }
        }
        finally {
            LineIterator.closeQuietly(i);
        }
        
        aJCas.setDocumentText(text.toString());
    }
}
 
Example 17
Source Project: webanno   File: AutomationUtil.java    License: Apache License 2.0 4 votes vote down vote up
public static void addTabSepTrainDocument(MiraTemplate aTemplate,
        AutomationService aAutomationService)
    throws IOException, UIMAException, ClassNotFoundException, AutomationException
{
    File miraDir = aAutomationService.getMiraDir(aTemplate.getTrainFeature());
    if (!miraDir.exists()) {
        FileUtils.forceMkdir(miraDir);
    }

    AutomationStatus status = aAutomationService.getAutomationStatus(aTemplate);

    boolean documentChanged = false;
    for (TrainingDocument document : aAutomationService
            .listTabSepDocuments(aTemplate.getTrainFeature().getProject())) {
        if (!document.isProcessed()) {
            documentChanged = true;
            break;
        }
    }
    if (!documentChanged) {
        return;
    }

    for (TrainingDocument trainingDocument : aAutomationService.listTabSepDocuments(aTemplate
            .getTrainFeature().getProject())) {
        if (trainingDocument.getFeature() != null) { // This is a target layer train document
            continue;
        }
        File trainFile = new File(miraDir, trainingDocument.getId()
                + trainingDocument.getProject().getId() + ".train");
        BufferedWriter trainOut = new BufferedWriter(new FileWriter(trainFile));
        File tabSepFile = new File(aAutomationService.getDocumentFolder(trainingDocument),
                trainingDocument.getName());
        LineIterator it = IOUtils.lineIterator(new FileReader(tabSepFile));
        while (it.hasNext()) {
            String line = it.next();
            if (line.trim().equals("")) {
                trainOut.append("\n");
            }
            else {
                StringTokenizer st = new StringTokenizer(line, "\t");
                if (st.countTokens() != 2) {
                    trainOut.close();
                    throw new AutomationException("This is not a valid TAB-SEP document");
                }
                trainOut.append(getMiraLineForTabSep(st.nextToken(), st.nextToken()));
            }
        }
        trainingDocument.setProcessed(false);
        status.setTrainDocs(status.getTrainDocs() - 1);
        trainOut.close();
    }

}
 
Example 18
private static LineIterator getLineIterator(String fileName) throws IOException {
  ClassLoader classLoader = CodeFileReader.class.getClassLoader();
  return IOUtils.lineIterator(classLoader.getResourceAsStream(fileName), Charset.defaultCharset());
}
 
Example 19
/**
 * Loads an in memory cache from the given input stream (sets syn0 and the vocab).
 *
 * @param inputStream  input stream
 * @return a {@link Pair} holding the lookup table and the vocab cache.
 */
public static Pair<InMemoryLookupTable, VocabCache> loadTxt(@NonNull InputStream inputStream) {
    AbstractCache<VocabWord> cache = new AbstractCache<>();
    LineIterator lines = null;

    try (InputStreamReader inputStreamReader = new InputStreamReader(inputStream);
         BufferedReader reader = new BufferedReader(inputStreamReader)) {
        lines = IOUtils.lineIterator(reader);

        String line = null;
        boolean hasHeader = false;

        /* Check if first line is a header */
        if (lines.hasNext()) {
            line = lines.nextLine();
            hasHeader = isHeader(line, cache);
        }

        if (hasHeader) {
            log.debug("First line is a header");
            line = lines.nextLine();
        }

        List<INDArray> arrays = new ArrayList<>();
        long[] vShape = new long[]{ 1, -1 };

        do {
            String[] tokens = line.split(" ");
            String word = ReadHelper.decodeB64(tokens[0]);
            VocabWord vocabWord = new VocabWord(1.0, word);
            vocabWord.setIndex(cache.numWords());

            cache.addToken(vocabWord);
            cache.addWordToIndex(vocabWord.getIndex(), word);
            cache.putVocabWord(word);

            float[] vector = new float[tokens.length - 1];
            for (int i = 1; i < tokens.length; i++) {
                vector[i - 1] = Float.parseFloat(tokens[i]);
            }

            vShape[1] = vector.length;
            INDArray row = Nd4j.create(vector, vShape);

            arrays.add(row);

            line = lines.hasNext() ? lines.next() : null;
        } while (line != null);

        INDArray syn = Nd4j.vstack(arrays);

        InMemoryLookupTable<VocabWord> lookupTable = new InMemoryLookupTable
                .Builder<VocabWord>()
                .vectorLength(arrays.get(0).columns())
                .useAdaGrad(false)
                .cache(cache)
                .useHierarchicSoftmax(false)
                .build();

        lookupTable.setSyn0(syn);

        return new Pair<>((InMemoryLookupTable) lookupTable, (VocabCache) cache);
    } catch (IOException readeTextStreamException) {
        throw new RuntimeException(readeTextStreamException);
    } finally {
        if (lines != null) {
            lines.close();
        }
    }
}
 
Example 20
Source Project: docker-java   File: DockerClientTest.java    License: Apache License 2.0 4 votes vote down vote up
private String dockerfileBuild(File baseDir, String expectedText)
		throws DockerException, IOException {

	// Build image
	ClientResponse response = dockerClient.build(baseDir);

	StringWriter logwriter = new StringWriter();

	try {
		LineIterator itr = IOUtils.lineIterator(
				response.getEntityInputStream(), "UTF-8");
		while (itr.hasNext()) {
			String line = itr.next();
			logwriter.write(line + "\n");
			LOG.info(line);
		}
	} finally {
		IOUtils.closeQuietly(response.getEntityInputStream());
	}

	String fullLog = logwriter.toString();
	assertThat(fullLog, containsString("Successfully built"));

	String imageId = StringUtils.substringBetween(fullLog,
			"Successfully built ", "\\n\"}").trim();

	// Create container based on image
	ContainerConfig containerConfig = new ContainerConfig();
	containerConfig.setImage(imageId);
	ContainerCreateResponse container = dockerClient
			.createContainer(containerConfig);
	LOG.info("Created container: {}", container.toString());
	assertThat(container.getId(), not(isEmptyString()));

	dockerClient.startContainer(container.getId());
	dockerClient.waitContainer(container.getId());

	tmpContainers.add(container.getId());

	// Log container
	ClientResponse logResponse = dockerClient.logContainer(container
			.getId());

	assertThat(logResponseStream(logResponse), containsString(expectedText));

	return container.getId();
}