Java Code Examples for org.apache.commons.io.IOUtils#lineIterator()

public static int createInputFile(Path file, Path targetFile)
    throws IOException {
  Configuration conf = new Configuration();
  FileSystem fs = file.getFileSystem(conf);

  int numNodes = getNumNodes(file);
  double initialPageRank = 1.0 / (double) numNodes;

  OutputStream os = fs.create(targetFile);
  LineIterator iter = IOUtils
      .lineIterator(fs.open(file), "UTF8");

  while (iter.hasNext()) {
    String line = iter.nextLine();

    String[] parts = StringUtils.split(line);

    Node node = new Node()
        .setPageRank(initialPageRank)
        .setAdjacentNodeNames(
            Arrays.copyOfRange(parts, 1, parts.length));
    IOUtils.write(parts[0] + '\t' + node.toString() + '\n', os);
  }
  os.close();
  return numNodes;
}
 

public static Table<String, String, Long> loadTable(InputStream stream)
        throws IOException
{
    Table<String, String, Long> result = TreeBasedTable.create();

    LineIterator lineIterator = IOUtils.lineIterator(stream, "utf-8");
    while (lineIterator.hasNext()) {
        String line = lineIterator.next();

        System.out.println(line);

        String[] split = line.split("\t");
        String language = split[0];
        String license = split[1];
        Long documents = Long.valueOf(split[2]);
        Long tokens = Long.valueOf(split[3]);

        result.put(language, "docs " + license, documents);
        result.put(language, "tokens " + license, tokens);
    }

    return result;
}
 

/**
 * Reads headers from the batch starting from the given position.
 * <p>
 * Retrieved headers will be added to the map given by target parameter.
 *
 * @param iterator batch iterator.
 * @param target destination of the retrieved headers.
 */
public static void readHeaders(
        final ODataBatchLineIterator iterator, final Map<String, Collection<String>> target) {

  try {
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    readBatchPart(new ODataBatchController(iterator, null), baos, true);

    final LineIterator headers = IOUtils.lineIterator(new ByteArrayInputStream(baos.toByteArray()), Constants.UTF8);
    while (headers.hasNext()) {
      final String line = headers.nextLine().trim();
      if (StringUtils.isNotBlank(line)) {
        addHeaderLine(line, target);
      }
    }
  } catch (Exception e) {
    LOG.error("Error retrieving headers", e);
    throw new IllegalStateException(e);
  }
}
 

public static void printTarGzFile(File tarFile) throws IOException {
    BufferedInputStream bin = new BufferedInputStream(FileUtils.openInputStream(tarFile));
    CompressorInputStream cis = new GzipCompressorInputStream(bin);

    try (TarArchiveInputStream tais = new TarArchiveInputStream(cis)) {
        TarArchiveEntry entry;
        while ((entry = tais.getNextTarEntry()) != null) {
            if (entry.isDirectory()) {
                LOGGER.warn("dir:{}", entry.getName());
            } else {
                int size = (int) entry.getSize();
                byte[] content = new byte[size];
                int readCount = tais.read(content, 0, size);
                LOGGER.info("fileName:{}", entry.getName());
                ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(content, 0, readCount);
                try (LineIterator iterator = IOUtils.lineIterator(byteArrayInputStream, Charset.forName("utf-8"))) {
                    while (iterator.hasNext()) {
                        LOGGER.info("line:{}", iterator.nextLine());
                    }
                }
            }
        }
        LOGGER.info("===============finish===============");
    }
}
 

/**
 * Compare two files in order to check if they contain the same lines (but maybe not in the same order).
 *
 * @param expected the file expected to match the actual one
 * @param actual the actual file
 * @return <code>true</code> if both files contains the same lines, <code>false</code> else.
 * @throws IOException in case of reading file problem.
 */
public static boolean doesFilesContainSameLines(@NotNull InputStream expected, @NotNull InputStream actual)
        throws IOException {
    LineIterator actualLI = IOUtils.lineIterator(actual, StandardCharsets.UTF_8);
    LineIterator expectedLI = IOUtils.lineIterator(expected, StandardCharsets.UTF_8);
    Map<String, Integer> unmatchedLinesFromActualFile = new HashMap<>();
    Map<String, Integer> unmatchedLinesFromExpectedFile = new HashMap<>();

    while (actualLI.hasNext() || expectedLI.hasNext()) {
        String actualLine = actualLI.hasNext() ? actualLI.nextLine() : null;
        String expectedLine = expectedLI.hasNext() ? expectedLI.nextLine() : null;
        if (actualLine != null && expectedLine != null) {
            if (!actualLine.equals(expectedLine)) {
                if (!removeOrDecrement(unmatchedLinesFromExpectedFile, actualLine)) {
                    putOrIncrement(unmatchedLinesFromActualFile, actualLine);
                }
                if (!removeOrDecrement(unmatchedLinesFromActualFile, expectedLine)) {
                    putOrIncrement(unmatchedLinesFromExpectedFile, expectedLine);
                }
            }
        } else if (actualLine != null) {
            putOrIncrement(unmatchedLinesFromActualFile, actualLine);
        } else {
            putOrIncrement(unmatchedLinesFromExpectedFile, expectedLine);
        }
    }
    if (unmatchedLinesFromActualFile.isEmpty() && unmatchedLinesFromExpectedFile.isEmpty()) {
        return true;
    }
    if (!unmatchedLinesFromActualFile.isEmpty()) {
        LOGGER.warn("Lines present only in actual file :\n" + getKeys(unmatchedLinesFromActualFile));
    }
    if (!unmatchedLinesFromExpectedFile.isEmpty()) {
        LOGGER.warn("Lines present only in expected file :\n" + getKeys(unmatchedLinesFromExpectedFile));
    }
    return false;
}
 

private Iterator<List<Writable>> lineIterator(InputStreamReader reader) {
 	lineIter = IOUtils.lineIterator(reader);
 	
 	return new Iterator<List<Writable>>() {
@Override
public boolean hasNext() {
	return lineIter.hasNext();
}

@Override
public List<Writable> next() {
	return parseLine(lineIter.next());
}
 	};
 }
 

private void logErrorMessage(ClientResponse response) throws IOException {
  LOG.error("Response status={}", response.getClientResponseStatus().toString());
  LineIterator it = null;
  try {
    it = IOUtils.lineIterator(response.getEntityInputStream(), UTF8);
    while (it.hasNext()) {
      String line = it.nextLine();
      LOG.error(line);
    }
  } finally {
    if (it != null) {
      it.close();
    }
  }
}
 

List<Long> assigns(Hbck hbck, String[] args) throws IOException {
  Options options = new Options();
  Option override = Option.builder("o").longOpt("override").build();
  Option inputFile = Option.builder("i").longOpt("inputFiles").build();
  options.addOption(override);
  options.addOption(inputFile);
  // Parse command-line.
  CommandLineParser parser = new DefaultParser();
  CommandLine commandLine;
  try {
    commandLine = parser.parse(options, args, false);
  } catch (ParseException e) {
    showErrorMessage(e.getMessage());
    return null;
  }
  boolean overrideFlag = commandLine.hasOption(override.getOpt());

  List<String> argList = commandLine.getArgList();
  if (!commandLine.hasOption(inputFile.getOpt())) {
    return hbck.assigns(argList, overrideFlag);
  }
  List<String> assignmentList = new ArrayList<>();
  for (String filePath : argList) {
    try (InputStream fileStream = new FileInputStream(filePath)){
      LineIterator it = IOUtils.lineIterator(fileStream, "UTF-8");
      while (it.hasNext()) {
        assignmentList.add(it.nextLine().trim());
      }
    }
  }
  return hbck.assigns(assignmentList, overrideFlag);
}
 

private static void buildPredictFile(File apredFt, File aPredFile,
        List<List<String>> aPredictions, AnnotationFeature aFeature)
    throws IOException
{
    LineIterator it = IOUtils.lineIterator(new FileReader(apredFt));
    StringBuilder predBuffer = new StringBuilder();
    int i = 0;
    while (it.hasNext()) {
        String line = it.next();
        if (line.trim().equals("")) {
            predBuffer.append("\n");
            continue;
        }
        StringTokenizer st = new StringTokenizer(line, " ");
        // if the target feature is on multiple token, we do not need the morphological features
        // in the prediction file
        switch (aFeature.getLayer().getAnchoringMode()) {
        case TOKENS:
            predBuffer.append(st.nextToken()).append(" ");
            break;
        case SINGLE_TOKEN:
            while (st.hasMoreTokens()) {
                predBuffer.append(st.nextToken()).append(" ");
            }
            break;
        default:
            throw new IllegalStateException("Unsupported anchoring mode: ["
                    + aFeature.getLayer().getAnchoringMode() + "]");
        }
        
        for (List<String> prediction : aPredictions) {
            predBuffer.append(prediction.get(i)).append(" ");
        }
        // add its
        predBuffer.append("\n");
        i++;
    }
    
    IOUtils.write(predBuffer.toString(), new FileOutputStream(aPredFile));
}
 

@Override
public Boolean call() throws IOException, InterruptedException {
    try (Reader reader = new InputStreamReader(in)) {
        LineIterator it = IOUtils.lineIterator(reader);
        while (it.hasNext()) {
            String line = it.nextLine();
            if (outputObserver.onReadLine(line)) {
                return true; // interrupted by OutputObserver
            }
        }
    }
    return false;
}
 

@Override
public void run() {
    try (LineIterator lineIterator = IOUtils.lineIterator(in)) {
        while (lineIterator.hasNext()) {
            consumeLine(lineIterator.nextLine());
        }
    } catch (Exception ignore) {
    } finally {
        completed = true;
    }
}
 

private static String readShaderSource(ResourceLocation source) {
	try {
		final InputStream is = Minecraft.getMinecraft().getResourceManager().getResource(source).getInputStream();
		final Iterator<String> lines = IOUtils.lineIterator(is, StandardCharsets.UTF_8);
		final StringBuilder out = new StringBuilder();
		Joiner.on('\n').appendTo(out, lines);
		return out.toString();
	} catch (IOException e) {
		throw new OpenGLException("Failed to read resource " + source, e);
	}

}
 

private List<List<Writable>> loadAndClose(InputStream inputStream) {
    LineIterator lineIter = null;
    try {
        lineIter = IOUtils.lineIterator(new BufferedReader(new InputStreamReader(inputStream)));
        return load(lineIter);
    } finally {
        if (lineIter != null) {
            lineIter.close();
        }
        IOUtils.closeQuietly(inputStream);
    }
}
 

@Test
public void testNginxDockerfileBuilder() throws DockerException,
		IOException {
	File baseDir = new File(Thread.currentThread().getContextClassLoader()
			.getResource("nginx").getFile());

	ClientResponse response = dockerClient.build(baseDir);

	StringWriter logwriter = new StringWriter();

	try {
		LineIterator itr = IOUtils.lineIterator(
				response.getEntityInputStream(), "UTF-8");
		while (itr.hasNext()) {
			String line = itr.next();
			logwriter.write(line + "\n");
			LOG.info(line);
		}
	} finally {
		IOUtils.closeQuietly(response.getEntityInputStream());
	}

	String fullLog = logwriter.toString();
	assertThat(fullLog, containsString("Successfully built"));

	String imageId = StringUtils.substringBetween(fullLog,
			"Successfully built ", "\\n\"}").trim();

	ImageInspectResponse imageInspectResponse = dockerClient
			.inspectImage(imageId);
	assertThat(imageInspectResponse, not(nullValue()));
	LOG.info("Image Inspect: {}", imageInspectResponse.toString());
	tmpImgs.add(imageInspectResponse.getId());

	assertThat(imageInspectResponse.getAuthor(),
			equalTo("Guillaume J. Charmes \"[email protected]\""));
}
 

private List<List<Writable>> loadAndClose(InputStream inputStream) {
    LineIterator lineIter = null;
    try {
        lineIter = IOUtils.lineIterator(new BufferedReader(new InputStreamReader(inputStream)));
        return load(lineIter);
    } finally {
        if (lineIter != null) {
            lineIter.close();
        }
        IOUtils.closeQuietly(inputStream);
    }
}
 

@Override
public void getNext(JCas aJCas) throws IOException, CollectionException
{
    Resource res = nextFile();
    initCas(aJCas, res);

    StringBuilder text = new StringBuilder();
    
    try (InputStream is = new BufferedInputStream(
            CompressionUtils.getInputStream(res.getLocation(), res.getInputStream()))) {
        LineIterator i = IOUtils.lineIterator(is, "UTF-8");
        try {
            while (i.hasNext()) {
                String line = i.next();
                
                if (line.startsWith("#")) {
                    continue;
                }
                
                String[] fields = line.split("\\s", 2);
                
                if (text.length() > 0) {
                    text.append("\n");
                }
                
                int sentenceBegin = text.length();
                text.append(fields[1]);
                
                NamedEntity ne = new NamedEntity(aJCas,  sentenceBegin, text.length());
                ne.setValue(fields[0]);
                ne.addToIndexes();
                
                new Sentence(aJCas, sentenceBegin, text.length()).addToIndexes();
            }
        }
        finally {
            LineIterator.closeQuietly(i);
        }
        
        aJCas.setDocumentText(text.toString());
    }
}
 

public static void addTabSepTrainDocument(MiraTemplate aTemplate,
        AutomationService aAutomationService)
    throws IOException, UIMAException, ClassNotFoundException, AutomationException
{
    File miraDir = aAutomationService.getMiraDir(aTemplate.getTrainFeature());
    if (!miraDir.exists()) {
        FileUtils.forceMkdir(miraDir);
    }

    AutomationStatus status = aAutomationService.getAutomationStatus(aTemplate);

    boolean documentChanged = false;
    for (TrainingDocument document : aAutomationService
            .listTabSepDocuments(aTemplate.getTrainFeature().getProject())) {
        if (!document.isProcessed()) {
            documentChanged = true;
            break;
        }
    }
    if (!documentChanged) {
        return;
    }

    for (TrainingDocument trainingDocument : aAutomationService.listTabSepDocuments(aTemplate
            .getTrainFeature().getProject())) {
        if (trainingDocument.getFeature() != null) { // This is a target layer train document
            continue;
        }
        File trainFile = new File(miraDir, trainingDocument.getId()
                + trainingDocument.getProject().getId() + ".train");
        BufferedWriter trainOut = new BufferedWriter(new FileWriter(trainFile));
        File tabSepFile = new File(aAutomationService.getDocumentFolder(trainingDocument),
                trainingDocument.getName());
        LineIterator it = IOUtils.lineIterator(new FileReader(tabSepFile));
        while (it.hasNext()) {
            String line = it.next();
            if (line.trim().equals("")) {
                trainOut.append("\n");
            }
            else {
                StringTokenizer st = new StringTokenizer(line, "\t");
                if (st.countTokens() != 2) {
                    trainOut.close();
                    throw new AutomationException("This is not a valid TAB-SEP document");
                }
                trainOut.append(getMiraLineForTabSep(st.nextToken(), st.nextToken()));
            }
        }
        trainingDocument.setProcessed(false);
        status.setTrainDocs(status.getTrainDocs() - 1);
        trainOut.close();
    }

}
 

private static LineIterator getLineIterator(String fileName) throws IOException {
  ClassLoader classLoader = CodeFileReader.class.getClassLoader();
  return IOUtils.lineIterator(classLoader.getResourceAsStream(fileName), Charset.defaultCharset());
}
 

/**
 * Loads an in memory cache from the given input stream (sets syn0 and the vocab).
 *
 * @param inputStream  input stream
 * @return a {@link Pair} holding the lookup table and the vocab cache.
 */
public static Pair<InMemoryLookupTable, VocabCache> loadTxt(@NonNull InputStream inputStream) {
    AbstractCache<VocabWord> cache = new AbstractCache<>();
    LineIterator lines = null;

    try (InputStreamReader inputStreamReader = new InputStreamReader(inputStream);
         BufferedReader reader = new BufferedReader(inputStreamReader)) {
        lines = IOUtils.lineIterator(reader);

        String line = null;
        boolean hasHeader = false;

        /* Check if first line is a header */
        if (lines.hasNext()) {
            line = lines.nextLine();
            hasHeader = isHeader(line, cache);
        }

        if (hasHeader) {
            log.debug("First line is a header");
            line = lines.nextLine();
        }

        List<INDArray> arrays = new ArrayList<>();
        long[] vShape = new long[]{ 1, -1 };

        do {
            String[] tokens = line.split(" ");
            String word = ReadHelper.decodeB64(tokens[0]);
            VocabWord vocabWord = new VocabWord(1.0, word);
            vocabWord.setIndex(cache.numWords());

            cache.addToken(vocabWord);
            cache.addWordToIndex(vocabWord.getIndex(), word);
            cache.putVocabWord(word);

            float[] vector = new float[tokens.length - 1];
            for (int i = 1; i < tokens.length; i++) {
                vector[i - 1] = Float.parseFloat(tokens[i]);
            }

            vShape[1] = vector.length;
            INDArray row = Nd4j.create(vector, vShape);

            arrays.add(row);

            line = lines.hasNext() ? lines.next() : null;
        } while (line != null);

        INDArray syn = Nd4j.vstack(arrays);

        InMemoryLookupTable<VocabWord> lookupTable = new InMemoryLookupTable
                .Builder<VocabWord>()
                .vectorLength(arrays.get(0).columns())
                .useAdaGrad(false)
                .cache(cache)
                .useHierarchicSoftmax(false)
                .build();

        lookupTable.setSyn0(syn);

        return new Pair<>((InMemoryLookupTable) lookupTable, (VocabCache) cache);
    } catch (IOException readeTextStreamException) {
        throw new RuntimeException(readeTextStreamException);
    } finally {
        if (lines != null) {
            lines.close();
        }
    }
}
 

private String dockerfileBuild(File baseDir, String expectedText)
		throws DockerException, IOException {

	// Build image
	ClientResponse response = dockerClient.build(baseDir);

	StringWriter logwriter = new StringWriter();

	try {
		LineIterator itr = IOUtils.lineIterator(
				response.getEntityInputStream(), "UTF-8");
		while (itr.hasNext()) {
			String line = itr.next();
			logwriter.write(line + "\n");
			LOG.info(line);
		}
	} finally {
		IOUtils.closeQuietly(response.getEntityInputStream());
	}

	String fullLog = logwriter.toString();
	assertThat(fullLog, containsString("Successfully built"));

	String imageId = StringUtils.substringBetween(fullLog,
			"Successfully built ", "\\n\"}").trim();

	// Create container based on image
	ContainerConfig containerConfig = new ContainerConfig();
	containerConfig.setImage(imageId);
	ContainerCreateResponse container = dockerClient
			.createContainer(containerConfig);
	LOG.info("Created container: {}", container.toString());
	assertThat(container.getId(), not(isEmptyString()));

	dockerClient.startContainer(container.getId());
	dockerClient.waitContainer(container.getId());

	tmpContainers.add(container.getId());

	// Log container
	ClientResponse logResponse = dockerClient.logContainer(container
			.getId());

	assertThat(logResponseStream(logResponse), containsString(expectedText));

	return container.getId();
}