Jave Code Examples for 'read input file'

60 Jave code examples are found related to "read input file". These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example.
Example 1
Project: MyTv   Author: 52Jolynn   File: FileUtils.java    License: Apache License 2.0 7 votes vote down vote up
/**
 * 从输入流中读取数据,NIO方式
 * 
 * @param in
 * @return
 * @throws IOException
 */
public static byte[] readFromInputStream(InputStream in) throws IOException {
	if (in == null) {
		return null;
	}
	ReadableByteChannel rbc = Channels.newChannel(in);
	ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
	byte[] data = new byte[BUFFER_SIZE];
	ByteArrayOutputStream baos = new ByteArrayOutputStream();
	while (true) {
		int count = rbc.read(buffer);
		if (count == -1) {
			break;
		}
		if (count < BUFFER_SIZE) {
			data = new byte[count];
		}
		buffer.flip();
		buffer.get(data, 0, count);
		baos.write(data);
		buffer.clear();
	}
	rbc.close();
	return baos.toByteArray();
}
 
Example 2
Project: herddb   Author: diennea   File: ODirectFileInputStreamTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void readFullAndPartialBlock() throws Exception {
    Random random = new Random();

    Path path = tmp.newFile().toPath();
    int blockSize = (int) OpenFileUtils.getBlockSize(path);

    int writeSize = blockSize + blockSize / 2;
    assertTrue(writeSize > blockSize);

    byte[] data = new byte[writeSize];
    random.nextBytes(data);
    Files.write(path, data);

    try (ODirectFileInputStream oo = new ODirectFileInputStream(path)) {
        byte[] fullread = new byte[blockSize * 2];
        int read = oo.read(fullread);

        assertEquals(writeSize, read);
        assertEquals(2, oo.getReadBlocks());
    }
}
 
Example 3
Project: biojava   Author: biojava   File: CommandPrompt.java    License: GNU Lesser General Public License v2.1 6 votes vote down vote up
private static LinkedHashMap<String, ProteinSequence> readInputFile(String inputLocation, AminoAcidCompositionTable aaTable) throws Exception{
	FileInputStream inStream = new FileInputStream(inputLocation);
	CompoundSet<AminoAcidCompound>	set;
	if(aaTable == null){
		set = CaseFreeAminoAcidCompoundSet.getAminoAcidCompoundSet();
	}else{
		set = aaTable.getAminoAcidCompoundSet();
	}
	LinkedHashMap<String, ProteinSequence> ret;
	if ( inputLocation.toLowerCase().contains(".gb")) {
		GenbankReader<ProteinSequence, AminoAcidCompound> genbankReader = new GenbankReader<ProteinSequence, AminoAcidCompound>(
				inStream, new GenericGenbankHeaderParser<ProteinSequence, AminoAcidCompound>(),
				new ProteinSequenceCreator(set));
		ret = genbankReader.process();


	} else {
		FastaReader<ProteinSequence, AminoAcidCompound> fastaReader = new FastaReader<ProteinSequence, AminoAcidCompound>(
				inStream, new GenericFastaHeaderParser<ProteinSequence, AminoAcidCompound>(),
				new ProteinSequenceCreator(set));
		ret = fastaReader.process();

	}
	return ret;
}
 
Example 4
Project: ixa-pipe-pos   Author: ixa-ehu   File: InputOutputUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Read the file into an {@code ObjectStream}.
 * 
 * @param infile
 *          the string pointing to the file
 * @return the object stream
 */
public static ObjectStream<String> readFileIntoMarkableStreamFactory(
    final String infile) {

  InputStreamFactory inputStreamFactory = null;
  try {
    inputStreamFactory = new MarkableFileInputStreamFactory(new File(infile));
  } catch (final FileNotFoundException e) {
    e.printStackTrace();
  }
  ObjectStream<String> lineStream = null;
  try {
    lineStream = new PlainTextByLineStream(inputStreamFactory, "UTF-8");
  } catch (final IOException e) {
    CmdLineUtil.handleCreateObjectStreamError(e);
  }
  return lineStream;
}
 
Example 5
Project: adaptive-alerting   Author: ExpediaDotCom   File: MetricFunctionsReaderTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testReadFromInputFile() {
    val functionInputFileName = "config/functions-test.txt";
    List<MetricFunctionsSpec> metricFunctionsSpecList = MetricFunctionsReader
            .readFromInputFile(ClassLoader.getSystemResource(functionInputFileName).getPath());
    assertEquals(1, metricFunctionsSpecList.size());
    MetricFunctionsSpec metricFunctionsSpec = metricFunctionsSpecList.get(0);
    assertEquals("sumSeries(a.b.c)", metricFunctionsSpec.getFunction());
    assertEquals(60, metricFunctionsSpec.getIntervalInSecs());
    Map<String, String> tags = metricFunctionsSpec.getTags();
    assertEquals(3, tags.size());
    assertEquals("sample_app1", tags.get("app_name"));
    assertEquals("test", tags.get("env"));
    assertEquals("custom_tag_value", tags.get("custom_tag"));
    assertEquals(true, metricFunctionsSpec.getMergeTags());
}
 
Example 6
Project: flink   Author: apache   File: OrcRowInputFormatTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testReadDecimalTypeFile() throws IOException {
	rowOrcInputFormat = new OrcRowInputFormat(getPath(TEST_FILE_DECIMAL), TEST_SCHEMA_DECIMAL, new Configuration());

	FileInputSplit[] splits = rowOrcInputFormat.createInputSplits(1);
	assertEquals(1, splits.length);
	rowOrcInputFormat.openInputFormat();
	rowOrcInputFormat.open(splits[0]);

	assertFalse(rowOrcInputFormat.reachedEnd());
	Row row = rowOrcInputFormat.nextRecord(null);

	// validate first row
	assertNotNull(row);
	assertEquals(1, row.getArity());
	assertEquals(BigDecimal.valueOf(-1000.5d), row.getField(0));

	// check correct number of rows
	long cnt = 1;
	while (!rowOrcInputFormat.reachedEnd()) {
		assertNotNull(rowOrcInputFormat.nextRecord(null));
		cnt++;
	}
	assertEquals(6000, cnt);
}
 
Example 7
Project: flink   Author: apache   File: OrcRowInputFormatTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testReadTimeTypeFile() throws IOException {
	rowOrcInputFormat = new OrcRowInputFormat(getPath(TEST_FILE_TIMETYPES), TEST_SCHEMA_TIMETYPES, new Configuration());

	FileInputSplit[] splits = rowOrcInputFormat.createInputSplits(1);
	assertEquals(1, splits.length);
	rowOrcInputFormat.openInputFormat();
	rowOrcInputFormat.open(splits[0]);

	assertFalse(rowOrcInputFormat.reachedEnd());
	Row row = rowOrcInputFormat.nextRecord(null);

	// validate first row
	assertNotNull(row);
	assertEquals(2, row.getArity());
	assertEquals(Timestamp.valueOf("1900-05-05 12:34:56.1"), row.getField(0));
	assertEquals(Date.valueOf("1900-12-25"), row.getField(1));

	// check correct number of rows
	long cnt = 1;
	while (!rowOrcInputFormat.reachedEnd()) {
		assertNotNull(rowOrcInputFormat.nextRecord(null));
		cnt++;
	}
	assertEquals(70000, cnt);
}
 
Example 8
Project: flink   Author: apache   File: FileInputFormatTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testReadMultiplePatterns() throws Exception {
	final String contents = "CONTENTS";

	// create some accepted, some ignored files

	File child1 = temporaryFolder.newFile("dataFile1.txt");
	File child2 = temporaryFolder.newFile("another_file.bin");
	createTempFiles(contents.getBytes(ConfigConstants.DEFAULT_CHARSET), child1, child2);

	// test that only the valid files are accepted

	Configuration configuration = new Configuration();

	final DummyFileInputFormat format = new DummyFileInputFormat();
	format.setFilePath(temporaryFolder.getRoot().toURI().toString());
	format.configure(configuration);
	format.setFilesFilter(new GlobFilePathFilter(
		Collections.singletonList("**"),
		Arrays.asList("**/another_file.bin", "**/dataFile1.txt")
	));
	FileInputSplit[] splits = format.createInputSplits(1);

	Assert.assertEquals(0, splits.length);
}
 
Example 9
Project: micro-server   Author: aol   File: ReadUtilsTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
@SneakyThrows
public void getFileInputStream() {
    TransferManager transferManager = mock(TransferManager.class);
    Download download = mock(Download.class);
    when(transferManager.download(anyString(), anyString(), any())).thenReturn(download);

    ReadUtils readUtils = new ReadUtils(transferManager,System.getProperty("java.io.tmpdir"));

    InputStream fileInputStream = readUtils.getFileInputStream("bucket", "key");
    assertNotNull(fileInputStream);

    verify(transferManager, times(1)).download(anyString(), anyString(), any(File.class));
    verify(download, times(1)).waitForCompletion();

    fileInputStream.close();
}
 
Example 10
Project: incubator-retired-mrql   Author: apache   File: FlinkMRQLFileInputFormat.java    License: Apache License 2.0 6 votes vote down vote up
private static DataSource read_directory ( String buffer ) {
    try {
        String[] s = buffer.split(DataSource.separator);
        int n = Integer.parseInt(s[1]);
        if (s[0].equals("Binary"))
            return new BinaryDataSource(s[2],Plan.conf);
        else if (s[0].equals("Generator"))
            return new GeneratorDataSource(s[2],Plan.conf);
        else if (s[0].equals("Text"))
            return new FlinkParsedDataSource(s[3],DataSource.parserDirectory.get(s[2]),
                                             ((Node)Tree.parse(s[4])).children());
        else throw new Error("Unrecognized data source: "+s[0]);
    } catch (Exception e) {
        throw new Error(e);
    }
}
 
Example 11
Project: gemfirexd-oss   Author: gemxd   File: FileStreamInputOutput.java    License: Apache License 2.0 6 votes vote down vote up
private final DataValueDescriptor[] readDVDBuffer() throws StandardException {
  try {

    if (!diagnoseFileData) {
      return DataType.readDVDArray(this);
    }

    // we are in diagnostic mode ....
    final DataValueDescriptor[] dvd = DataType.readDVDArray(this);

    if (GemFireXDUtils.TraceTempFileIO) {
      final String srcDvd = rowObjectsWritten.get(rowObjectsindex++);
      String dvdS = toString(dvd, new StringBuilder()).toString();
      SanityManager.ASSERT(srcDvd.equals(dvdS), srcDvd + " != " + dvdS);
    }

    return dvd;

  } catch (final IOException ioe) {
    throw StandardException.newException(SQLState.LOG_CANNOT_FLUSH, ioe);
  } catch (final ClassNotFoundException cnf) {
    throw StandardException.newException(SQLState.LOG_CANNOT_FLUSH, cnf);
  }
}
 
Example 12
Project: gemfirexd-oss   Author: gemxd   File: FileStreamInputOutput.java    License: Apache License 2.0 6 votes vote down vote up
private final void checkReadBounds(final int expected) throws IOException {
  if (rwBuffer.remaining() >= expected) {
    return;
  }

  if (GemFireXDUtils.TraceTempFileIO) {
    SanityManager.DEBUG_PRINT(GfxdConstants.TRACE_TEMP_FILE_IO,
        "[" + this.file + "] moving onto next buffer as remaining "
            + rwBuffer.remaining() + " doesn't fits expected " + expected);
    // new Throwable());
  }

  getNextBuffer();

  assert rwBuffer.remaining() >= expected: "remaining="
      + rwBuffer.remaining() + " expected=" + expected;
}
 
Example 13
Project: gemfirexd-oss   Author: gemxd   File: FileStreamInputOutput.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void readFully(final byte[] b, int off, int len) throws IOException {

  int remaining = rwBuffer.remaining();

  if (remaining >= len) {
    rwBuffer.get(b, off, len);
    return;
  }

  while (len > 0) {
    if (remaining >= len) {
      rwBuffer.get(b, off, len);
      return;
    }
    rwBuffer.get(b, off, remaining);
    off += remaining;
    len -= remaining;
    getNextBuffer();
    remaining = rwBuffer.capacity();
  }
}
 
Example 14
Project: Flink-CEPplus   Author: ljygz   File: OrcRowInputFormatTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testReadFileInSplits() throws IOException {

	rowOrcInputFormat = new OrcRowInputFormat(getPath(TEST_FILE_FLAT), TEST_SCHEMA_FLAT, new Configuration());
	rowOrcInputFormat.selectFields(0, 1);

	FileInputSplit[] splits = rowOrcInputFormat.createInputSplits(4);
	assertEquals(4, splits.length);
	rowOrcInputFormat.openInputFormat();

	long cnt = 0;
	// read all splits
	for (FileInputSplit split : splits) {

		// open split
		rowOrcInputFormat.open(split);
		// read and count all rows
		while (!rowOrcInputFormat.reachedEnd()) {
			assertNotNull(rowOrcInputFormat.nextRecord(null));
			cnt++;
		}
	}
	// check that all rows have been read
	assertEquals(1920800, cnt);
}
 
Example 15
Project: iaf   Author: ibissource   File: FileHandler.java    License: Apache License 2.0 6 votes vote down vote up
public SkipBomAndDeleteFileAfterReadInputStream(InputStream inputStream,
		File file, boolean deleteAfterRead, IPipeLineSession session)
		throws FileNotFoundException {
	super(inputStream);
	this.file = file;
	this.deleteAfterRead = deleteAfterRead;
	this.session = session;
	if (deleteAfterRead) {
		if (file == null) {
			// This should not happen. A configuration warning for
			// read_delete in combination with classpath should have
			// occurred already.
			throw new FileNotFoundException("No file object provided");
		} else {
			file.deleteOnExit();
		}
	}
}
 
Example 16
Project: big-c   Author: yncxcw   File: EditLogFileInputStream.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Read the header of fsedit log
 * @param in fsedit stream
 * @return the edit log version number
 * @throws IOException if error occurs
 */
@VisibleForTesting
static int readLogVersion(DataInputStream in, boolean verifyLayoutVersion)
    throws IOException, LogHeaderCorruptException {
  int logVersion;
  try {
    logVersion = in.readInt();
  } catch (EOFException eofe) {
    throw new LogHeaderCorruptException(
        "Reached EOF when reading log header");
  }
  if (verifyLayoutVersion &&
      (logVersion < HdfsConstants.NAMENODE_LAYOUT_VERSION || // future version
       logVersion > Storage.LAST_UPGRADABLE_LAYOUT_VERSION)) { // unsupported
    throw new LogHeaderCorruptException(
        "Unexpected version of the file system log file: "
        + logVersion + ". Current version = "
        + HdfsConstants.NAMENODE_LAYOUT_VERSION + ".");
  }
  return logVersion;
}
 
Example 17
Project: htmlunit   Author: HtmlUnit   File: HTMLInputElementTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * @throws Exception if the test fails
 */
@Test
@Alerts({"false", "true"})
public void readOnlyInputFile() throws Exception {
    final String html
        = HtmlPageTest.STANDARDS_MODE_PREFIX_
        + "<html>\n"
        + "<head><title>foo</title>\n"
        + "<script>\n"
        + "  function test() {\n"
        + "    var input = document.getElementById('myInput');\n"
        + "    alert(input.readOnly);\n"
        + "    input = document.getElementById('myReadonlyInput');\n"
        + "    alert(input.readOnly);\n"
        + "  }\n"
        + "  </script>\n"
        + "</head>\n"
        + "<body onload='test()'>\n"
        + "  <input id='myInput' type='file' value='some test'>\n"
        + "  <input id='myReadonlyInput' type='file' value='some test' readonly='false'>\n"
        + "</body></html>";

    loadPageWithAlerts2(html);
}
 
Example 18
Project: hadoopoffice   Author: ZuInnoTe   File: AbstractSpreadSheetFlinkFileInputFormat.java    License: Apache License 2.0 6 votes vote down vote up
/***
 * Read truststore for establishing certificate chain for signature validation
 * 
 * @param conf
 * @throws IOException
 * @throws FormatNotUnderstoodException
 */
private void readTrustStore() throws IOException, FormatNotUnderstoodException {
	if (((this.hocr.getSigTruststoreFile() != null) && (!"".equals(this.hocr.getSigTruststoreFile())))) {
		LOG.info("Reading truststore to validate certificate chain for signatures");
		FlinkKeyStoreManager fksm = new FlinkKeyStoreManager();
		try {
			fksm.openKeyStore(new Path(this.hocr.getSigTruststoreFile()), this.hocr.getSigTruststoreType(),
					this.hocr.getSigTruststorePassword());
			this.hocr.setX509CertificateChain(fksm.getAllX509Certificates());
		} catch (NoSuchAlgorithmException | CertificateException | KeyStoreException | IllegalArgumentException e) {
			LOG.error("Exception: ",e);
			throw new FormatNotUnderstoodException(
					"Cannot read truststore to establish certificate chain for signature validation " + e);
		}

	}
}
 
Example 19
Project: XmlToJson   Author: smart-fun   File: FileReader.java    License: Apache License 2.0 6 votes vote down vote up
public static String readFileFromInputStream(@NonNull InputStream inputStream) {

        InputStreamReader inputStreamReader = new InputStreamReader(inputStream);
        BufferedReader bufferedReader = new BufferedReader(inputStreamReader);

        StringBuilder result = new StringBuilder();
        String line;
        try {
            while ((line = bufferedReader.readLine()) != null) {
                result.append(line);
            }
            return result.toString();
        } catch (IOException exception) {
        } finally {
            try {
                bufferedReader.close();
            } catch (IOException e2) {
            }
            try {
                inputStreamReader.close();
            } catch (IOException e2) {
            }
        }
        return null;
    }
 
Example 20
Project: BlackLight   Author: PeterCxy   File: FileCacheManager.java    License: GNU General Public License v3.0 6 votes vote down vote up
private byte[] readInputStream(InputStream in) throws IOException {
	ByteArrayOutputStream opt = new ByteArrayOutputStream();
	byte[] buf = new byte[1024];
	int len = 0;
	while ((len = in.read(buf)) != -1) {
		opt.write(buf, 0, len);
	}
	in.close();
	byte[] ret;
	try {
		ret = opt.toByteArray();
	} catch (OutOfMemoryError e) {
		ret = null;
	}
	opt.close();
	return ret;
}
 
Example 21
Project: sftp-fs   Author: robtimus   File: SFTPFileSystemInputStreamTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testReadBulk() throws IOException {
    final String content = "Hello World";

    Path file = addFile("/foo");
    setContents(file, content);

    byte[] b = new byte[20];
    try (InputStream input = fileSystem.newInputStream(createPath("/foo"))) {
        assertEquals(0, input.read(b, 0, 0));
        assertEquals(5, input.read(b, 1, 5));
        assertArrayEquals(content.substring(0, 5).getBytes(), Arrays.copyOfRange(b, 1, 6));
        assertEquals(content.length() - 5, input.read(b));
        assertArrayEquals(content.substring(5).getBytes(), Arrays.copyOfRange(b, 0, content.length() - 5));
        assertEquals(-1, input.read(b));
    }
}
 
Example 22
Project: sftp-fs   Author: robtimus   File: SFTPFileSystemInputStreamTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testReadSingle() throws IOException {
    final String content = "Hello World";

    Path file = addFile("/foo");
    setContents(file, content);

    try (InputStream input = fileSystem.newInputStream(createPath("/foo"))) {
        assertEquals('H', input.read());
        assertEquals('e', input.read());
        assertEquals('l', input.read());
        assertEquals('l', input.read());
        assertEquals('o', input.read());
        assertEquals(' ', input.read());
        assertEquals('W', input.read());
        assertEquals('o', input.read());
        assertEquals('r', input.read());
        assertEquals('l', input.read());
        assertEquals('d', input.read());
        assertEquals(-1, input.read());
    }
}
 
Example 23
Project: TranskribusCore   Author: Transkribus   File: DeaFileUtils.java    License: GNU General Public License v3.0 6 votes vote down vote up
public static String readInputStreamAsString(InputStream is) throws FileNotFoundException, IOException {
	BufferedReader br = null;
	StringBuffer result = new StringBuffer();
	try {		
		InputStream in = new DataInputStream(is);
		br = new BufferedReader(new InputStreamReader(in));
		String strLine;
		// Read File Line By Line
		while ((strLine = br.readLine()) != null) {
			result.append(strLine + "\n");
		}
	} finally {
		br.close();
	}

	return result.toString();
}
 
Example 24
@Test(expectedExceptions = CommandLineException.class)
public void testEmptyInputFileListProvided() throws Exception {
    @SuppressWarnings("serial")
    final List<Target> phonyTargets = SimulatedTargets.phonyTargets(3);
    final List<File> inputFiles = Collections.emptyList();
    final File targetFile = createTargetFile(phonyTargets);
    final File inputListFile = createInputListFile(inputFiles);
    try {
        runTool(targetFile, inputFiles, inputListFile);
    } catch (final Exception ex) {
        throw ex;
    } finally {
        targetFile.delete();
        inputListFile.delete();
    }
}
 
Example 25
Project: wisdom   Author: wisdom-framework   File: AsyncInputStreamTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testReadSmallFileFromUrl() throws IOException, InterruptedException {
    latch = new CountDownLatch(1);
    final ByteArrayOutputStream bos = new ByteArrayOutputStream();
    File file = new File("src/test/resources/a_file.txt");
    URL url = file.toURI().toURL();
    Context context = vertx.getOrCreateContext();
    final AsyncInputStream async = new AsyncInputStream(vertx, executor,
            url.openStream())
            .endHandler(event -> {
                assertThat(bos.toString()).startsWith("This is a file.");
                latch.countDown();
            }).setContext(context);
    context.runOnContext(event -> async.handler(buffer -> {
        try {
            bos.write(buffer.getBytes());
        } catch (IOException e) {
            e.printStackTrace();
        }
    }));
    latch.await(30, TimeUnit.SECONDS);
}
 
Example 26
Project: hortonmachine   Author: TheHortonMachine   File: FileUtilities.java    License: GNU General Public License v3.0 6 votes vote down vote up
/**
 * Read from an inoutstream and convert the readed stuff to a String. Usefull for text files that are available as
 * streams.
 *
 * @param inputStream
 *
 * @return the read string
 *
 * @throws IOException
 */
public static String readInputStreamToString( InputStream inputStream ) throws IOException {
    // Create the byte list to hold the data
    List<Byte> bytesList = new ArrayList<Byte>();

    byte b = 0;
    while( (b = (byte) inputStream.read()) != -1 ) {
        bytesList.add(b);
    }
    // Close the input stream and return bytes
    inputStream.close();

    byte[] bArray = new byte[bytesList.size()];
    for( int i = 0; i < bArray.length; i++ ) {
        bArray[i] = bytesList.get(i);
    }

    String file = new String(bArray);
    return file;
}
 
Example 27
Project: actor-platform   Author: actorapp   File: AndroidInputFile.java    License: GNU Affero General Public License v3.0 6 votes vote down vote up
@Override
public Promise<FilePart> read(int fileOffset, int len) {
    return new Promise<>(resolver -> {
        executor.execute(() -> {
            try {
                byte[] data = new byte[len];
                randomAccessFile.seek(fileOffset);
                // TODO: Better reading. For big len result can be truncated
                randomAccessFile.read(data, 0, len);
                resolver.result(new FilePart(fileOffset, len, data));
            } catch (Exception e) {
                e.printStackTrace();
                resolver.error(e);
            }
        });
    });
}
 
Example 28
Project: Mahuta   Author: ConsenSys   File: BytesUtils.java    License: Apache License 2.0 6 votes vote down vote up
public static InputStream readFileInputStream(String path) {
    
    if(ValidatorUtils.isEmpty(path)) {
        return null;
    }

    try {
        ClassLoader classLoader = BytesUtils.class.getClassLoader();
        File file = new File(Objects.requireNonNull(classLoader.getResource(path)).getFile());

        return new FileInputStream(file);

    } catch (FileNotFoundException e) {
        throw new TechnicalException("File cannot be found...", e);
    }
}
 
Example 29
Project: Telegram   Author: DrKLO   File: EncryptedFileInputStream.java    License: GNU General Public License v2.0 6 votes vote down vote up
@Override
public int read(byte[] b, int off, int len) throws IOException {
    if (currentMode == MODE_CBC && fileOffset == 0) {
        byte[] temp = new byte[32];
        super.read(temp, 0, 32);
        Utilities.aesCbcEncryptionByteArraySafe(b, key, iv, off, len, fileOffset, 0);
        fileOffset += 32;
        skip((temp[0] & 0xff) - 32);
    }
    int result = super.read(b, off, len);
    if (currentMode == MODE_CBC) {
        Utilities.aesCbcEncryptionByteArraySafe(b, key, iv, off, len, fileOffset, 0);
    } else if (currentMode == MODE_CTR) {
        Utilities.aesCtrDecryptionByteArray(b, key, iv, off, len, fileOffset);
    }
    fileOffset += len;
    return result;
}
 
Example 30
Project: journaldev   Author: journaldev   File: InputStreamToString.java    License: MIT License 6 votes vote down vote up
/**
 * reading file and converting InputStream to String using BufferedReader
 * @param fileName
 * @return
 * @throws IOException
 */
public static String readFileToStringUsingBufferedReader(String fileName) throws IOException{
	FileInputStream fileInputStream = null;
	BufferedReader bufferedReader = null;
	StringBuilder inputSB = null;
	try{
	fileInputStream = new FileInputStream(fileName);
	bufferedReader = new BufferedReader(new InputStreamReader(fileInputStream, "UTF-8"));
	inputSB = new StringBuilder();
	String line = bufferedReader.readLine();
       while(line != null){
       	inputSB.append(line);
           line = bufferedReader.readLine();
           if(line != null){
           	//add new line character
           	inputSB.append("\n");
           }
       }
	}finally{
		bufferedReader.close();
		fileInputStream.close();
	}
	return inputSB.toString();
}
 
Example 31
Project: flink   Author: flink-tpc-ds   File: OrcRowInputFormatTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testReadTimeTypeFile() throws IOException {
	rowOrcInputFormat = new OrcRowInputFormat(getPath(TEST_FILE_TIMETYPES), TEST_SCHEMA_TIMETYPES, new Configuration());

	FileInputSplit[] splits = rowOrcInputFormat.createInputSplits(1);
	assertEquals(1, splits.length);
	rowOrcInputFormat.openInputFormat();
	rowOrcInputFormat.open(splits[0]);

	assertFalse(rowOrcInputFormat.reachedEnd());
	Row row = rowOrcInputFormat.nextRecord(null);

	// validate first row
	assertNotNull(row);
	assertEquals(2, row.getArity());
	assertEquals(Timestamp.valueOf("1900-05-05 12:34:56.1"), row.getField(0));
	assertEquals(Date.valueOf("1900-12-25"), row.getField(1));

	// check correct number of rows
	long cnt = 1;
	while (!rowOrcInputFormat.reachedEnd()) {
		assertNotNull(rowOrcInputFormat.nextRecord(null));
		cnt++;
	}
	assertEquals(70000, cnt);
}
 
Example 32
Project: flink   Author: flink-tpc-ds   File: FileInputFormatTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testReadMultiplePatterns() throws Exception {
	final String contents = "CONTENTS";

	// create some accepted, some ignored files

	File child1 = temporaryFolder.newFile("dataFile1.txt");
	File child2 = temporaryFolder.newFile("another_file.bin");
	createTempFiles(contents.getBytes(ConfigConstants.DEFAULT_CHARSET), child1, child2);

	// test that only the valid files are accepted

	Configuration configuration = new Configuration();

	final DummyFileInputFormat format = new DummyFileInputFormat();
	format.setFilePath(temporaryFolder.getRoot().toURI().toString());
	format.configure(configuration);
	format.setFilesFilter(new GlobFilePathFilter(
		Collections.singletonList("**"),
		Arrays.asList("**/another_file.bin", "**/dataFile1.txt")
	));
	FileInputSplit[] splits = format.createInputSplits(1);

	Assert.assertEquals(0, splits.length);
}
 
Example 33
Project: parquet-mr   Author: apache   File: ParquetFileReader.java    License: Apache License 2.0 6 votes vote down vote up
public BytesInput readAsBytesInput(int size) throws IOException {
  int available = stream.available();
  if (size > available) {
    // this is to workaround a bug where the compressedLength
    // of the chunk is missing the size of the header of the dictionary
    // to allow reading older files (using dictionary) we need this.
    // usually 13 to 19 bytes are missing
    int missingBytes = size - available;
    LOG.info("completed the column chunk with {} bytes", missingBytes);

    List<ByteBuffer> buffers = new ArrayList<>();
    buffers.addAll(stream.sliceBuffers(available));

    ByteBuffer lastBuffer = ByteBuffer.allocate(missingBytes);
    f.readFully(lastBuffer);
    buffers.add(lastBuffer);

    return BytesInput.from(buffers);
  }

  return super.readAsBytesInput(size);
}
 
Example 34
Project: ignite   Author: apache   File: LockedReadFileInput.java    License: Apache License 2.0 6 votes vote down vote up
/** {@inheritDoc} */
@Override public void ensure(int requested) throws IOException {
    int available = buffer().remaining();

    if (available >= requested)
        return;

    boolean readArchive = segmentAware.checkCanReadArchiveOrReserveWorkSegment(segmentId);
    try {
        if (readArchive && !isLastReadFromArchive) {
            isLastReadFromArchive = true;

            refreshIO();
        }

        super.ensure(requested);
    }
    finally {
        if (!readArchive)
            segmentAware.releaseWorkSegment(segmentId);
    }
}
 
Example 35
Project: ECFileCache   Author: XiaoMi   File: ECFileCacheInputStream.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Reads up to <code>len</code> bytes of data from the input stream into
 * an array of bytes.  An attempt is made to read as many as
 * <code>len</code> bytes, but a smaller number may be read.
 * The number of bytes actually read is returned as an integer.
 *
 * @param b     the buffer into which the data is read.
 * @param off   the start offset in array <code>b</code> at which the data is written.
 * @param len   the maximum number of bytes to read.
 * @return     the total number of bytes read into the buffer, or
 *             <code>-1</code> if the end of the stream has been reached.
 * @throws IOException
 */
@Override
public int read(byte[] b, int off, int len) throws IOException {
  checkIfClosed();
  if ((off | len | (off + len) | (b.length - (off + len))) < 0) {
    throw new IndexOutOfBoundsException();
  } else if (len == 0) {
    return 0;
  }

  int n = 0;
  while (true) {
    int nread = readRedisIfNeed(b, off + n, len - n);
    if (nread <= 0) {
      return (n == 0) ? nread : n;
    }
    n += nread;
    if (n >= len) {
      return n;
    }
    if (available() <= 0) {
      return n;
    }
  }
}
 
Example 36
Project: evosql   Author: SERG-Delft   File: TarFileInputStream.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Work-around for the problem that compressed InputReaders don't fill
 * the read buffer before returning.
 *
 * Has visibility 'protected' so that subclasses may override with
 * different algorithms, or use different algorithms for different
 * compression stream.
 */
protected void readCompressedBlocks(int blocks) throws IOException {

    int bytesSoFar    = 0;
    int requiredBytes = 512 * blocks;

    // This method works with individual bytes!
    int i;

    while (bytesSoFar < requiredBytes) {
        i = readStream.read(readBuffer, bytesSoFar,
                            requiredBytes - bytesSoFar);

        if (i < 0) {
            throw new EOFException(
                RB.decompression_ranout.getString(
                    bytesSoFar, requiredBytes));
        }

        bytesRead  += i;
        bytesSoFar += i;
    }
}
 
Example 37
Project: Telegram-FOSS   Author: Telegram-FOSS-Team   File: EncryptedFileInputStream.java    License: GNU General Public License v2.0 6 votes vote down vote up
@Override
public int read(byte[] b, int off, int len) throws IOException {
    if (currentMode == MODE_CBC && fileOffset == 0) {
        byte[] temp = new byte[32];
        super.read(temp, 0, 32);
        Utilities.aesCbcEncryptionByteArraySafe(b, key, iv, off, len, fileOffset, 0);
        fileOffset += 32;
        skip((temp[0] & 0xff) - 32);
    }
    int result = super.read(b, off, len);
    if (currentMode == MODE_CBC) {
        Utilities.aesCbcEncryptionByteArraySafe(b, key, iv, off, len, fileOffset, 0);
    } else if (currentMode == MODE_CTR) {
        Utilities.aesCtrDecryptionByteArray(b, key, iv, off, len, fileOffset);
    }
    fileOffset += len;
    return result;
}
 
Example 38
Project: hop   Author: project-hop   File: TextFileInputTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void readInputWithDefaultValues() throws Exception {
  final String virtualFile = createVirtualFile( "pdi-14832.txt", "1,\n" );

  BaseFileField col2 = field( "col2" );
  col2.setIfNullValue( "DEFAULT" );

  TextFileInputMeta meta = createMetaObject( field( "col1" ), col2 );
  TextFileInputData data = createDataObject( virtualFile, ",", "col1", "col2" );

  TextFileInput input = TransformMockUtil.getTransform( TextFileInput.class, TextFileInputMeta.class, "test" );
  List<Object[]> output = PipelineTestingUtil.execute( input, meta, data, 1, false );
  PipelineTestingUtil.assertResult( new Object[] { "1", "DEFAULT" }, output.get( 0 ) );

  deleteVfsFile( virtualFile );
}
 
Example 39
Project: hop   Author: project-hop   File: TextFileInputTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void readInputWithNonEmptyNullif() throws Exception {
  final String virtualFile = createVirtualFile( "pdi-14358.txt", "-,-\n" );

  BaseFileField col2 = field( "col2" );
  col2.setNullString( "-" );

  TextFileInputMeta meta = createMetaObject( field( "col1" ), col2 );
  TextFileInputData data = createDataObject( virtualFile, ",", "col1", "col2" );

  TextFileInput input = TransformMockUtil.getTransform( TextFileInput.class, TextFileInputMeta.class, "test" );
  List<Object[]> output = PipelineTestingUtil.execute( input, meta, data, 1, false );
  PipelineTestingUtil.assertResult( new Object[] { "-" }, output.get( 0 ) );

  deleteVfsFile( virtualFile );
}
 
Example 40
Project: arvo2parquet   Author: tideworks   File: InputFile.java    License: MIT License 6 votes vote down vote up
private static int readDirectBuffer(ByteBuffer byteBufr, byte[] tmpBuf, ByteBufReader rdr) throws IOException {
  // copy all the bytes that return immediately, stopping at the first
  // read that doesn't return a full buffer.
  int nextReadLength = Math.min(byteBufr.remaining(), tmpBuf.length);
  int totalBytesRead = 0;
  int bytesRead;

  while ((bytesRead = rdr.read(tmpBuf, 0, nextReadLength)) == tmpBuf.length) {
    byteBufr.put(tmpBuf);
    totalBytesRead += bytesRead;
    nextReadLength = Math.min(byteBufr.remaining(), tmpBuf.length);
  }

  if (bytesRead < 0) {
    // return -1 if nothing was read
    return totalBytesRead == 0 ? -1 : totalBytesRead;
  } else {
    // copy the last partial buffer
    byteBufr.put(tmpBuf, 0, bytesRead);
    totalBytesRead += bytesRead;
    return totalBytesRead;
  }
}
 
Example 41
Project: Leo   Author: QihooTest   File: FileUtil.java    License: Apache License 2.0 6 votes vote down vote up
/**
 *  读取字符流,指定编码
 * @param input
 * @param enCoding
 * @return String
 */
public static String readInputStreamToString(InputStream input,String enCoding) {
	BufferedReader reader ;
	StringBuilder sb = new StringBuilder();
	String line = null;
	try {
		reader = new BufferedReader(new InputStreamReader(input,enCoding));
		while ((line = reader.readLine()) != null) {
			sb.append(line + "\r\n");
		}
	} catch (IOException e) {
		log.error(e.getMessage());
	} finally {
		try {
			input.close();
		} catch (IOException e) {
			log.error(e.getMessage());
		}
	}
	return sb.toString();

}
 
Example 42
Project: hadoop   Author: naver   File: EditLogFileInputStream.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Read the header of fsedit log
 * @param in fsedit stream
 * @return the edit log version number
 * @throws IOException if error occurs
 */
@VisibleForTesting
static int readLogVersion(DataInputStream in, boolean verifyLayoutVersion)
    throws IOException, LogHeaderCorruptException {
  int logVersion;
  try {
    logVersion = in.readInt();
  } catch (EOFException eofe) {
    throw new LogHeaderCorruptException(
        "Reached EOF when reading log header");
  }
  if (verifyLayoutVersion &&
      (logVersion < HdfsConstants.NAMENODE_LAYOUT_VERSION || // future version
       logVersion > Storage.LAST_UPGRADABLE_LAYOUT_VERSION)) { // unsupported
    throw new LogHeaderCorruptException(
        "Unexpected version of the file system log file: "
        + logVersion + ". Current version = "
        + HdfsConstants.NAMENODE_LAYOUT_VERSION + ".");
  }
  return logVersion;
}
 
Example 43
Project: rtg-tools   Author: RealTimeGenomics   File: ClosedFileInputStreamTest.java    License: BSD 2-Clause "Simplified" License 6 votes vote down vote up
public void readAndCheckBlock(ClosedFileInputStream cfi, byte[] src, int position) throws IOException {
  final int toread = 100;
  final byte[] expected = new byte[toread];
  System.arraycopy(src, position, expected, 0, toread);

  final byte[] readdata = new byte[toread];
  cfi.seek(position);

  int read = 0;
  int batchread;
  while (read < toread && (batchread = cfi.read(readdata, read, toread - read)) != -1) {
    read += batchread;
  }

  assertTrue(Arrays.equals(expected, readdata));
}
 
Example 44
Project: envelope   Author: cloudera-labs   File: TestFileSystemInput.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void readJsonNoOptions() throws Exception {
  Map<String, Object> paramMap = new HashMap<>();
  paramMap.put(FileSystemInput.FORMAT_CONFIG, "json");
  paramMap.put(FileSystemInput.PATH_CONFIG, FileSystemInput.class.getResource(JSON_DATA).getPath());
  config = ConfigFactory.parseMap(paramMap);

  FileSystemInput csvInput = new FileSystemInput();
  assertNoValidationFailures(csvInput, config);
  csvInput.configure(config);

  Dataset<Row> dataFrame = csvInput.read();
  assertEquals(4, dataFrame.count());

  Row first = dataFrame.first();
  assertEquals("dog", first.getString(3));
  assertEquals("field1", first.schema().fields()[0].name());
  assertEquals(DataTypes.LongType, first.schema().fields()[0].dataType());
}
 
Example 45
Project: envelope   Author: cloudera-labs   File: TestFileSystemInput.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void readCsvNoOptions() throws Exception {
  Map<String, Object> paramMap = new HashMap<>();
  paramMap.put(FileSystemInput.FORMAT_CONFIG, "csv");
  paramMap.put(FileSystemInput.PATH_CONFIG, FileSystemInput.class.getResource(CSV_DATA).getPath());
  config = ConfigFactory.parseMap(paramMap);

  FileSystemInput csvInput = new FileSystemInput();
  assertNoValidationFailures(csvInput, config);
  csvInput.configure(config);

  Dataset<Row> dataFrame = csvInput.read();
  assertEquals(4, dataFrame.count());

  Row first = dataFrame.first();
  assertEquals("Four", first.getString(3));
}
 
Example 46
Project: envelope   Author: cloudera-labs   File: TestFileSystemInput.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void readCsvWithFieldsSchema() throws Exception {
  Map<String, Object> paramMap = new HashMap<>();
  paramMap.put(FileSystemInput.FORMAT_CONFIG, "csv");
  paramMap.put(FileSystemInput.PATH_CONFIG, FileSystemInput.class.getResource(CSV_DATA).getPath());
  paramMap.put(FileSystemInput.CSV_HEADER_CONFIG, "true");
  paramMap.put(FileSystemInput.SCHEMA_CONFIG + "." + ComponentFactory.TYPE_CONFIG_NAME, "flat");
  paramMap.put(FileSystemInput.SCHEMA_CONFIG + "." + FlatSchema.FIELD_NAMES_CONFIG,
               Lists.newArrayList("A Long", "An Int", "A String", "Another String"));
  paramMap.put(FileSystemInput.SCHEMA_CONFIG + "." + FlatSchema.FIELD_TYPES_CONFIG,
               Lists.newArrayList("long", "int", "string", "string"));
  config = ConfigFactory.parseMap(paramMap);

  FileSystemInput csvInput = new FileSystemInput();
  assertNoValidationFailures(csvInput, config);
  csvInput.configure(config);

  Dataset<Row> dataFrame = csvInput.read();
  assertEquals(3, dataFrame.count());

  Row first = dataFrame.first();
  assertEquals("four", first.getString(3));
  assertEquals("Another String", first.schema().fields()[3].name());
  assertEquals(1L, first.get(0));
  assertEquals(DataTypes.LongType, first.schema().fields()[0].dataType());
}
 
Example 47
Project: envelope   Author: cloudera-labs   File: TestFileSystemInput.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void readCsvWithAvroSchema() throws Exception {
  Map<String, Object> paramMap = new HashMap<>();
  paramMap.put(FileSystemInput.FORMAT_CONFIG, "csv");
  paramMap.put(FileSystemInput.PATH_CONFIG, FileSystemInput.class.getResource(CSV_DATA).getPath());
  paramMap.put(FileSystemInput.CSV_HEADER_CONFIG, "true");
  paramMap.put(FileSystemInput.SCHEMA_CONFIG + "." + ComponentFactory.TYPE_CONFIG_NAME, "avro");
  paramMap.put(FileSystemInput.SCHEMA_CONFIG + "." + AvroSchema.AVRO_FILE_CONFIG,
               FileSystemInput.class.getResource(TestAvroSchema.AVRO_SCHEMA_DATA).getPath());
  config = ConfigFactory.parseMap(paramMap);

  FileSystemInput csvInput = new FileSystemInput();
  assertNoValidationFailures(csvInput, config);
  csvInput.configure(config);

  Dataset<Row> dataFrame = csvInput.read();
  assertEquals(3, dataFrame.count());

  Row first = dataFrame.first();
  assertEquals("four", first.getString(3));
  assertEquals("Another_String", first.schema().fields()[3].name());
  assertEquals(1L, first.get(0));
  assertEquals(DataTypes.LongType, first.schema().fields()[0].dataType());
}
 
Example 48
Project: envelope   Author: cloudera-labs   File: FileSystemInput.java    License: Apache License 2.0 6 votes vote down vote up
private Dataset<Row> readText(String path) {
  Dataset<Row> lines = Contexts.getSparkSession().read().text(path);

  if (hasTranslator) {
    TranslateFunction translateFunction = getTranslateFunction(translatorConfig);
    TranslationResults results = new TranslationResults(
        lines.javaRDD().flatMap(translateFunction),
        translateFunction.getProvidingSchema(),
        getProvidingSchema());
    errors = results.getErrored();

    return results.getTranslated();
  }
  else {
    return lines;
  }
}
 
Example 49
Project: envelope   Author: cloudera-labs   File: TestFileSystemInput.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void readCsvWithOptions() throws Exception {
  Map<String, Object> paramMap = new HashMap<>();
  paramMap.put(FileSystemInput.FORMAT_CONFIG, "csv");
  paramMap.put(FileSystemInput.PATH_CONFIG, FileSystemInput.class.getResource(CSV_DATA).getPath());
  paramMap.put(FileSystemInput.CSV_HEADER_CONFIG, "true");
  config = ConfigFactory.parseMap(paramMap);

  FileSystemInput csvInput = new FileSystemInput();
  assertNoValidationFailures(csvInput, config);
  csvInput.configure(config);

  Dataset<Row> dataFrame = csvInput.read();
  assertEquals(3, dataFrame.count());

  Row first = dataFrame.first();
  assertEquals("four", first.getString(3));
}
 
Example 50
Project: envelope   Author: cloudera-labs   File: TestFileSystemInput.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void readTextWithoutTranslator() throws Exception {
  Map<String, Object> configMap = Maps.newHashMap();
  configMap.put(FileSystemInput.FORMAT_CONFIG, FileSystemInput.TEXT_FORMAT);
  configMap.put(FileSystemInput.PATH_CONFIG, FileSystemInput.class.getResource(TEXT_DATA).getPath());
  config = ConfigFactory.parseMap(configMap);
  
  FileSystemInput formatInput = new FileSystemInput();
  assertNoValidationFailures(formatInput, config);
  formatInput.configure(config);
  
  List<Row> results = formatInput.read().collectAsList();
  
  assertEquals(2, results.size());
  assertTrue(results.contains(RowFactory.create("a=1,b=hello,c=true")));
  assertTrue(results.contains(RowFactory.create("a=2,b=world,c=false")));
}
 
Example 51
Project: envelope   Author: cloudera-labs   File: FileSystemInput.java    License: Apache License 2.0 6 votes vote down vote up
private Dataset<Row> readInputFormat(String path) throws Exception {
  LOG.debug("Reading InputFormat[{}]: {}", inputType, path);

  Class<? extends InputFormat> inputFormatClass = Class.forName(inputType).asSubclass(InputFormat.class);

  TranslateFunction translateFunction = getTranslateFunction(translatorConfig);

  Dataset<Row> encoded = getEncodedRowsFromInputFormat(path, inputFormatClass);

  TranslationResults results = new TranslationResults(
      encoded.javaRDD().flatMap(translateFunction),
      translateFunction.getProvidingSchema(),
      getProvidingSchema());
  errors = results.getErrored();

  return results.getTranslated();
}
 
Example 52
Project: vespa   Author: vespa-engine   File: MockService.java    License: Apache License 2.0 6 votes vote down vote up
private void readInputFile(BufferedReader reader) throws IOException {
    StringBuilder sb = new StringBuilder();
    int ch;
    char prevChar = 0;
    while ((ch = reader.read()) >= 0) {
        char c = (char) ch;
        if (prevChar == '\n') {
            if (c == '\n') {
                parseEntry(sb.toString());
                sb = new StringBuilder();
                prevChar = 0;
                continue;
            } else {
                sb.append(prevChar);
            }
        }
        if (c != '\n') {
            sb.append(c);
        }
        prevChar = c;
    }
    parseEntry(sb.toString());
}
 
Example 53
Project: BiglyBT   Author: BiglySoftware   File: FileUtil.java    License: GNU General Public License v2.0 6 votes vote down vote up
public static String readInputStreamAsString(InputStream is, int size_limit,
		int timeoutMillis, String charSet)
		throws IOException {
	StringBuilder result = new StringBuilder(1024);
	long maxTimeMillis = System.currentTimeMillis() + timeoutMillis;
	byte[] buffer = new byte[1024];

	while (System.currentTimeMillis() < maxTimeMillis) {
		int readLength = Math.min(is.available(), buffer.length);

		int len = is.read(buffer, 0, readLength);
		if (len == -1)
			break;

		result.append(new String(buffer, 0, len, charSet));

		if (size_limit >= 0 && result.length() > size_limit) {

			result.setLength(size_limit);

			break;
		}
	}

	return (result.toString());
}
 
Example 54
Project: mt-flume   Author: javachen   File: TestResettableFileInputStream.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Ensure that we can simply read bytes from a file.
 * @throws IOException
 */
@Test
public void testBasicRead() throws IOException {
  String output = singleLineFileInit(file, Charsets.UTF_8);

  PositionTracker tracker = new DurablePositionTracker(meta, file.getPath());
  ResettableInputStream in = new ResettableFileInputStream(file,  tracker);

  String result = readLine(in, output.length());
  assertEquals(output, result);

  String afterEOF = readLine(in, output.length());
  assertNull(afterEOF);

  in.close();
}
 
Example 55
Project: dble   Author: actiontech   File: FileSystemRepository.java    License: GNU General Public License v2.0 6 votes vote down vote up
private static Collection<CoordinatorLogEntry> readFromInputStream(
        InputStream in) {
    Map<String, CoordinatorLogEntry> coordinatorLogEntries = new HashMap<>();
    BufferedReader br = null;
    try {
        InputStreamReader isr = new InputStreamReader(in);
        br = new BufferedReader(isr);
        coordinatorLogEntries = readContent(br);
    } catch (Exception e) {
        LOGGER.warn("Error in recover", e);
        AlertUtil.alertSelf(AlarmCode.XA_READ_IO_FAIL, Alert.AlertLevel.WARN, "Error in recover:" + e.getMessage(), null);

    } finally {
        closeSilently(br);
    }
    return coordinatorLogEntries.values();
}
 
Example 56
Project: RDFS   Author: iVCE   File: EditLogFileInputStream.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Read the header of fsedit log
 * @param in fsedit stream
 * @return the edit log version number
 * @throws IOException if error occurs
 */
static int readLogVersion(DataInputStream in) throws IOException,
    LogHeaderCorruptException {
  int logVersion = 0;
  // Read log file version. Could be missing.
  in.mark(4);
  // If edits log is greater than 2G, available method will return negative
  // numbers, so we avoid having to call available
  boolean available = true;
  try {
    logVersion = in.readByte();
  } catch (EOFException e) {
    available = false;
  }

  if (available) {
    in.reset();
    logVersion = in.readInt();
    if (logVersion < FSConstants.LAYOUT_VERSION) { // future version
      throw new LogHeaderCorruptException(
          "Unexpected version of the file system log file: " + logVersion
              + ". Current version = " + FSConstants.LAYOUT_VERSION + ".");
    }
  }
  return logVersion;
}
 
Example 57
Project: Java8CN   Author: Java8-CNAPI-Team   File: FileCacheImageInputStream.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Ensures that at least <code>pos</code> bytes are cached,
 * or the end of the source is reached.  The return value
 * is equal to the smaller of <code>pos</code> and the
 * length of the source file.
 */
private long readUntil(long pos) throws IOException {
    // We've already got enough data cached
    if (pos < length) {
        return pos;
    }
    // pos >= length but length isn't getting any bigger, so return it
    if (foundEOF) {
        return length;
    }

    long len = pos - length;
    cache.seek(length);
    while (len > 0) {
        // Copy a buffer's worth of data from the source to the cache
        // BUFFER_LENGTH will always fit into an int so this is safe
        int nbytes =
            stream.read(buf, 0, (int)Math.min(len, (long)BUFFER_LENGTH));
        if (nbytes == -1) {
            foundEOF = true;
            return length;
        }

        cache.write(buf, 0, nbytes);
        len -= nbytes;
        length += nbytes;
    }

    return pos;
}
 
Example 58
Project: ApkToolPlus   Author: linchaolong   File: ClassFileReader.java    License: Apache License 2.0 5 votes vote down vote up
/**
    Converts a class file to a <tt>ClassFile</tt> structure.
    @param is the input stream from which to read the
              <tt>ClassFile</tt> structure
    @return the new <tt>ClassFile</tt> structure
    @throws InvalidByteCodeException if the code is invalid
    @throws IOException if an exception occurs while reading from
                        the input stream
 */
public static ClassFile readFromInputStream(InputStream is)
    throws InvalidByteCodeException, IOException
{

    DataInputStream in = new DataInputStream(
                            new BufferedInputStream(is));

    ClassFile classFile = new ClassFile();
    classFile.read(in);
    in.close();
    return classFile;
}
 
Example 59
Project: incubator-iotdb   Author: apache   File: RowTsFileInputFormatTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testReadData() throws IOException {
	TsFileInputFormat<Row> inputFormat = prepareInputFormat(sourceTsFilePath1);

	List<String> actual = new ArrayList<>();

	try {
		inputFormat.configure(new Configuration());
		inputFormat.openInputFormat();
		FileInputSplit[] inputSplits = inputFormat.createInputSplits(2);
		Row reuse = rowTypeInfo.createSerializer(new ExecutionConfig()).createInstance();
		for (FileInputSplit inputSplit : inputSplits) {
			try {
				inputFormat.open(inputSplit);
				assertEquals(config.getBatchSize(), TSFileDescriptor.getInstance().getConfig().getBatchSize());
				while (!inputFormat.reachedEnd()) {
					Row row = inputFormat.nextRecord(reuse);
					actual.add(row.toString());
				}
			} finally {
				inputFormat.close();
			}
		}
	} finally {
		inputFormat.closeInputFormat();
	}

	String[] expected = {
		"1,1.2,20,null,2.3,11,19",
		"2,null,20,50,25.4,10,21",
		"3,1.4,21,null,null,null,null",
		"4,1.2,20,51,null,null,null",
		"6,7.2,10,11,null,null,null",
		"7,6.2,20,21,null,null,null",
		"8,9.2,30,31,null,null,null"
	};
	assertArrayEquals(actual.toArray(), expected);
}
 
Example 60
private int readFromTail() {
    if (pos < tail.length()) {
        return tail.charAt(pos++);
    }
    pos = 0;
    if (readingHeader) {
        readingHeader = false;
        tail = dataTail;
    }
    return -1;
}