Jave read input file
60 Jave code examples are found related to "
read input file".
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example.
Example 1
Source File: FileUtils.java From MyTv with Apache License 2.0 | 7 votes |
/** * 从输入流中读取数据,NIO方式 * * @param in * @return * @throws IOException */ public static byte[] readFromInputStream(InputStream in) throws IOException { if (in == null) { return null; } ReadableByteChannel rbc = Channels.newChannel(in); ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE); byte[] data = new byte[BUFFER_SIZE]; ByteArrayOutputStream baos = new ByteArrayOutputStream(); while (true) { int count = rbc.read(buffer); if (count == -1) { break; } if (count < BUFFER_SIZE) { data = new byte[count]; } buffer.flip(); buffer.get(data, 0, count); baos.write(data); buffer.clear(); } rbc.close(); return baos.toByteArray(); }
Example 2
Source File: ODirectFileInputStreamTest.java From herddb with Apache License 2.0 | 6 votes |
@Test public void readFullAndPartialBlock() throws Exception { Random random = new Random(); Path path = tmp.newFile().toPath(); int blockSize = (int) OpenFileUtils.getBlockSize(path); int writeSize = blockSize + blockSize / 2; assertTrue(writeSize > blockSize); byte[] data = new byte[writeSize]; random.nextBytes(data); Files.write(path, data); try (ODirectFileInputStream oo = new ODirectFileInputStream(path)) { byte[] fullread = new byte[blockSize * 2]; int read = oo.read(fullread); assertEquals(writeSize, read); assertEquals(2, oo.getReadBlocks()); } }
Example 3
Source File: CommandPrompt.java From biojava with GNU Lesser General Public License v2.1 | 6 votes |
private static LinkedHashMap<String, ProteinSequence> readInputFile(String inputLocation, AminoAcidCompositionTable aaTable) throws Exception{ FileInputStream inStream = new FileInputStream(inputLocation); CompoundSet<AminoAcidCompound> set; if(aaTable == null){ set = CaseFreeAminoAcidCompoundSet.getAminoAcidCompoundSet(); }else{ set = aaTable.getAminoAcidCompoundSet(); } LinkedHashMap<String, ProteinSequence> ret; if ( inputLocation.toLowerCase().contains(".gb")) { GenbankReader<ProteinSequence, AminoAcidCompound> genbankReader = new GenbankReader<ProteinSequence, AminoAcidCompound>( inStream, new GenericGenbankHeaderParser<ProteinSequence, AminoAcidCompound>(), new ProteinSequenceCreator(set)); ret = genbankReader.process(); } else { FastaReader<ProteinSequence, AminoAcidCompound> fastaReader = new FastaReader<ProteinSequence, AminoAcidCompound>( inStream, new GenericFastaHeaderParser<ProteinSequence, AminoAcidCompound>(), new ProteinSequenceCreator(set)); ret = fastaReader.process(); } return ret; }
Example 4
Source File: InputOutputUtils.java From ixa-pipe-pos with Apache License 2.0 | 6 votes |
/** * Read the file into an {@code ObjectStream}. * * @param infile * the string pointing to the file * @return the object stream */ public static ObjectStream<String> readFileIntoMarkableStreamFactory( final String infile) { InputStreamFactory inputStreamFactory = null; try { inputStreamFactory = new MarkableFileInputStreamFactory(new File(infile)); } catch (final FileNotFoundException e) { e.printStackTrace(); } ObjectStream<String> lineStream = null; try { lineStream = new PlainTextByLineStream(inputStreamFactory, "UTF-8"); } catch (final IOException e) { CmdLineUtil.handleCreateObjectStreamError(e); } return lineStream; }
Example 5
Source File: MetricFunctionsReaderTest.java From adaptive-alerting with Apache License 2.0 | 6 votes |
@Test public void testReadFromInputFile() { val functionInputFileName = "config/functions-test.txt"; List<MetricFunctionsSpec> metricFunctionsSpecList = MetricFunctionsReader .readFromInputFile(ClassLoader.getSystemResource(functionInputFileName).getPath()); assertEquals(1, metricFunctionsSpecList.size()); MetricFunctionsSpec metricFunctionsSpec = metricFunctionsSpecList.get(0); assertEquals("sumSeries(a.b.c)", metricFunctionsSpec.getFunction()); assertEquals(60, metricFunctionsSpec.getIntervalInSecs()); Map<String, String> tags = metricFunctionsSpec.getTags(); assertEquals(3, tags.size()); assertEquals("sample_app1", tags.get("app_name")); assertEquals("test", tags.get("env")); assertEquals("custom_tag_value", tags.get("custom_tag")); assertEquals(true, metricFunctionsSpec.getMergeTags()); }
Example 6
Source File: OrcRowInputFormatTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testReadDecimalTypeFile() throws IOException { rowOrcInputFormat = new OrcRowInputFormat(getPath(TEST_FILE_DECIMAL), TEST_SCHEMA_DECIMAL, new Configuration()); FileInputSplit[] splits = rowOrcInputFormat.createInputSplits(1); assertEquals(1, splits.length); rowOrcInputFormat.openInputFormat(); rowOrcInputFormat.open(splits[0]); assertFalse(rowOrcInputFormat.reachedEnd()); Row row = rowOrcInputFormat.nextRecord(null); // validate first row assertNotNull(row); assertEquals(1, row.getArity()); assertEquals(BigDecimal.valueOf(-1000.5d), row.getField(0)); // check correct number of rows long cnt = 1; while (!rowOrcInputFormat.reachedEnd()) { assertNotNull(rowOrcInputFormat.nextRecord(null)); cnt++; } assertEquals(6000, cnt); }
Example 7
Source File: OrcRowInputFormatTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testReadTimeTypeFile() throws IOException { rowOrcInputFormat = new OrcRowInputFormat(getPath(TEST_FILE_TIMETYPES), TEST_SCHEMA_TIMETYPES, new Configuration()); FileInputSplit[] splits = rowOrcInputFormat.createInputSplits(1); assertEquals(1, splits.length); rowOrcInputFormat.openInputFormat(); rowOrcInputFormat.open(splits[0]); assertFalse(rowOrcInputFormat.reachedEnd()); Row row = rowOrcInputFormat.nextRecord(null); // validate first row assertNotNull(row); assertEquals(2, row.getArity()); assertEquals(Timestamp.valueOf("1900-05-05 12:34:56.1"), row.getField(0)); assertEquals(Date.valueOf("1900-12-25"), row.getField(1)); // check correct number of rows long cnt = 1; while (!rowOrcInputFormat.reachedEnd()) { assertNotNull(rowOrcInputFormat.nextRecord(null)); cnt++; } assertEquals(70000, cnt); }
Example 8
Source File: FileInputFormatTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testReadMultiplePatterns() throws Exception { final String contents = "CONTENTS"; // create some accepted, some ignored files File child1 = temporaryFolder.newFile("dataFile1.txt"); File child2 = temporaryFolder.newFile("another_file.bin"); createTempFiles(contents.getBytes(ConfigConstants.DEFAULT_CHARSET), child1, child2); // test that only the valid files are accepted Configuration configuration = new Configuration(); final DummyFileInputFormat format = new DummyFileInputFormat(); format.setFilePath(temporaryFolder.getRoot().toURI().toString()); format.configure(configuration); format.setFilesFilter(new GlobFilePathFilter( Collections.singletonList("**"), Arrays.asList("**/another_file.bin", "**/dataFile1.txt") )); FileInputSplit[] splits = format.createInputSplits(1); Assert.assertEquals(0, splits.length); }
Example 9
Source File: ReadUtilsTest.java From micro-server with Apache License 2.0 | 6 votes |
@Test @SneakyThrows public void getFileInputStream() { TransferManager transferManager = mock(TransferManager.class); Download download = mock(Download.class); when(transferManager.download(anyString(), anyString(), any())).thenReturn(download); ReadUtils readUtils = new ReadUtils(transferManager,System.getProperty("java.io.tmpdir")); InputStream fileInputStream = readUtils.getFileInputStream("bucket", "key"); assertNotNull(fileInputStream); verify(transferManager, times(1)).download(anyString(), anyString(), any(File.class)); verify(download, times(1)).waitForCompletion(); fileInputStream.close(); }
Example 10
Source File: FlinkMRQLFileInputFormat.java From incubator-retired-mrql with Apache License 2.0 | 6 votes |
private static DataSource read_directory ( String buffer ) { try { String[] s = buffer.split(DataSource.separator); int n = Integer.parseInt(s[1]); if (s[0].equals("Binary")) return new BinaryDataSource(s[2],Plan.conf); else if (s[0].equals("Generator")) return new GeneratorDataSource(s[2],Plan.conf); else if (s[0].equals("Text")) return new FlinkParsedDataSource(s[3],DataSource.parserDirectory.get(s[2]), ((Node)Tree.parse(s[4])).children()); else throw new Error("Unrecognized data source: "+s[0]); } catch (Exception e) { throw new Error(e); } }
Example 11
Source File: FileStreamInputOutput.java From gemfirexd-oss with Apache License 2.0 | 6 votes |
private final DataValueDescriptor[] readDVDBuffer() throws StandardException { try { if (!diagnoseFileData) { return DataType.readDVDArray(this); } // we are in diagnostic mode .... final DataValueDescriptor[] dvd = DataType.readDVDArray(this); if (GemFireXDUtils.TraceTempFileIO) { final String srcDvd = rowObjectsWritten.get(rowObjectsindex++); String dvdS = toString(dvd, new StringBuilder()).toString(); SanityManager.ASSERT(srcDvd.equals(dvdS), srcDvd + " != " + dvdS); } return dvd; } catch (final IOException ioe) { throw StandardException.newException(SQLState.LOG_CANNOT_FLUSH, ioe); } catch (final ClassNotFoundException cnf) { throw StandardException.newException(SQLState.LOG_CANNOT_FLUSH, cnf); } }
Example 12
Source File: FileStreamInputOutput.java From gemfirexd-oss with Apache License 2.0 | 6 votes |
private final void checkReadBounds(final int expected) throws IOException { if (rwBuffer.remaining() >= expected) { return; } if (GemFireXDUtils.TraceTempFileIO) { SanityManager.DEBUG_PRINT(GfxdConstants.TRACE_TEMP_FILE_IO, "[" + this.file + "] moving onto next buffer as remaining " + rwBuffer.remaining() + " doesn't fits expected " + expected); // new Throwable()); } getNextBuffer(); assert rwBuffer.remaining() >= expected: "remaining=" + rwBuffer.remaining() + " expected=" + expected; }
Example 13
Source File: FileStreamInputOutput.java From gemfirexd-oss with Apache License 2.0 | 6 votes |
@Override public void readFully(final byte[] b, int off, int len) throws IOException { int remaining = rwBuffer.remaining(); if (remaining >= len) { rwBuffer.get(b, off, len); return; } while (len > 0) { if (remaining >= len) { rwBuffer.get(b, off, len); return; } rwBuffer.get(b, off, remaining); off += remaining; len -= remaining; getNextBuffer(); remaining = rwBuffer.capacity(); } }
Example 14
Source File: OrcRowInputFormatTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void testReadFileInSplits() throws IOException { rowOrcInputFormat = new OrcRowInputFormat(getPath(TEST_FILE_FLAT), TEST_SCHEMA_FLAT, new Configuration()); rowOrcInputFormat.selectFields(0, 1); FileInputSplit[] splits = rowOrcInputFormat.createInputSplits(4); assertEquals(4, splits.length); rowOrcInputFormat.openInputFormat(); long cnt = 0; // read all splits for (FileInputSplit split : splits) { // open split rowOrcInputFormat.open(split); // read and count all rows while (!rowOrcInputFormat.reachedEnd()) { assertNotNull(rowOrcInputFormat.nextRecord(null)); cnt++; } } // check that all rows have been read assertEquals(1920800, cnt); }
Example 15
Source File: FileHandler.java From iaf with Apache License 2.0 | 6 votes |
public SkipBomAndDeleteFileAfterReadInputStream(InputStream inputStream, File file, boolean deleteAfterRead, IPipeLineSession session) throws FileNotFoundException { super(inputStream); this.file = file; this.deleteAfterRead = deleteAfterRead; this.session = session; if (deleteAfterRead) { if (file == null) { // This should not happen. A configuration warning for // read_delete in combination with classpath should have // occurred already. throw new FileNotFoundException("No file object provided"); } else { file.deleteOnExit(); } } }
Example 16
Source File: EditLogFileInputStream.java From big-c with Apache License 2.0 | 6 votes |
/** * Read the header of fsedit log * @param in fsedit stream * @return the edit log version number * @throws IOException if error occurs */ @VisibleForTesting static int readLogVersion(DataInputStream in, boolean verifyLayoutVersion) throws IOException, LogHeaderCorruptException { int logVersion; try { logVersion = in.readInt(); } catch (EOFException eofe) { throw new LogHeaderCorruptException( "Reached EOF when reading log header"); } if (verifyLayoutVersion && (logVersion < HdfsConstants.NAMENODE_LAYOUT_VERSION || // future version logVersion > Storage.LAST_UPGRADABLE_LAYOUT_VERSION)) { // unsupported throw new LogHeaderCorruptException( "Unexpected version of the file system log file: " + logVersion + ". Current version = " + HdfsConstants.NAMENODE_LAYOUT_VERSION + "."); } return logVersion; }
Example 17
Source File: HTMLInputElementTest.java From htmlunit with Apache License 2.0 | 6 votes |
/** * @throws Exception if the test fails */ @Test @Alerts({"false", "true"}) public void readOnlyInputFile() throws Exception { final String html = HtmlPageTest.STANDARDS_MODE_PREFIX_ + "<html>\n" + "<head><title>foo</title>\n" + "<script>\n" + " function test() {\n" + " var input = document.getElementById('myInput');\n" + " alert(input.readOnly);\n" + " input = document.getElementById('myReadonlyInput');\n" + " alert(input.readOnly);\n" + " }\n" + " </script>\n" + "</head>\n" + "<body onload='test()'>\n" + " <input id='myInput' type='file' value='some test'>\n" + " <input id='myReadonlyInput' type='file' value='some test' readonly='false'>\n" + "</body></html>"; loadPageWithAlerts2(html); }
Example 18
Source File: AbstractSpreadSheetFlinkFileInputFormat.java From hadoopoffice with Apache License 2.0 | 6 votes |
/*** * Read truststore for establishing certificate chain for signature validation * * @param conf * @throws IOException * @throws FormatNotUnderstoodException */ private void readTrustStore() throws IOException, FormatNotUnderstoodException { if (((this.hocr.getSigTruststoreFile() != null) && (!"".equals(this.hocr.getSigTruststoreFile())))) { LOG.info("Reading truststore to validate certificate chain for signatures"); FlinkKeyStoreManager fksm = new FlinkKeyStoreManager(); try { fksm.openKeyStore(new Path(this.hocr.getSigTruststoreFile()), this.hocr.getSigTruststoreType(), this.hocr.getSigTruststorePassword()); this.hocr.setX509CertificateChain(fksm.getAllX509Certificates()); } catch (NoSuchAlgorithmException | CertificateException | KeyStoreException | IllegalArgumentException e) { LOG.error("Exception: ",e); throw new FormatNotUnderstoodException( "Cannot read truststore to establish certificate chain for signature validation " + e); } } }
Example 19
Source File: FileReader.java From XmlToJson with Apache License 2.0 | 6 votes |
public static String readFileFromInputStream(@NonNull InputStream inputStream) { InputStreamReader inputStreamReader = new InputStreamReader(inputStream); BufferedReader bufferedReader = new BufferedReader(inputStreamReader); StringBuilder result = new StringBuilder(); String line; try { while ((line = bufferedReader.readLine()) != null) { result.append(line); } return result.toString(); } catch (IOException exception) { } finally { try { bufferedReader.close(); } catch (IOException e2) { } try { inputStreamReader.close(); } catch (IOException e2) { } } return null; }
Example 20
Source File: FileCacheManager.java From BlackLight with GNU General Public License v3.0 | 6 votes |
private byte[] readInputStream(InputStream in) throws IOException { ByteArrayOutputStream opt = new ByteArrayOutputStream(); byte[] buf = new byte[1024]; int len = 0; while ((len = in.read(buf)) != -1) { opt.write(buf, 0, len); } in.close(); byte[] ret; try { ret = opt.toByteArray(); } catch (OutOfMemoryError e) { ret = null; } opt.close(); return ret; }
Example 21
Source File: SFTPFileSystemInputStreamTest.java From sftp-fs with Apache License 2.0 | 6 votes |
@Test public void testReadBulk() throws IOException { final String content = "Hello World"; Path file = addFile("/foo"); setContents(file, content); byte[] b = new byte[20]; try (InputStream input = fileSystem.newInputStream(createPath("/foo"))) { assertEquals(0, input.read(b, 0, 0)); assertEquals(5, input.read(b, 1, 5)); assertArrayEquals(content.substring(0, 5).getBytes(), Arrays.copyOfRange(b, 1, 6)); assertEquals(content.length() - 5, input.read(b)); assertArrayEquals(content.substring(5).getBytes(), Arrays.copyOfRange(b, 0, content.length() - 5)); assertEquals(-1, input.read(b)); } }
Example 22
Source File: SFTPFileSystemInputStreamTest.java From sftp-fs with Apache License 2.0 | 6 votes |
@Test public void testReadSingle() throws IOException { final String content = "Hello World"; Path file = addFile("/foo"); setContents(file, content); try (InputStream input = fileSystem.newInputStream(createPath("/foo"))) { assertEquals('H', input.read()); assertEquals('e', input.read()); assertEquals('l', input.read()); assertEquals('l', input.read()); assertEquals('o', input.read()); assertEquals(' ', input.read()); assertEquals('W', input.read()); assertEquals('o', input.read()); assertEquals('r', input.read()); assertEquals('l', input.read()); assertEquals('d', input.read()); assertEquals(-1, input.read()); } }
Example 23
Source File: DeaFileUtils.java From TranskribusCore with GNU General Public License v3.0 | 6 votes |
public static String readInputStreamAsString(InputStream is) throws FileNotFoundException, IOException { BufferedReader br = null; StringBuffer result = new StringBuffer(); try { InputStream in = new DataInputStream(is); br = new BufferedReader(new InputStreamReader(in)); String strLine; // Read File Line By Line while ((strLine = br.readLine()) != null) { result.append(strLine + "\n"); } } finally { br.close(); } return result.toString(); }
Example 24
Source File: CombineReadCountsIntegrationTest.java From gatk-protected with BSD 3-Clause "New" or "Revised" License | 6 votes |
@Test(expectedExceptions = CommandLineException.class) public void testEmptyInputFileListProvided() throws Exception { @SuppressWarnings("serial") final List<Target> phonyTargets = SimulatedTargets.phonyTargets(3); final List<File> inputFiles = Collections.emptyList(); final File targetFile = createTargetFile(phonyTargets); final File inputListFile = createInputListFile(inputFiles); try { runTool(targetFile, inputFiles, inputListFile); } catch (final Exception ex) { throw ex; } finally { targetFile.delete(); inputListFile.delete(); } }
Example 25
Source File: AsyncInputStreamTest.java From wisdom with Apache License 2.0 | 6 votes |
@Test public void testReadSmallFileFromUrl() throws IOException, InterruptedException { latch = new CountDownLatch(1); final ByteArrayOutputStream bos = new ByteArrayOutputStream(); File file = new File("src/test/resources/a_file.txt"); URL url = file.toURI().toURL(); Context context = vertx.getOrCreateContext(); final AsyncInputStream async = new AsyncInputStream(vertx, executor, url.openStream()) .endHandler(event -> { assertThat(bos.toString()).startsWith("This is a file."); latch.countDown(); }).setContext(context); context.runOnContext(event -> async.handler(buffer -> { try { bos.write(buffer.getBytes()); } catch (IOException e) { e.printStackTrace(); } })); latch.await(30, TimeUnit.SECONDS); }
Example 26
Source File: FileUtilities.java From hortonmachine with GNU General Public License v3.0 | 6 votes |
/** * Read from an inoutstream and convert the readed stuff to a String. Usefull for text files that are available as * streams. * * @param inputStream * * @return the read string * * @throws IOException */ public static String readInputStreamToString( InputStream inputStream ) throws IOException { // Create the byte list to hold the data List<Byte> bytesList = new ArrayList<Byte>(); byte b = 0; while( (b = (byte) inputStream.read()) != -1 ) { bytesList.add(b); } // Close the input stream and return bytes inputStream.close(); byte[] bArray = new byte[bytesList.size()]; for( int i = 0; i < bArray.length; i++ ) { bArray[i] = bytesList.get(i); } String file = new String(bArray); return file; }
Example 27
Source File: AndroidInputFile.java From actor-platform with GNU Affero General Public License v3.0 | 6 votes |
@Override public Promise<FilePart> read(int fileOffset, int len) { return new Promise<>(resolver -> { executor.execute(() -> { try { byte[] data = new byte[len]; randomAccessFile.seek(fileOffset); // TODO: Better reading. For big len result can be truncated randomAccessFile.read(data, 0, len); resolver.result(new FilePart(fileOffset, len, data)); } catch (Exception e) { e.printStackTrace(); resolver.error(e); } }); }); }
Example 28
Source File: BytesUtils.java From Mahuta with Apache License 2.0 | 6 votes |
public static InputStream readFileInputStream(String path) { if(ValidatorUtils.isEmpty(path)) { return null; } try { ClassLoader classLoader = BytesUtils.class.getClassLoader(); File file = new File(Objects.requireNonNull(classLoader.getResource(path)).getFile()); return new FileInputStream(file); } catch (FileNotFoundException e) { throw new TechnicalException("File cannot be found...", e); } }
Example 29
Source File: EncryptedFileInputStream.java From Telegram with GNU General Public License v2.0 | 6 votes |
@Override public int read(byte[] b, int off, int len) throws IOException { if (currentMode == MODE_CBC && fileOffset == 0) { byte[] temp = new byte[32]; super.read(temp, 0, 32); Utilities.aesCbcEncryptionByteArraySafe(b, key, iv, off, len, fileOffset, 0); fileOffset += 32; skip((temp[0] & 0xff) - 32); } int result = super.read(b, off, len); if (currentMode == MODE_CBC) { Utilities.aesCbcEncryptionByteArraySafe(b, key, iv, off, len, fileOffset, 0); } else if (currentMode == MODE_CTR) { Utilities.aesCtrDecryptionByteArray(b, key, iv, off, len, fileOffset); } fileOffset += len; return result; }
Example 30
Source File: InputStreamToString.java From journaldev with MIT License | 6 votes |
/** * reading file and converting InputStream to String using BufferedReader * @param fileName * @return * @throws IOException */ public static String readFileToStringUsingBufferedReader(String fileName) throws IOException{ FileInputStream fileInputStream = null; BufferedReader bufferedReader = null; StringBuilder inputSB = null; try{ fileInputStream = new FileInputStream(fileName); bufferedReader = new BufferedReader(new InputStreamReader(fileInputStream, "UTF-8")); inputSB = new StringBuilder(); String line = bufferedReader.readLine(); while(line != null){ inputSB.append(line); line = bufferedReader.readLine(); if(line != null){ //add new line character inputSB.append("\n"); } } }finally{ bufferedReader.close(); fileInputStream.close(); } return inputSB.toString(); }
Example 31
Source File: OrcRowInputFormatTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testReadTimeTypeFile() throws IOException { rowOrcInputFormat = new OrcRowInputFormat(getPath(TEST_FILE_TIMETYPES), TEST_SCHEMA_TIMETYPES, new Configuration()); FileInputSplit[] splits = rowOrcInputFormat.createInputSplits(1); assertEquals(1, splits.length); rowOrcInputFormat.openInputFormat(); rowOrcInputFormat.open(splits[0]); assertFalse(rowOrcInputFormat.reachedEnd()); Row row = rowOrcInputFormat.nextRecord(null); // validate first row assertNotNull(row); assertEquals(2, row.getArity()); assertEquals(Timestamp.valueOf("1900-05-05 12:34:56.1"), row.getField(0)); assertEquals(Date.valueOf("1900-12-25"), row.getField(1)); // check correct number of rows long cnt = 1; while (!rowOrcInputFormat.reachedEnd()) { assertNotNull(rowOrcInputFormat.nextRecord(null)); cnt++; } assertEquals(70000, cnt); }
Example 32
Source File: FileInputFormatTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testReadMultiplePatterns() throws Exception { final String contents = "CONTENTS"; // create some accepted, some ignored files File child1 = temporaryFolder.newFile("dataFile1.txt"); File child2 = temporaryFolder.newFile("another_file.bin"); createTempFiles(contents.getBytes(ConfigConstants.DEFAULT_CHARSET), child1, child2); // test that only the valid files are accepted Configuration configuration = new Configuration(); final DummyFileInputFormat format = new DummyFileInputFormat(); format.setFilePath(temporaryFolder.getRoot().toURI().toString()); format.configure(configuration); format.setFilesFilter(new GlobFilePathFilter( Collections.singletonList("**"), Arrays.asList("**/another_file.bin", "**/dataFile1.txt") )); FileInputSplit[] splits = format.createInputSplits(1); Assert.assertEquals(0, splits.length); }
Example 33
Source File: ParquetFileReader.java From parquet-mr with Apache License 2.0 | 6 votes |
public BytesInput readAsBytesInput(int size) throws IOException { int available = stream.available(); if (size > available) { // this is to workaround a bug where the compressedLength // of the chunk is missing the size of the header of the dictionary // to allow reading older files (using dictionary) we need this. // usually 13 to 19 bytes are missing int missingBytes = size - available; LOG.info("completed the column chunk with {} bytes", missingBytes); List<ByteBuffer> buffers = new ArrayList<>(); buffers.addAll(stream.sliceBuffers(available)); ByteBuffer lastBuffer = ByteBuffer.allocate(missingBytes); f.readFully(lastBuffer); buffers.add(lastBuffer); return BytesInput.from(buffers); } return super.readAsBytesInput(size); }
Example 34
Source File: LockedReadFileInput.java From ignite with Apache License 2.0 | 6 votes |
/** {@inheritDoc} */ @Override public void ensure(int requested) throws IOException { int available = buffer().remaining(); if (available >= requested) return; boolean readArchive = segmentAware.checkCanReadArchiveOrReserveWorkSegment(segmentId); try { if (readArchive && !isLastReadFromArchive) { isLastReadFromArchive = true; refreshIO(); } super.ensure(requested); } finally { if (!readArchive) segmentAware.releaseWorkSegment(segmentId); } }
Example 35
Source File: ECFileCacheInputStream.java From ECFileCache with Apache License 2.0 | 6 votes |
/** * Reads up to <code>len</code> bytes of data from the input stream into * an array of bytes. An attempt is made to read as many as * <code>len</code> bytes, but a smaller number may be read. * The number of bytes actually read is returned as an integer. * * @param b the buffer into which the data is read. * @param off the start offset in array <code>b</code> at which the data is written. * @param len the maximum number of bytes to read. * @return the total number of bytes read into the buffer, or * <code>-1</code> if the end of the stream has been reached. * @throws IOException */ @Override public int read(byte[] b, int off, int len) throws IOException { checkIfClosed(); if ((off | len | (off + len) | (b.length - (off + len))) < 0) { throw new IndexOutOfBoundsException(); } else if (len == 0) { return 0; } int n = 0; while (true) { int nread = readRedisIfNeed(b, off + n, len - n); if (nread <= 0) { return (n == 0) ? nread : n; } n += nread; if (n >= len) { return n; } if (available() <= 0) { return n; } } }
Example 36
Source File: TarFileInputStream.java From evosql with Apache License 2.0 | 6 votes |
/** * Work-around for the problem that compressed InputReaders don't fill * the read buffer before returning. * * Has visibility 'protected' so that subclasses may override with * different algorithms, or use different algorithms for different * compression stream. */ protected void readCompressedBlocks(int blocks) throws IOException { int bytesSoFar = 0; int requiredBytes = 512 * blocks; // This method works with individual bytes! int i; while (bytesSoFar < requiredBytes) { i = readStream.read(readBuffer, bytesSoFar, requiredBytes - bytesSoFar); if (i < 0) { throw new EOFException( RB.decompression_ranout.getString( bytesSoFar, requiredBytes)); } bytesRead += i; bytesSoFar += i; } }
Example 37
Source File: EncryptedFileInputStream.java From Telegram-FOSS with GNU General Public License v2.0 | 6 votes |
@Override public int read(byte[] b, int off, int len) throws IOException { if (currentMode == MODE_CBC && fileOffset == 0) { byte[] temp = new byte[32]; super.read(temp, 0, 32); Utilities.aesCbcEncryptionByteArraySafe(b, key, iv, off, len, fileOffset, 0); fileOffset += 32; skip((temp[0] & 0xff) - 32); } int result = super.read(b, off, len); if (currentMode == MODE_CBC) { Utilities.aesCbcEncryptionByteArraySafe(b, key, iv, off, len, fileOffset, 0); } else if (currentMode == MODE_CTR) { Utilities.aesCtrDecryptionByteArray(b, key, iv, off, len, fileOffset); } fileOffset += len; return result; }
Example 38
Source File: TextFileInputTest.java From hop with Apache License 2.0 | 6 votes |
@Test public void readInputWithDefaultValues() throws Exception { final String virtualFile = createVirtualFile( "pdi-14832.txt", "1,\n" ); BaseFileField col2 = field( "col2" ); col2.setIfNullValue( "DEFAULT" ); TextFileInputMeta meta = createMetaObject( field( "col1" ), col2 ); TextFileInputData data = createDataObject( virtualFile, ",", "col1", "col2" ); TextFileInput input = TransformMockUtil.getTransform( TextFileInput.class, TextFileInputMeta.class, "test" ); List<Object[]> output = PipelineTestingUtil.execute( input, meta, data, 1, false ); PipelineTestingUtil.assertResult( new Object[] { "1", "DEFAULT" }, output.get( 0 ) ); deleteVfsFile( virtualFile ); }
Example 39
Source File: TextFileInputTest.java From hop with Apache License 2.0 | 6 votes |
@Test public void readInputWithNonEmptyNullif() throws Exception { final String virtualFile = createVirtualFile( "pdi-14358.txt", "-,-\n" ); BaseFileField col2 = field( "col2" ); col2.setNullString( "-" ); TextFileInputMeta meta = createMetaObject( field( "col1" ), col2 ); TextFileInputData data = createDataObject( virtualFile, ",", "col1", "col2" ); TextFileInput input = TransformMockUtil.getTransform( TextFileInput.class, TextFileInputMeta.class, "test" ); List<Object[]> output = PipelineTestingUtil.execute( input, meta, data, 1, false ); PipelineTestingUtil.assertResult( new Object[] { "-" }, output.get( 0 ) ); deleteVfsFile( virtualFile ); }
Example 40
Source File: InputFile.java From arvo2parquet with MIT License | 6 votes |
private static int readDirectBuffer(ByteBuffer byteBufr, byte[] tmpBuf, ByteBufReader rdr) throws IOException { // copy all the bytes that return immediately, stopping at the first // read that doesn't return a full buffer. int nextReadLength = Math.min(byteBufr.remaining(), tmpBuf.length); int totalBytesRead = 0; int bytesRead; while ((bytesRead = rdr.read(tmpBuf, 0, nextReadLength)) == tmpBuf.length) { byteBufr.put(tmpBuf); totalBytesRead += bytesRead; nextReadLength = Math.min(byteBufr.remaining(), tmpBuf.length); } if (bytesRead < 0) { // return -1 if nothing was read return totalBytesRead == 0 ? -1 : totalBytesRead; } else { // copy the last partial buffer byteBufr.put(tmpBuf, 0, bytesRead); totalBytesRead += bytesRead; return totalBytesRead; } }
Example 41
Source File: FileUtil.java From Leo with Apache License 2.0 | 6 votes |
/** * 读取字符流,指定编码 * @param input * @param enCoding * @return String */ public static String readInputStreamToString(InputStream input,String enCoding) { BufferedReader reader ; StringBuilder sb = new StringBuilder(); String line = null; try { reader = new BufferedReader(new InputStreamReader(input,enCoding)); while ((line = reader.readLine()) != null) { sb.append(line + "\r\n"); } } catch (IOException e) { log.error(e.getMessage()); } finally { try { input.close(); } catch (IOException e) { log.error(e.getMessage()); } } return sb.toString(); }
Example 42
Source File: EditLogFileInputStream.java From hadoop with Apache License 2.0 | 6 votes |
/** * Read the header of fsedit log * @param in fsedit stream * @return the edit log version number * @throws IOException if error occurs */ @VisibleForTesting static int readLogVersion(DataInputStream in, boolean verifyLayoutVersion) throws IOException, LogHeaderCorruptException { int logVersion; try { logVersion = in.readInt(); } catch (EOFException eofe) { throw new LogHeaderCorruptException( "Reached EOF when reading log header"); } if (verifyLayoutVersion && (logVersion < HdfsConstants.NAMENODE_LAYOUT_VERSION || // future version logVersion > Storage.LAST_UPGRADABLE_LAYOUT_VERSION)) { // unsupported throw new LogHeaderCorruptException( "Unexpected version of the file system log file: " + logVersion + ". Current version = " + HdfsConstants.NAMENODE_LAYOUT_VERSION + "."); } return logVersion; }
Example 43
Source File: ClosedFileInputStreamTest.java From rtg-tools with BSD 2-Clause "Simplified" License | 6 votes |
public void readAndCheckBlock(ClosedFileInputStream cfi, byte[] src, int position) throws IOException { final int toread = 100; final byte[] expected = new byte[toread]; System.arraycopy(src, position, expected, 0, toread); final byte[] readdata = new byte[toread]; cfi.seek(position); int read = 0; int batchread; while (read < toread && (batchread = cfi.read(readdata, read, toread - read)) != -1) { read += batchread; } assertTrue(Arrays.equals(expected, readdata)); }
Example 44
Source File: TestFileSystemInput.java From envelope with Apache License 2.0 | 6 votes |
@Test public void readJsonNoOptions() throws Exception { Map<String, Object> paramMap = new HashMap<>(); paramMap.put(FileSystemInput.FORMAT_CONFIG, "json"); paramMap.put(FileSystemInput.PATH_CONFIG, FileSystemInput.class.getResource(JSON_DATA).getPath()); config = ConfigFactory.parseMap(paramMap); FileSystemInput csvInput = new FileSystemInput(); assertNoValidationFailures(csvInput, config); csvInput.configure(config); Dataset<Row> dataFrame = csvInput.read(); assertEquals(4, dataFrame.count()); Row first = dataFrame.first(); assertEquals("dog", first.getString(3)); assertEquals("field1", first.schema().fields()[0].name()); assertEquals(DataTypes.LongType, first.schema().fields()[0].dataType()); }
Example 45
Source File: TestFileSystemInput.java From envelope with Apache License 2.0 | 6 votes |
@Test public void readCsvNoOptions() throws Exception { Map<String, Object> paramMap = new HashMap<>(); paramMap.put(FileSystemInput.FORMAT_CONFIG, "csv"); paramMap.put(FileSystemInput.PATH_CONFIG, FileSystemInput.class.getResource(CSV_DATA).getPath()); config = ConfigFactory.parseMap(paramMap); FileSystemInput csvInput = new FileSystemInput(); assertNoValidationFailures(csvInput, config); csvInput.configure(config); Dataset<Row> dataFrame = csvInput.read(); assertEquals(4, dataFrame.count()); Row first = dataFrame.first(); assertEquals("Four", first.getString(3)); }
Example 46
Source File: TestFileSystemInput.java From envelope with Apache License 2.0 | 6 votes |
@Test public void readCsvWithFieldsSchema() throws Exception { Map<String, Object> paramMap = new HashMap<>(); paramMap.put(FileSystemInput.FORMAT_CONFIG, "csv"); paramMap.put(FileSystemInput.PATH_CONFIG, FileSystemInput.class.getResource(CSV_DATA).getPath()); paramMap.put(FileSystemInput.CSV_HEADER_CONFIG, "true"); paramMap.put(FileSystemInput.SCHEMA_CONFIG + "." + ComponentFactory.TYPE_CONFIG_NAME, "flat"); paramMap.put(FileSystemInput.SCHEMA_CONFIG + "." + FlatSchema.FIELD_NAMES_CONFIG, Lists.newArrayList("A Long", "An Int", "A String", "Another String")); paramMap.put(FileSystemInput.SCHEMA_CONFIG + "." + FlatSchema.FIELD_TYPES_CONFIG, Lists.newArrayList("long", "int", "string", "string")); config = ConfigFactory.parseMap(paramMap); FileSystemInput csvInput = new FileSystemInput(); assertNoValidationFailures(csvInput, config); csvInput.configure(config); Dataset<Row> dataFrame = csvInput.read(); assertEquals(3, dataFrame.count()); Row first = dataFrame.first(); assertEquals("four", first.getString(3)); assertEquals("Another String", first.schema().fields()[3].name()); assertEquals(1L, first.get(0)); assertEquals(DataTypes.LongType, first.schema().fields()[0].dataType()); }
Example 47
Source File: TestFileSystemInput.java From envelope with Apache License 2.0 | 6 votes |
@Test public void readCsvWithAvroSchema() throws Exception { Map<String, Object> paramMap = new HashMap<>(); paramMap.put(FileSystemInput.FORMAT_CONFIG, "csv"); paramMap.put(FileSystemInput.PATH_CONFIG, FileSystemInput.class.getResource(CSV_DATA).getPath()); paramMap.put(FileSystemInput.CSV_HEADER_CONFIG, "true"); paramMap.put(FileSystemInput.SCHEMA_CONFIG + "." + ComponentFactory.TYPE_CONFIG_NAME, "avro"); paramMap.put(FileSystemInput.SCHEMA_CONFIG + "." + AvroSchema.AVRO_FILE_CONFIG, FileSystemInput.class.getResource(TestAvroSchema.AVRO_SCHEMA_DATA).getPath()); config = ConfigFactory.parseMap(paramMap); FileSystemInput csvInput = new FileSystemInput(); assertNoValidationFailures(csvInput, config); csvInput.configure(config); Dataset<Row> dataFrame = csvInput.read(); assertEquals(3, dataFrame.count()); Row first = dataFrame.first(); assertEquals("four", first.getString(3)); assertEquals("Another_String", first.schema().fields()[3].name()); assertEquals(1L, first.get(0)); assertEquals(DataTypes.LongType, first.schema().fields()[0].dataType()); }
Example 48
Source File: FileSystemInput.java From envelope with Apache License 2.0 | 6 votes |
private Dataset<Row> readText(String path) { Dataset<Row> lines = Contexts.getSparkSession().read().text(path); if (hasTranslator) { TranslateFunction translateFunction = getTranslateFunction(translatorConfig); TranslationResults results = new TranslationResults( lines.javaRDD().flatMap(translateFunction), translateFunction.getProvidingSchema(), getProvidingSchema()); errors = results.getErrored(); return results.getTranslated(); } else { return lines; } }
Example 49
Source File: TestFileSystemInput.java From envelope with Apache License 2.0 | 6 votes |
@Test public void readCsvWithOptions() throws Exception { Map<String, Object> paramMap = new HashMap<>(); paramMap.put(FileSystemInput.FORMAT_CONFIG, "csv"); paramMap.put(FileSystemInput.PATH_CONFIG, FileSystemInput.class.getResource(CSV_DATA).getPath()); paramMap.put(FileSystemInput.CSV_HEADER_CONFIG, "true"); config = ConfigFactory.parseMap(paramMap); FileSystemInput csvInput = new FileSystemInput(); assertNoValidationFailures(csvInput, config); csvInput.configure(config); Dataset<Row> dataFrame = csvInput.read(); assertEquals(3, dataFrame.count()); Row first = dataFrame.first(); assertEquals("four", first.getString(3)); }
Example 50
Source File: TestFileSystemInput.java From envelope with Apache License 2.0 | 6 votes |
@Test public void readTextWithoutTranslator() throws Exception { Map<String, Object> configMap = Maps.newHashMap(); configMap.put(FileSystemInput.FORMAT_CONFIG, FileSystemInput.TEXT_FORMAT); configMap.put(FileSystemInput.PATH_CONFIG, FileSystemInput.class.getResource(TEXT_DATA).getPath()); config = ConfigFactory.parseMap(configMap); FileSystemInput formatInput = new FileSystemInput(); assertNoValidationFailures(formatInput, config); formatInput.configure(config); List<Row> results = formatInput.read().collectAsList(); assertEquals(2, results.size()); assertTrue(results.contains(RowFactory.create("a=1,b=hello,c=true"))); assertTrue(results.contains(RowFactory.create("a=2,b=world,c=false"))); }
Example 51
Source File: FileSystemInput.java From envelope with Apache License 2.0 | 6 votes |
private Dataset<Row> readInputFormat(String path) throws Exception { LOG.debug("Reading InputFormat[{}]: {}", inputType, path); Class<? extends InputFormat> inputFormatClass = Class.forName(inputType).asSubclass(InputFormat.class); TranslateFunction translateFunction = getTranslateFunction(translatorConfig); Dataset<Row> encoded = getEncodedRowsFromInputFormat(path, inputFormatClass); TranslationResults results = new TranslationResults( encoded.javaRDD().flatMap(translateFunction), translateFunction.getProvidingSchema(), getProvidingSchema()); errors = results.getErrored(); return results.getTranslated(); }
Example 52
Source File: MockService.java From vespa with Apache License 2.0 | 6 votes |
private void readInputFile(BufferedReader reader) throws IOException { StringBuilder sb = new StringBuilder(); int ch; char prevChar = 0; while ((ch = reader.read()) >= 0) { char c = (char) ch; if (prevChar == '\n') { if (c == '\n') { parseEntry(sb.toString()); sb = new StringBuilder(); prevChar = 0; continue; } else { sb.append(prevChar); } } if (c != '\n') { sb.append(c); } prevChar = c; } parseEntry(sb.toString()); }
Example 53
Source File: FileUtil.java From BiglyBT with GNU General Public License v2.0 | 6 votes |
public static String readInputStreamAsString(InputStream is, int size_limit, int timeoutMillis, String charSet) throws IOException { StringBuilder result = new StringBuilder(1024); long maxTimeMillis = System.currentTimeMillis() + timeoutMillis; byte[] buffer = new byte[1024]; while (System.currentTimeMillis() < maxTimeMillis) { int readLength = Math.min(is.available(), buffer.length); int len = is.read(buffer, 0, readLength); if (len == -1) break; result.append(new String(buffer, 0, len, charSet)); if (size_limit >= 0 && result.length() > size_limit) { result.setLength(size_limit); break; } } return (result.toString()); }
Example 54
Source File: TestResettableFileInputStream.java From mt-flume with Apache License 2.0 | 6 votes |
/** * Ensure that we can simply read bytes from a file. * @throws IOException */ @Test public void testBasicRead() throws IOException { String output = singleLineFileInit(file, Charsets.UTF_8); PositionTracker tracker = new DurablePositionTracker(meta, file.getPath()); ResettableInputStream in = new ResettableFileInputStream(file, tracker); String result = readLine(in, output.length()); assertEquals(output, result); String afterEOF = readLine(in, output.length()); assertNull(afterEOF); in.close(); }
Example 55
Source File: FileSystemRepository.java From dble with GNU General Public License v2.0 | 6 votes |
private static Collection<CoordinatorLogEntry> readFromInputStream( InputStream in) { Map<String, CoordinatorLogEntry> coordinatorLogEntries = new HashMap<>(); BufferedReader br = null; try { InputStreamReader isr = new InputStreamReader(in); br = new BufferedReader(isr); coordinatorLogEntries = readContent(br); } catch (Exception e) { LOGGER.warn("Error in recover", e); AlertUtil.alertSelf(AlarmCode.XA_READ_IO_FAIL, Alert.AlertLevel.WARN, "Error in recover:" + e.getMessage(), null); } finally { closeSilently(br); } return coordinatorLogEntries.values(); }
Example 56
Source File: EditLogFileInputStream.java From RDFS with Apache License 2.0 | 6 votes |
/** * Read the header of fsedit log * @param in fsedit stream * @return the edit log version number * @throws IOException if error occurs */ static int readLogVersion(DataInputStream in) throws IOException, LogHeaderCorruptException { int logVersion = 0; // Read log file version. Could be missing. in.mark(4); // If edits log is greater than 2G, available method will return negative // numbers, so we avoid having to call available boolean available = true; try { logVersion = in.readByte(); } catch (EOFException e) { available = false; } if (available) { in.reset(); logVersion = in.readInt(); if (logVersion < FSConstants.LAYOUT_VERSION) { // future version throw new LogHeaderCorruptException( "Unexpected version of the file system log file: " + logVersion + ". Current version = " + FSConstants.LAYOUT_VERSION + "."); } } return logVersion; }
Example 57
Source File: FileCacheImageInputStream.java From Java8CN with Apache License 2.0 | 5 votes |
/** * Ensures that at least <code>pos</code> bytes are cached, * or the end of the source is reached. The return value * is equal to the smaller of <code>pos</code> and the * length of the source file. */ private long readUntil(long pos) throws IOException { // We've already got enough data cached if (pos < length) { return pos; } // pos >= length but length isn't getting any bigger, so return it if (foundEOF) { return length; } long len = pos - length; cache.seek(length); while (len > 0) { // Copy a buffer's worth of data from the source to the cache // BUFFER_LENGTH will always fit into an int so this is safe int nbytes = stream.read(buf, 0, (int)Math.min(len, (long)BUFFER_LENGTH)); if (nbytes == -1) { foundEOF = true; return length; } cache.write(buf, 0, nbytes); len -= nbytes; length += nbytes; } return pos; }
Example 58
Source File: ClassFileReader.java From ApkToolPlus with Apache License 2.0 | 5 votes |
/** Converts a class file to a <tt>ClassFile</tt> structure. @param is the input stream from which to read the <tt>ClassFile</tt> structure @return the new <tt>ClassFile</tt> structure @throws InvalidByteCodeException if the code is invalid @throws IOException if an exception occurs while reading from the input stream */ public static ClassFile readFromInputStream(InputStream is) throws InvalidByteCodeException, IOException { DataInputStream in = new DataInputStream( new BufferedInputStream(is)); ClassFile classFile = new ClassFile(); classFile.read(in); in.close(); return classFile; }
Example 59
Source File: RowTsFileInputFormatTest.java From incubator-iotdb with Apache License 2.0 | 5 votes |
@Test public void testReadData() throws IOException { TsFileInputFormat<Row> inputFormat = prepareInputFormat(sourceTsFilePath1); List<String> actual = new ArrayList<>(); try { inputFormat.configure(new Configuration()); inputFormat.openInputFormat(); FileInputSplit[] inputSplits = inputFormat.createInputSplits(2); Row reuse = rowTypeInfo.createSerializer(new ExecutionConfig()).createInstance(); for (FileInputSplit inputSplit : inputSplits) { try { inputFormat.open(inputSplit); assertEquals(config.getBatchSize(), TSFileDescriptor.getInstance().getConfig().getBatchSize()); while (!inputFormat.reachedEnd()) { Row row = inputFormat.nextRecord(reuse); actual.add(row.toString()); } } finally { inputFormat.close(); } } } finally { inputFormat.closeInputFormat(); } String[] expected = { "1,1.2,20,null,2.3,11,19", "2,null,20,50,25.4,10,21", "3,1.4,21,null,null,null,null", "4,1.2,20,51,null,null,null", "6,7.2,10,11,null,null,null", "7,6.2,20,21,null,null,null", "8,9.2,30,31,null,null,null" }; assertArrayEquals(actual.toArray(), expected); }
Example 60
Source File: FileSetInputStream.java From jdbc-driver-csv with GNU Lesser General Public License v3.0 | 5 votes |
private int readFromTail() { if (pos < tail.length()) { return tail.charAt(pos++); } pos = 0; if (readingHeader) { readingHeader = false; tail = dataTail; } return -1; }