Java Code Examples for java.nio.channels.FileChannel#map()
The following examples show how to use
java.nio.channels.FileChannel#map() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: MappedReadBuffer.java From jdk8u_jdk with GNU General Public License v2.0 | 6 votes |
|
static ReadBuffer create(RandomAccessFile file) throws IOException {
FileChannel ch = file.getChannel();
long size = ch.size();
// if file size is more than 2 GB and when file mapping is
// configured (default), use mapped file reader
if (canUseFileMap() && (size <= Integer.MAX_VALUE)) {
MappedByteBuffer buf;
try {
buf = ch.map(FileChannel.MapMode.READ_ONLY, 0, size);
ch.close();
return new MappedReadBuffer(buf);
} catch (IOException exp) {
exp.printStackTrace();
System.err.println("File mapping failed, will use direct read");
// fall through
}
} // else fall through
return new FileReadBuffer(file);
}
Example 2
| Source File: MappedReadBuffer.java From hottub with GNU General Public License v2.0 | 6 votes |
|
static ReadBuffer create(RandomAccessFile file) throws IOException {
FileChannel ch = file.getChannel();
long size = ch.size();
// if file size is more than 2 GB and when file mapping is
// configured (default), use mapped file reader
if (canUseFileMap() && (size <= Integer.MAX_VALUE)) {
MappedByteBuffer buf;
try {
buf = ch.map(FileChannel.MapMode.READ_ONLY, 0, size);
ch.close();
return new MappedReadBuffer(buf);
} catch (IOException exp) {
exp.printStackTrace();
System.err.println("File mapping failed, will use direct read");
// fall through
}
} // else fall through
return new FileReadBuffer(file);
}
Example 3
| Source File: TcpStorageServer.java From crail with Apache License 2.0 | 6 votes |
|
@Override
public StorageResource allocateResource() throws Exception {
StorageResource resource = null;
if (keys < regions){
int fileId = (int) keys++;
String dataFilePath = Paths.get(dataDirPath, Integer.toString(fileId)).toString();
RandomAccessFile dataFile = new RandomAccessFile(dataFilePath, "rw");
FileChannel dataChannel = dataFile.getChannel();
ByteBuffer buffer = dataChannel.map(MapMode.READ_WRITE, 0, TcpStorageConstants.STORAGE_TCP_ALLOCATION_SIZE);
dataBuffers.put(fileId, buffer);
dataFile.close();
dataChannel.close();
long address = CrailUtils.getAddress(buffer);
resource = StorageResource.createResource(address, buffer.capacity(), fileId);
// LOG.info("allocating resource, key " + resource.getKey() + ", address " + resource.getAddress() + ", length " + resource.getLength());
}
return resource;
}
Example 4
| Source File: MappedReadBuffer.java From TencentKona-8 with GNU General Public License v2.0 | 6 votes |
|
/**
* Factory method to create correct ReadBuffer for a given file.
*
* The initial purpose of this method was to choose how to read hprof file for parsing
* depending on the size of the file and the system property 'jhat.disableFileMap':
* "If file size is more than 2 GB and when file mapping is configured (default),
* use mapped file reader".
*
* However, it has been discovered a problem with this approach.
* Creating java.nio.MappedByteBuffer from inside the test leads to hprof file
* is locked on Windows until test process dies since there is no good way to
* release this resource.
*
* java.nio.MappedByteBuffer will be used only if 'jhat.enableFileMap' is set to true.
* Per default 'jhat.enableFileMap' is not set.
*/
static ReadBuffer create(RandomAccessFile file) throws IOException {
if (canUseFileMap()) {
MappedByteBuffer buf;
try {
FileChannel ch = file.getChannel();
long size = ch.size();
buf = ch.map(FileChannel.MapMode.READ_ONLY, 0, size);
ch.close();
return new MappedReadBuffer(file, buf);
} catch (IOException exp) {
exp.printStackTrace();
System.err.println("File mapping failed, will use direct read");
// fall through
}
} // else fall through
return new FileReadBuffer(file);
}
Example 5
| Source File: AnnotationParser.java From hortonmachine with GNU General Public License v3.0 | 6 votes |
|
/**
* Handle a file with an annotation handler.
*
* @param file
* @param ah
* @throws java.lang.Exception
*/
public static void handle(File srcFile, AnnotationHandler ah) throws Exception {
FileInputStream fis = new FileInputStream(srcFile);
FileChannel fc = fis.getChannel();
// Get a CharBuffer from the source file
ByteBuffer bb = fc.map(FileChannel.MapMode.READ_ONLY, 0, fc.size());
CharsetDecoder cd = Charset.forName("8859_1").newDecoder();
CharBuffer cb = cd.decode(bb);
// handle the content.
ah.start(cb.toString());
handle(cb.toString(), ah);
ah.done();
fis.close();
}
Example 6
| Source File: MappedByteBufferQueue.java From phoenix with Apache License 2.0 | 5 votes |
|
private void flush(T entry) throws IOException {
Queue<T> inMemQueue = getInMemoryQueue();
int resultSize = sizeOf(entry);
maxResultSize = Math.max(maxResultSize, resultSize);
totalResultSize = hasMaxQueueSize ? maxResultSize * inMemQueue.size() : (totalResultSize + resultSize);
if (totalResultSize >= thresholdBytes) {
this.file = File.createTempFile(UUID.randomUUID().toString(), null);
RandomAccessFile af = new RandomAccessFile(file, "rw");
FileChannel fc = af.getChannel();
int writeIndex = 0;
mappingSize = Math.min(Math.max(maxResultSize, DEFAULT_MAPPING_SIZE), totalResultSize);
MappedByteBuffer writeBuffer = fc.map(MapMode.READ_WRITE, writeIndex, mappingSize);
int resSize = inMemQueue.size();
for (int i = 0; i < resSize; i++) {
T e = inMemQueue.poll();
writeToBuffer(writeBuffer, e);
// buffer close to exhausted, re-map.
if (mappingSize - writeBuffer.position() < maxResultSize) {
writeIndex += writeBuffer.position();
writeBuffer = fc.map(MapMode.READ_WRITE, writeIndex, mappingSize);
}
}
writeBuffer.putInt(EOF); // end
fc.force(true);
fc.close();
af.close();
flushedCount = resSize;
inMemQueue.clear();
flushBuffer = true;
}
}
Example 7
| Source File: StoreReadWriteMmapNio.java From baratine with GNU General Public License v2.0 | 5 votes |
|
public MmapFile(FileChannel channel, long address, int size)
throws IOException
{
_mmap = channel.map(MapMode.READ_WRITE, address, size);
_address = address;
_size = size;
}
Example 8
| Source File: FileUtil.java From TestChat with Apache License 2.0 | 5 votes |
|
public static void writeToCache(ResponseBody responseBody, String savedFilePath, long readLength, long totalLength) {
try {
File file = newFile(savedFilePath);
long writeLength;
if (totalLength == 0) {
writeLength = responseBody.contentLength();
} else {
writeLength = totalLength;
}
FileChannel fileChannel;
RandomAccessFile randomAccessFile = new RandomAccessFile(file, "rwd");
fileChannel = randomAccessFile.getChannel();
MappedByteBuffer mappedByteBuffer = fileChannel.map(FileChannel.MapMode.READ_WRITE, readLength, writeLength);
byte[] buffer = new byte[1024 * 8];
int length = 0;
while ((length = responseBody.byteStream().read(buffer)) != -1) {
mappedByteBuffer.put(buffer, 0, length);
}
responseBody.byteStream().close();
if (fileChannel != null) {
fileChannel.close();
}
if (randomAccessFile != null) {
randomAccessFile.close();
}
} catch (IOException e) {
e.printStackTrace();
}
}
Example 9
| Source File: IPSeeker.java From codes-scratch-crawler with Apache License 2.0 | 5 votes |
|
/**
* 给定一个地点的不完全名字，得到一系列包含s子串的IP范围记录
*
* @param s 地点子串
* @return 包含IPEntry类型的List
*/
public List<IPEntry> getIPEntries(String s) {
List<IPEntry> ret = new ArrayList<IPEntry>();
try {
// 映射IP信息文件到内存中
if (mbb == null) {
FileChannel fc = ipFile.getChannel();
mbb = fc.map(FileChannel.MapMode.READ_ONLY, 0, ipFile.length());
mbb.order(ByteOrder.LITTLE_ENDIAN);
}
int endOffset = (int) ipEnd;
for (int offset = (int) ipBegin + 4; offset <= endOffset; offset += IP_RECORD_LENGTH) {
int temp = readInt3(offset);
if (temp != -1) {
IPLocation loc = getIPLocation(temp);
// 判断是否这个地点里面包含了s子串，如果包含了，添加这个记录到List中，如果没有，继续
if (loc.country.indexOf(s) != -1 || loc.area.indexOf(s) != -1) {
IPEntry entry = new IPEntry();
entry.country = loc.country;
entry.area = loc.area;
// 得到起始IP
readIP(offset - 4, b4);
entry.beginIp = Util.getIpStringFromBytes(b4);
// 得到结束IP
readIP(temp, b4);
entry.endIp = Util.getIpStringFromBytes(b4);
// 添加该记录
ret.add(entry);
}
}
}
} catch (IOException e) {
System.out.println(e.getMessage());
}
return ret;
}
Example 10
| Source File: RdmaStorageLocalEndpoint.java From incubator-crail with Apache License 2.0 | 5 votes |
|
private MappedByteBuffer mmap(File file) throws IOException{
RandomAccessFile randomFile = new RandomAccessFile(file.getAbsolutePath(), "rw");
FileChannel channel = randomFile.getChannel();
MappedByteBuffer mappedBuffer = channel.map(MapMode.READ_WRITE, 0, RdmaConstants.STORAGE_RDMA_ALLOCATION_SIZE);
randomFile.close();
return mappedBuffer;
}
Example 11
| Source File: IPSeeker.java From BigDataArchitect with Apache License 2.0 | 5 votes |
|
/**
* 给定一个地点的不完全名字，得到一系列包含s子串的IP范围记录
*
* @param s
* 地点子串
* @return 包含IPEntry类型的List
*/
public List getIPEntries(String s) {
List ret = new ArrayList();
try {
// 映射IP信息文件到内存中
if (mbb == null) {
FileChannel fc = ipFile.getChannel();
mbb = fc.map(FileChannel.MapMode.READ_ONLY, 0, ipFile.length());
mbb.order(ByteOrder.LITTLE_ENDIAN);
}
int endOffset = (int) ipEnd;
for (int offset = (int) ipBegin + 4; offset <= endOffset; offset += IP_RECORD_LENGTH) {
int temp = readInt3(offset);
if (temp != -1) {
IPLocation loc = getIPLocation(temp);
// 判断是否这个地点里面包含了s子串，如果包含了，添加这个记录到List中，如果没有，继续
if (loc.country.indexOf(s) != -1 || loc.area.indexOf(s) != -1) {
IPEntry entry = new IPEntry();
entry.country = loc.country;
entry.area = loc.area;
// 得到起始IP
readIP(offset - 4, b4);
entry.beginIp = IPSeekerUtils.getIpStringFromBytes(b4);
// 得到结束IP
readIP(temp, b4);
entry.endIp = IPSeekerUtils.getIpStringFromBytes(b4);
// 添加该记录
ret.add(entry);
}
}
}
} catch (IOException e) {
System.out.println(e.getMessage());
}
return ret;
}
Example 12
| Source File: EncryptUtils.java From open-cloud with MIT License | 5 votes |
|
/**
* 获取文件SHA256值
*
* @return String
*/
public static String sha256(File file) {
try {
MessageDigest messagedigest = MessageDigest.getInstance("SHA-256");
FileInputStream in = new FileInputStream(file);
FileChannel ch = in.getChannel();
MappedByteBuffer byteBuffer = ch.map(FileChannel.MapMode.READ_ONLY, 0, file.length());
messagedigest.update(byteBuffer);
return bufferToHex(messagedigest.digest());
} catch (Exception e) {
return null;
}
}
Example 13
| Source File: DataParser.java From iBioSim with Apache License 2.0 | 5 votes |
|
/**
* A helper function. Read a file into a string. Thanks to erickson at
* http:/
* /stackoverflow.com/questions/326390/how-to-create-a-java-string-from
* -the-contents-of-a-file
*
* @param path
* : the path to the file
* @return
* @throws IOException
*/
public static String readFileToString(String path) throws IOException {
FileInputStream stream = new FileInputStream(new File(path));
try {
FileChannel fc = stream.getChannel();
MappedByteBuffer bb = fc.map(FileChannel.MapMode.READ_ONLY, 0, fc.size());
/* Instead of using default, pass in a decoder. */
return Charset.defaultCharset().decode(bb).toString();
}
finally {
stream.close();
}
}
Example 14
| Source File: FileUtil.java From jumbune with GNU Lesser General Public License v3.0 | 5 votes |
|
/**
* Read the contents of a file into String.
*
* @param path the path
* @return the string
* @throws IOException Signals that an I/O exception has occurred.
*/
public static String readFileIntoString(String path) throws IOException {
FileInputStream stream = new FileInputStream(new File(path));
try {
FileChannel fileChannel = stream.getChannel();
MappedByteBuffer mappedByteBuffer = fileChannel.map(FileChannel.MapMode.READ_ONLY, 0, fileChannel.size());
return Charset.defaultCharset().decode(mappedByteBuffer).toString();
} finally {
if(stream != null){
stream.close();
}
}
}
Example 15
| Source File: LongMappedBuffer.java From jelectrum with MIT License | 5 votes |
|
public LongMappedBuffer(File f, long total_size)
throws IOException
{
RandomAccessFile raf = new RandomAccessFile(f, "rw");
FileChannel chan = raf.getChannel();
this.total_size = total_size;
map_list=new ArrayList<>();
long opened = 0;
while(opened < total_size)
{
long len = Math.min(total_size - opened, MAP_SIZE);
MappedByteBuffer buf = chan.map(FileChannel.MapMode.READ_WRITE, opened, len);
opened += len;
map_list.add(buf);
}
byte_mappings = new byte[8];
for(int i=0; i<8; i++)
{
BitSet bs = new BitSet(8);
bs.set(i);
byte[] b = bs.toByteArray();
byte_mappings[i]=b[0];
}
}
Example 16
| Source File: PigTest.java From spork with Apache License 2.0 | 5 votes |
|
private static String readFile(File file) throws IOException {
FileInputStream stream = new FileInputStream(file);
try {
FileChannel fc = stream.getChannel();
MappedByteBuffer bb = fc.map(FileChannel.MapMode.READ_ONLY, 0, fc.size());
return Charset.defaultCharset().decode(bb).toString();
}
finally {
stream.close();
}
}
Example 17
| Source File: Machos.java From buck with Apache License 2.0 | 5 votes |
|
static boolean isMacho(FileChannel file) throws IOException {
MappedByteBuffer map = file.map(FileChannel.MapMode.READ_ONLY, 0, MH_MAGIC.length);
byte[] magic = ObjectFileScrubbers.getBytes(map, MH_MAGIC.length);
return Arrays.equals(MH_MAGIC, magic)
|| Arrays.equals(MH_CIGAM, magic)
|| Arrays.equals(MH_MAGIC_64, magic)
|| Arrays.equals(MH_CIGAM_64, magic);
}
Example 18
| Source File: V5PullConsumer.java From coding-snippets with MIT License | 4 votes |
|
public V5PullConsumer(KeyValue properties) throws IOException {
// System.out.println("DefaultPullConsumer start:"+Thread.currentThread().getName());
this.properties = properties;
/**
* 暂未考虑同步问题。现在测试例使用反射，没有并发，所以测试问题没发现问题。
* files为null代表是新的测试例,为避免本次测试影响下次需要初始化各个属性。
*/
if (files == null) {
String path = properties.getString("STORE_PATH");
files = new File(path).listFiles();
fileNum = files.length;
fileNames = new String[fileNum];
index = new AtomicInteger(0);
finshed = new AtomicInteger(0);
isFileFinshed = new HashMap<>();
// latch = new CountDownLatch(fileNum);
data = new HashMap<String, HashMap<String, List<BytesMessage>>>(256);
allOffset = new HashMap<String, HashMap<String, HashMap<Integer, Integer>>>(256);
for(int i=0; i<fileNum; i++){
fileNames[i] = files[i].getName();
// System.out.println(fileNames[i]);
isFileFinshed.put(fileNames[i], 2);
}
}
queueOffset = new HashMap<String, Integer>();
for(int i=0; i<fileNum; i++){
queueOffset.put(fileNames[i], 0);
}
int fileIndex = index.getAndIncrement();
//考虑文件数小于线程数
if(fileIndex < fileNum){
File file = files[fileIndex];
fileName = fileNames[fileIndex];
FileChannel fc = new FileInputStream(file).getChannel();
mbb = fc.map(FileChannel.MapMode.READ_ONLY, 0, fc.size());
// mbb.load();
}
consumerId = count++;
// listOffset = new HashMap<String, HashMap<String, Integer>>(256);
// System.out.println("DefaultPullConsumer end:"+Thread.currentThread().getName());
}
Example 19
| Source File: ExpandingMap.java From dragonwell8_jdk with GNU General Public License v2.0 | 4 votes |
|
public static void main(String[] args) throws Exception {
int initialSize = 20480*1024;
int maximumMapSize = 16*1024*1024;
int maximumFileSize = 300000000;
File file = File.createTempFile("exp", "tmp");
file.deleteOnExit();
RandomAccessFile f = new RandomAccessFile(file, "rw");
f.setLength(initialSize);
FileChannel fc = f.getChannel();
ByteBuffer[] buffers = new ByteBuffer[128];
System.out.format("map %d -> %d\n", 0, initialSize);
buffers[0] = fc.map(FileChannel.MapMode.READ_WRITE, 0, initialSize);
int currentBuffer = 0;
int currentSize = initialSize;
int currentPosition = 0;
ArrayList<String> junk = new ArrayList<String>();
while (currentPosition+currentSize < maximumFileSize) {
int inc = Math.max(1000*1024, (currentPosition+currentSize)/8);
int size = currentPosition+currentSize+inc;
f.setLength(size);
while (currentSize+inc > maximumMapSize) {
if (currentSize < maximumMapSize) {
System.out.format("map %d -> %d\n", currentPosition,
(currentPosition + maximumMapSize));
buffers[currentBuffer] = fc.map(FileChannel.MapMode.READ_WRITE,
currentPosition, maximumMapSize);
fillBuffer(buffers[currentBuffer], currentSize);
}
currentPosition += maximumMapSize;
inc = currentSize+inc-maximumMapSize;
currentSize = 0;
currentBuffer++;
if (currentBuffer == buffers.length) {
ByteBuffer[] old = buffers;
buffers = new ByteBuffer[currentBuffer+currentBuffer/2];
System.arraycopy(old, 0, buffers, 0, currentBuffer); }
}
currentSize += inc;
if (currentSize > 0) {
System.out.format("map %d -> %d\n", currentPosition,
(currentPosition + currentSize));
buffers[currentBuffer] = fc.map(FileChannel.MapMode.READ_WRITE,
currentPosition, currentSize);
fillBuffer(buffers[currentBuffer], currentSize-inc);
}
// busy loop needed to reproduce issue
long t = System.currentTimeMillis();
while (System.currentTimeMillis() < t+500) {
junk.add(String.valueOf(t));
if (junk.size() > 100000) junk.clear();
}
}
fc.close();
// cleanup the ref to mapped buffers so they can be GCed
for (int i = 0; i < buffers.length; i++)
buffers[i] = null;
System.gc();
// Take a nap to wait for the Cleaner to cleanup those unrefed maps
Thread.sleep(1000);
System.out.println("TEST PASSED");
}
Example 20
| Source File: ExpandingMap.java From openjdk-jdk9 with GNU General Public License v2.0 | 4 votes |
|
public static void main(String[] args) throws Exception {
int initialSize = 20480*1024;
int maximumMapSize = 16*1024*1024;
int maximumFileSize = 300000000;
File file = File.createTempFile("exp", "tmp");
file.deleteOnExit();
RandomAccessFile f = new RandomAccessFile(file, "rw");
f.setLength(initialSize);
FileChannel fc = f.getChannel();
ByteBuffer[] buffers = new ByteBuffer[128];
System.out.format("map %d -> %d\n", 0, initialSize);
buffers[0] = fc.map(FileChannel.MapMode.READ_WRITE, 0, initialSize);
int currentBuffer = 0;
int currentSize = initialSize;
int currentPosition = 0;
ArrayList<String> junk = new ArrayList<String>();
while (currentPosition+currentSize < maximumFileSize) {
int inc = Math.max(1000*1024, (currentPosition+currentSize)/8);
int size = currentPosition+currentSize+inc;
f.setLength(size);
while (currentSize+inc > maximumMapSize) {
if (currentSize < maximumMapSize) {
System.out.format("map %d -> %d\n", currentPosition,
(currentPosition + maximumMapSize));
buffers[currentBuffer] = fc.map(FileChannel.MapMode.READ_WRITE,
currentPosition, maximumMapSize);
fillBuffer(buffers[currentBuffer], currentSize);
}
currentPosition += maximumMapSize;
inc = currentSize+inc-maximumMapSize;
currentSize = 0;
currentBuffer++;
if (currentBuffer == buffers.length) {
ByteBuffer[] old = buffers;
buffers = new ByteBuffer[currentBuffer+currentBuffer/2];
System.arraycopy(old, 0, buffers, 0, currentBuffer); }
}
currentSize += inc;
if (currentSize > 0) {
System.out.format("map %d -> %d\n", currentPosition,
(currentPosition + currentSize));
buffers[currentBuffer] = fc.map(FileChannel.MapMode.READ_WRITE,
currentPosition, currentSize);
fillBuffer(buffers[currentBuffer], currentSize-inc);
}
// busy loop needed to reproduce issue
long t = System.currentTimeMillis();
while (System.currentTimeMillis() < t+500) {
junk.add(String.valueOf(t));
if (junk.size() > 100000) junk.clear();
}
}
fc.close();
// cleanup the ref to mapped buffers so they can be GCed
for (int i = 0; i < buffers.length; i++)
buffers[i] = null;
System.gc();
// Take a nap to wait for the Cleaner to cleanup those unrefed maps
Thread.sleep(1000);
System.out.println("TEST PASSED");
}
