backtype.storm.utils.BufferFileInputStream Java Examples

The following examples show how to use backtype.storm.utils.BufferFileInputStream. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: NimbusData.java    From jstorm with Apache License 2.0 6 votes vote down vote up
public void createFileHandler() {
    ExpiredCallback<Object, Object> expiredCallback = new ExpiredCallback<Object, Object>() {
        @Override
        public void expire(Object key, Object val) {
            try {
                LOG.info("Close file " + String.valueOf(key));
                if (val != null) {
                    if (val instanceof Channel) {
                        Channel channel = (Channel) val;
                        channel.close();
                    } else if (val instanceof BufferFileInputStream) {
                        BufferFileInputStream is = (BufferFileInputStream) val;
                        is.close();
                    }
                }
            } catch (IOException e) {
                LOG.error(e.getMessage(), e);
            }

        }
    };
    int file_copy_expiration_secs = JStormUtils.parseInt(conf.get(Config.NIMBUS_FILE_COPY_EXPIRATION_SECS), 30);
    uploaders = new TimeCacheMap<>(file_copy_expiration_secs, expiredCallback);
    downloaders = new TimeCacheMap<>(file_copy_expiration_secs, expiredCallback);
}
 
Example #2
Source File: StormSubmitterForUCar.java    From PoseidonX with Apache License 2.0 5 votes vote down vote up
public static String submitJar(Map conf, String localJar, String uploadLocation, NimbusClient client) {
    if (localJar == null) {
        throw new RuntimeException("Must submit topologies using the 'storm' client script so that StormSubmitter knows which jar to upload.");
    }

    try {

        LOG.info("Uploading topology jar " + localJar + " to assigned location: " + uploadLocation);
        int bufferSize = 512 * 1024;
        Object maxBufSizeObject = conf.get(Config.NIMBUS_THRIFT_MAX_BUFFER_SIZE);
        if (maxBufSizeObject != null) {
            bufferSize = Utils.getInt(maxBufSizeObject) / 2;
        }

        BufferFileInputStream is = new BufferFileInputStream(localJar, bufferSize);
        while (true) {
            byte[] toSubmit = is.read();
            if (toSubmit.length == 0)
                break;
            client.getClient().uploadChunk(uploadLocation, ByteBuffer.wrap(toSubmit));
        }
        client.getClient().finishFileUpload(uploadLocation);
        LOG.info("Successfully uploaded topology jar to assigned location: " + uploadLocation);
        return uploadLocation;
    } catch (Exception e) {
        throw new RuntimeException(e);
    } finally {

    }
}
 
Example #3
Source File: StormSubmitterForUCar.java    From PoseidonX with Apache License 2.0 5 votes vote down vote up
public static String submitJar(Map conf, String localJar, String uploadLocation, NimbusClient client) {
    if (localJar == null) {
        throw new RuntimeException("Must submit topologies using the 'storm' client script so that StormSubmitter knows which jar to upload.");
    }

    try {

        LOG.info("Uploading topology jar " + localJar + " to assigned location: " + uploadLocation);
        int bufferSize = 512 * 1024;
        Object maxBufSizeObject = conf.get(Config.NIMBUS_THRIFT_MAX_BUFFER_SIZE);
        if (maxBufSizeObject != null) {
            bufferSize = Utils.getInt(maxBufSizeObject) / 2;
        }

        BufferFileInputStream is = new BufferFileInputStream(localJar, bufferSize);
        while (true) {
            byte[] toSubmit = is.read();
            if (toSubmit.length == 0)
                break;
            client.getClient().uploadChunk(uploadLocation, ByteBuffer.wrap(toSubmit));
        }
        client.getClient().finishFileUpload(uploadLocation);
        LOG.info("Successfully uploaded topology jar to assigned location: " + uploadLocation);
        return uploadLocation;
    } catch (Exception e) {
        throw new RuntimeException(e);
    } finally {

    }
}
 
Example #4
Source File: ServiceHandler.java    From jstorm with Apache License 2.0 5 votes vote down vote up
@Override
public ByteBuffer downloadChunk(String id) throws TException {
    TimeCacheMap<Object, Object> downloaders = data.getDownloaders();
    Object obj = downloaders.get(id);
    if (obj == null) {
        throw new TException("Could not find input stream for that id");
    }

    try {
        if (obj instanceof BufferFileInputStream) {

            BufferFileInputStream is = (BufferFileInputStream) obj;
            byte[] ret = is.read();
            if (ret != null) {
                downloaders.put(id, is);
                return ByteBuffer.wrap(ret);
            }
        } else {
            throw new TException("Object isn't BufferFileInputStream for " + id);
        }
    } catch (IOException e) {
        LOG.error("BufferFileInputStream read failed when downloadChunk ", e);
        throw new TException(e);
    }
    byte[] empty = {};
    return ByteBuffer.wrap(empty);
}
 
Example #5
Source File: StormSubmitter.java    From jstorm with Apache License 2.0 5 votes vote down vote up
public static String submitJar(Map conf, String localJar, String uploadLocation, NimbusClient client) {
    if (localJar == null) {
        throw new RuntimeException("Must submit topologies using the 'jstorm' client script so that " +
                "StormSubmitter knows which jar to upload.");
    }

    try {
        LOG.info("Uploading topology jar " + localJar + " to assigned location: " + uploadLocation);
        int bufferSize = 512 * 1024;
        Object maxBufSizeObject = conf.get(Config.NIMBUS_THRIFT_MAX_BUFFER_SIZE);
        if (maxBufSizeObject != null) {
            bufferSize = Utils.getInt(maxBufSizeObject) / 2;
        }

        BufferFileInputStream is = new BufferFileInputStream(localJar, bufferSize);
        while (true) {
            byte[] toSubmit = is.read();
            if (toSubmit.length == 0)
                break;
            client.getClient().uploadChunk(uploadLocation, ByteBuffer.wrap(toSubmit));
        }
        client.getClient().finishFileUpload(uploadLocation);
        LOG.info("Successfully uploaded topology jar to assigned location: " + uploadLocation);
        return uploadLocation;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}
 
Example #6
Source File: StormSubmitter.java    From eagle with Apache License 2.0 4 votes vote down vote up
/**
 * Submit jar file
 * @param conf the topology-specific configuration. See {@link Config}.
 * @param localJar file path of the jar file to submit
 * @param listener progress listener to track the jar file upload
 * @return the remote location of the submitted jar
 */
public static String submitJar(Map conf, String localJar, ProgressListener listener) {
    if (localJar == null) {
        throw new RuntimeException("Must submit topologies using the 'storm' client script so that StormSubmitter knows which jar to upload.");
    }

    NimbusClient client = NimbusClient.getConfiguredClient(conf);
    try {
        String uploadLocation = client.getClient().beginFileUpload();
        LOG.info("Uploading topology jar " + localJar + " to assigned location: " + uploadLocation);
        BufferFileInputStream is = new BufferFileInputStream(localJar, THRIFT_CHUNK_SIZE_BYTES);

        long totalSize = new File(localJar).length();
        if (listener != null) {
            listener.onStart(localJar, uploadLocation, totalSize);
        }

        long bytesUploaded = 0;
        while (true) {
            byte[] toSubmit = is.read();
            bytesUploaded += toSubmit.length;
            if (listener != null) {
                listener.onProgress(localJar, uploadLocation, bytesUploaded, totalSize);
            }

            if (toSubmit.length == 0) {
                break;
            }
            client.getClient().uploadChunk(uploadLocation, ByteBuffer.wrap(toSubmit));
        }
        client.getClient().finishFileUpload(uploadLocation);

        if (listener != null) {
            listener.onCompleted(localJar, uploadLocation, totalSize);
        }

        LOG.info("Successfully uploaded topology jar to assigned location: " + uploadLocation);
        return uploadLocation;
    } catch (Exception e) {
        throw new RuntimeException(e);
    } finally {
        client.close();
    }
}