Python keras.__version__() Examples

def collect_environment(self, overwrite_variables=None):
        import socket
        import os
        import pip
        import platform

        env = {}

        if not overwrite_variables:
            overwrite_variables = {}

        import aetros
        env['aetros_version'] = aetros.__version__
        env['python_version'] = platform.python_version()
        env['python_executable'] = sys.executable

        env['hostname'] = socket.gethostname()
        env['variables'] = dict(os.environ)
        env['variables'].update(overwrite_variables)

        if 'AETROS_SSH_KEY' in env['variables']: del env['variables']['AETROS_SSH_KEY']
        if 'AETROS_SSH_KEY_BASE64' in env['variables']: del env['variables']['AETROS_SSH_KEY_BASE64']

        env['pip_packages'] = sorted([[i.key, i.version] for i in pip.get_installed_distributions()])
        self.set_system_info('environment', env) 

def _get_decoder_initial_states(self):
        """
        Return decoder states as Input layers
        """
        decoder_states_inputs = []
        for units in self.encoder_layers:
            decoder_state_input_h = Input(shape=(units,))
            input_states = [decoder_state_input_h]
            if self.cell == LSTMCell:
                decoder_state_input_c = Input(shape=(units,))
                input_states = [decoder_state_input_h, decoder_state_input_c]
            decoder_states_inputs.extend(input_states)
        if keras.__version__ < '2.2':
            return list(reversed(decoder_states_inputs))
        else:
            return decoder_states_inputs 

def __init__(self, params):
        super(NeuralNetworkAlgorithm, self).__init__(params)

        self.library_version = keras.__version__

        self.rounds = additional.get("one_step", 1)
        self.max_iters = additional.get("max_steps", 1)
        self.learner_params = {
            "dense_layers": params.get("dense_layers"),
            "dense_1_size": params.get("dense_1_size"),
            "dense_2_size": params.get("dense_2_size"),
            "dropout": params.get("dropout"),
            "learning_rate": params.get("learning_rate"),
            "momentum": params.get("momentum"),
            "decay": params.get("decay"),
        }
        self.model = None  # we need input data shape to construct model

        if "model_architecture_json" in params:
            self.model = model_from_json(
                json.loads(params.get("model_architecture_json"))
            )
            self.compile_model()

        logger.debug("NeuralNetworkAlgorithm __init__") 

def _load_model(model_path, keras_module, **kwargs):
    keras_models = importlib.import_module(keras_module.__name__ + ".models")
    custom_objects = kwargs.pop("custom_objects", {})
    custom_objects_path = None
    if os.path.isdir(model_path):
        if os.path.isfile(os.path.join(model_path, _CUSTOM_OBJECTS_SAVE_PATH)):
            custom_objects_path = os.path.join(model_path, _CUSTOM_OBJECTS_SAVE_PATH)
        model_path = os.path.join(model_path, _MODEL_SAVE_PATH)
    if custom_objects_path is not None:
        import cloudpickle
        with open(custom_objects_path, "rb") as in_f:
            pickled_custom_objects = cloudpickle.load(in_f)
            pickled_custom_objects.update(custom_objects)
            custom_objects = pickled_custom_objects
    from distutils.version import StrictVersion
    if StrictVersion(keras_module.__version__.split('-')[0]) >= StrictVersion("2.2.3"):
        # NOTE: Keras 2.2.3 does not work with unicode paths in python2. Pass in h5py.File instead
        # of string to avoid issues.
        import h5py
        with h5py.File(os.path.abspath(model_path), "r") as model_path:
            return keras_models.load_model(model_path, custom_objects=custom_objects, **kwargs)
    else:
        # NOTE: Older versions of Keras only handle filepath.
        return keras_models.load_model(model_path, custom_objects=custom_objects, **kwargs) 

def version():
        import tensorflow # pylint: disable=E0401
        tensorflow.logging.set_verbosity(tensorflow.logging.ERROR)
        return tensorflow.__version__ #pylint: disable=E1101 

def conv_2d(filters, kernel_shape, strides, padding, input_shape=None):
    """
    Defines the right convolutional layer according to the
    version of Keras that is installed.
    :param filters: (required integer) the dimensionality of the output
                    space (i.e. the number output of filters in the
                    convolution)
    :param kernel_shape: (required tuple or list of 2 integers) specifies
                         the strides of the convolution along the width and
                         height.
    :param padding: (required string) can be either 'valid' (no padding around
                    input or feature map) or 'same' (pad to ensure that the
                    output feature map size is identical to the layer input)
    :param input_shape: (optional) give input shape if this is the first
                        layer of the model
    :return: the Keras layer
    """
    if LooseVersion(keras.__version__) >= LooseVersion('2.0.0'):
        if input_shape is not None:
            return Conv2D(filters=filters, kernel_size=kernel_shape,
                          strides=strides, padding=padding,
                          input_shape=input_shape)
        else:
            return Conv2D(filters=filters, kernel_size=kernel_shape,
                          strides=strides, padding=padding)
    else:
        if input_shape is not None:
            return Convolution2D(filters, kernel_shape[0], kernel_shape[1],
                                 subsample=strides, border_mode=padding,
                                 input_shape=input_shape)
        else:
            return Convolution2D(filters, kernel_shape[0], kernel_shape[1],
                                 subsample=strides, border_mode=padding) 

def save_model(model, filepath, overwrite=True):

    def get_json_type(obj):
        if hasattr(obj, 'get_config'):
            return {'class_name': obj.__class__.__name__,
                    'config': obj.get_config()}

        if type(obj).__module__ == np.__name__:
            return obj.item()

        if callable(obj) or type(obj).__name__ == type.__name__:
            return obj.__name__

        raise TypeError('Not JSON Serializable:', obj)

    import h5py
    from keras import __version__ as keras_version

    if not overwrite and os.path.isfile(filepath):
        proceed = keras.models.ask_to_proceed_with_overwrite(filepath)
        if not proceed:
            return

    f = h5py.File(filepath, 'w')
    f.attrs['keras_version'] = str(keras_version).encode('utf8')
    f.attrs['generator_config'] = json.dumps({
        'class_name': model.discriminator.__class__.__name__,
        'config': model.generator.get_config(),
    }, default=get_json_type).encode('utf8')
    f.attrs['discriminator_config'] = json.dumps({
        'class_name': model.discriminator.__class__.__name__,
        'config': model.discriminator.get_config(),
    }, default=get_json_type).encode('utf8')

    generator_weights_group = f.create_group('generator_weights')
    discriminator_weights_group = f.create_group('discriminator_weights')
    model.generator.save_weights_to_hdf5_group(generator_weights_group)
    model.discriminator.save_weights_to_hdf5_group(discriminator_weights_group)

    f.flush()
    f.close() 

def upload_keras_graph(self, model):
        from aetros.keras import model_to_graph
        import keras

        if keras.__version__[0] == '2':
            graph = model_to_graph(model)
            self.set_graph(graph) 

def set_generator_validation_nb(self, number):
        """
        sets self.nb_val_samples which is used in model.fit if input is a generator
        :param number:
        :return:
        """

        self.nb_val_samples = number
        diff_to_batch = number % self.get_batch_size()
        if diff_to_batch > 0:
            self.nb_val_samples += self.get_batch_size() - diff_to_batch

        import keras
        if '1' != keras.__version__[0]:
            self.nb_val_samples = self.nb_val_samples // self.get_batch_size() 

def conv_2d(filters, kernel_shape, strides, padding, input_shape=None):
    """
    Defines the right convolutional layer according to the
    version of Keras that is installed.
    :param filters: (required integer) the dimensionality of the output
                    space (i.e. the number output of filters in the
                    convolution)
    :param kernel_shape: (required tuple or list of 2 integers) specifies
                         the strides of the convolution along the width and
                         height.
    :param padding: (required string) can be either 'valid' (no padding around
                    input or feature map) or 'same' (pad to ensure that the
                    output feature map size is identical to the layer input)
    :param input_shape: (optional) give input shape if this is the first
                        layer of the model
    :return: the Keras layer
    """
    if LooseVersion(keras.__version__) >= LooseVersion('2.0.0'):
        print "Using Keras version", keras.__version__
        if input_shape is not None:
            return Conv2D(filters=filters, kernel_size=kernel_shape,
                          strides=strides, padding=padding,
                          input_shape=input_shape)
        else:
            return Conv2D(filters=filters, kernel_size=kernel_shape,
                          strides=strides, padding=padding)
    else:
        print "Using *old* keras version", keras.__version__
        if input_shape is not None:
            return Convolution2D(filters, kernel_shape[0], kernel_shape[1],
                                 subsample=strides, border_mode=padding,
                                 input_shape=input_shape)
        else:
            return Convolution2D(filters, kernel_shape[0], kernel_shape[1],
                                 subsample=strides, border_mode=padding) 

def conv_2d(filters, kernel_shape, strides, padding, input_shape=None):
    """
    Defines the right convolutional layer according to the
    version of Keras that is installed.
    :param filters: (required integer) the dimensionality of the output
                    space (i.e. the number output of filters in the
                    convolution)
    :param kernel_shape: (required tuple or list of 2 integers) specifies
                         the strides of the convolution along the width and
                         height.
    :param padding: (required string) can be either 'valid' (no padding around
                    input or feature map) or 'same' (pad to ensure that the
                    output feature map size is identical to the layer input)
    :param input_shape: (optional) give input shape if this is the first
                        layer of the model
    :return: the Keras layer
    """
    if LooseVersion(keras.__version__) >= LooseVersion('2.0.0'):
        print "Using Keras version", keras.__version__
        if input_shape is not None:
            return Conv2D(filters=filters, kernel_size=kernel_shape,
                          strides=strides, padding=padding,
                          input_shape=input_shape)
        else:
            return Conv2D(filters=filters, kernel_size=kernel_shape,
                          strides=strides, padding=padding)
    else:
        print "Using *old* keras version", keras.__version__
        if input_shape is not None:
            return Convolution2D(filters, kernel_shape[0], kernel_shape[1],
                                 subsample=strides, border_mode=padding,
                                 input_shape=input_shape)
        else:
            return Convolution2D(filters, kernel_shape[0], kernel_shape[1],
                                 subsample=strides, border_mode=padding) 

def keras_version():
    return tuple(map(int, keras.__version__.split('.'))) 

def assert_keras_version():
    detected = keras.__version__
    required = '.'.join(map(str, minimum_keras_version))
    assert(keras_version() >= minimum_keras_version), 'You are using keras version {}. The minimum required version is {}.'.format(detected, required) 

def keras_version():
    """ Get the Keras version.

    Returns
        tuple of (major, minor, patch).
    """
    return tuple(map(int, keras.__version__.split('.'))) 

def assert_keras_version():
    """ Assert that the Keras version is up to date.
    """
    detected = keras.__version__
    required = '.'.join(map(str, minimum_keras_version))
    assert(keras_version() >= minimum_keras_version), 'You are using keras version {}. The minimum required version is {}.'.format(detected, required) 

def test_import_keras(self):
        self.query(udf.fixindent('''
                CREATE OR REPLACE python3 scalar SCRIPT tfbasic.import_keras()
                returns varchar(1000) as
                import keras
                import tensorflow
                import tensorflow_hub
                
                def run(ctx):
                    return str(keras.__version__)
                /
                '''))

        row = self.query("select tfbasic.import_keras()")[0] 

def _format_encoder_states(self, encoder_states, use_first=True):
        """
        Format the encoder states in such a way that only the last state from the first layer of the encoder
        is used to init the first layer of the decoder.
        If the cell type used is LSTM then both c and h are kept.
        :param encoder_states: Keras.tensor
            (last) hidden state of the decoder
        :param use_first: bool
            if True use only the last hidden state from first layer of the encoder, while the other are init to zero.
            if False use last hidden state for all layers
        :return:
            masked encoder states
        """
        if use_first:
            # Keras version 2.1.4 has encoder states reversed w.r.t later versions
            if keras.__version__ < '2.2':
                if self.cell == 'lstm':
                    encoder_states = [Lambda(lambda x: K.zeros_like(x))(s) for s in encoder_states[:-2]] + [
                        encoder_states[-2]]
                else:
                    encoder_states = [Lambda(lambda x: K.zeros_like(x))(s) for s in encoder_states[:-1]] + [
                        encoder_states[-1]]
            else:
                if self.cell == 'lstm':
                    encoder_states = encoder_states[:2] + [Lambda(lambda x: K.zeros_like(x))(s) for s in
                                                                encoder_states[2:]]
                else:
                    encoder_states = encoder_states[:1] + [Lambda(lambda x: K.zeros_like(x))(s) for s in
                                                                encoder_states[1:]]
        return encoder_states 

def keras_version():
    """ Get the Keras version.

    Returns
        tuple of (major, minor, patch).
    """
    return tuple(map(int, keras.__version__.split('.'))) 

def assert_keras_version():
    """ Assert that the Keras version is up to date.
    """
    detected = keras.__version__
    required = '.'.join(map(str, minimum_keras_version))
    assert(keras_version() >= minimum_keras_version), 'You are using keras version {}. The minimum required version is {}.'.format(detected, required) 

def keras_version():
    """ Get the Keras version.

    Returns
        tuple of (major, minor, patch).
    """
    return tuple(map(int, keras.__version__.split('.'))) 

def assert_keras_version():
    """ Assert that the Keras version is up to date.
    """
    detected = keras.__version__
    required = '.'.join(map(str, minimum_keras_version))
    assert(keras_version() >= minimum_keras_version), 'You are using keras version {}. The minimum required version is {}.'.format(detected, required) 

def __init__(self, axes='YX', n_channel_in=1, n_channel_out=1, probabilistic=False, allow_new_parameters=False, **kwargs):
        """See class docstring."""

        super(Config, self).__init__(axes, n_channel_in, n_channel_out)
        not ('Z' in self.axes and 'T' in self.axes) or _raise(ValueError('using Z and T axes together not supported.'))

        self.probabilistic         = bool(probabilistic)

        # default config (can be overwritten by kwargs below)
        self.unet_residual         = self.n_channel_in == self.n_channel_out
        self.unet_n_depth          = 2
        self.unet_kern_size        = 5 if self.n_dim==2 else 3
        self.unet_n_first          = 32
        self.unet_last_activation  = 'linear'
        if backend_channels_last():
            self.unet_input_shape  = self.n_dim*(None,) + (self.n_channel_in,)
        else:
            self.unet_input_shape  = (self.n_channel_in,) + self.n_dim*(None,)

        self.train_loss            = 'laplace' if self.probabilistic else 'mae'
        self.train_epochs          = 100
        self.train_steps_per_epoch = 400
        self.train_learning_rate   = 0.0004
        self.train_batch_size      = 16
        self.train_tensorboard     = True

        # the parameter 'min_delta' was called 'epsilon' for keras<=2.1.5
        min_delta_key = 'epsilon' if LooseVersion(keras.__version__)<=LooseVersion('2.1.5') else 'min_delta'
        self.train_reduce_lr       = {'factor': 0.5, 'patience': 10, min_delta_key: 0}

        # disallow setting 'n_dim' manually
        try:
            del kwargs['n_dim']
            # warnings.warn("ignoring parameter 'n_dim'")
        except:
            pass

        self.update_parameters(allow_new_parameters, **kwargs) 

def prog(self):#Show progress
        nb_batches_total=(self.params['epochs'] if kv-1 else self.params['nb_epoch'])*(self.params['samples'] if kv-1 else self.params['nb_sample'])/self.params['batch_size']
        nb_batches_epoch=(self.params['samples'] if kv-1 else self.params['nb_sample'])/self.params['batch_size']
        prog_total=(self.t_batches/nb_batches_total if nb_batches_total else 0)+0.01
        prog_epoch=(self.c_batches/nb_batches_epoch if nb_batches_epoch else 0)+0.01
        if self.t_epochs:
            now=time.time()
            t_mean=float(sum(self.t_epochs)) / len(self.t_epochs)
            eta_t=(now-self.train_start)*((1/prog_total)-1)
            eta_e=t_mean*(1-prog_epoch)
            t_end=time.asctime(time.localtime(now+eta_t))
            e_end=time.asctime(time.localtime(now+eta_e))
            m='\nTotal:\nProg:'+str(prog_total*100.)[:5]+'%\nEpoch:'+str(self.epoch[-1])+'/'+str(self.stopped_epoch)+'\nETA:'+str(eta_t)[:8]+'sec\nTrain will be finished at '+t_end+'\nCurrent epoch:\nPROG:'+str(prog_epoch*100.)[:5]+'%\nETA:'+str(eta_e)[:8]+'sec\nCurrent epoch will be finished at '+e_end
            self.t_send(msg=m)
            print(m)
        else:
            now=time.time()
            eta_t=(now-self.train_start)*((1/prog_total)-1)
            eta_e=(now-self.train_start)*((1/prog_epoch)-1)
            t_end=time.asctime(time.localtime(now+eta_t))
            e_end=time.asctime(time.localtime(now+eta_e))
            m='\nTotal:\nProg:'+str(prog_total*100.)[:5]+'%\nEpoch:'+str(len(self.epoch))+'/'+str(self.stopped_epoch)+'\nETA:'+str(eta_t)[:8]+'sec\nTrain will be finished at '+t_end+'\nCurrent epoch:\nPROG:'+str(prog_epoch*100.)[:5]+'%\nETA:'+str(eta_e)[:8]+'sec\nCurrent epoch will be finished at '+e_end
            self.t_send(msg=m)
            print(m)
            
#==============================================================================
# 
#============================================================================== 

def on_step_end(self, step, logs):
        """ Update progression bar at the end of each step """
        if self.info_names is None:
            self.info_names = logs['info'].keys()
        values = [('reward', logs['reward'])]
        if KERAS_VERSION > '2.1.3':
            self.progbar.update((self.step % self.interval) + 1, values=values)
        else:
            self.progbar.update((self.step % self.interval) +
                                1, values=values, force=True)
        self.step += 1
        self.metrics.append(logs['metrics'])
        if len(self.info_names) > 0:
            self.infos.append([logs['info'][k] for k in self.info_names]) 

def keras_version():
    """ Get the Keras version.

    Returns
        tuple of (major, minor, patch).
    """
    return tuple(map(int, keras.__version__.split('.'))) 

def assert_keras_version():
    """ Assert that the Keras version is up to date.
    """
    detected = keras.__version__
    required = '.'.join(map(str, minimum_keras_version))
    assert(keras_version() >= minimum_keras_version), 'You are using keras version {}. The minimum required version is {}.'.format(detected, required) 

def log_model(keras_model, artifact_path, image_dims, domain):
    """
    Log a KerasImageClassifierPyfunc model as an MLflow artifact for the current run.

    :param keras_model: Keras model to be saved.
    :param artifact_path: Run-relative artifact path this model is to be saved to.
    :param image_dims: Image dimensions the Keras model expects.
    :param domain: Labels for the classes this model can predict.
    """

    with TempDir() as tmp:
        data_path = tmp.path("image_model")
        os.mkdir(data_path)
        conf = {
            "image_dims": "/".join(map(str, image_dims)),
            "domain": "/".join(map(str, domain))
        }
        with open(os.path.join(data_path, "conf.yaml"), "w") as f:
            yaml.safe_dump(conf, stream=f)
        keras_path = os.path.join(data_path, "keras_model")
        mlflow.keras.save_model(keras_model, path=keras_path)
        conda_env = tmp.path("conda_env.yaml")
        with open(conda_env, "w") as f:
            f.write(conda_env_template.format(python_version=PYTHON_VERSION,
                                              keras_version=keras.__version__,
                                              tf_name=tf.__name__,  # can have optional -gpu suffix
                                              tf_version=tf.__version__,
                                              pillow_version=PIL.__version__))

        mlflow.pyfunc.log_model(artifact_path=artifact_path,
                                loader_module=__name__,
                                code_path=[__file__],
                                data_path=data_path,
                                conda_env=conda_env) 

def log_model(keras_model, artifact_path, image_dims, domain):
    """
    Log a KerasImageClassifierPyfunc model as an MLflow artifact for the current run.

    :param keras_model: Keras model to be saved.
    :param artifact_path: Run-relative artifact path this model is to be saved to.
    :param image_dims: Image dimensions the Keras model expects.
    :param domain: Labels for the classes this model can predict.
    """

    with TempDir() as tmp:
        data_path = tmp.path("image_model")
        os.mkdir(data_path)
        conf = {
            "image_dims": "/".join(map(str, image_dims)),
            "domain": "/".join(map(str, domain))
        }
        with open(os.path.join(data_path, "conf.yaml"), "w") as f:
            yaml.safe_dump(conf, stream=f)
        keras_path = os.path.join(data_path, "keras_model")
        mlflow.keras.save_model(keras_model, path=keras_path)
        conda_env = tmp.path("conda_env.yaml")
        with open(conda_env, "w") as f:
            f.write(conda_env_template.format(python_version=PYTHON_VERSION,
                                              keras_version=keras.__version__,
                                              tf_name=tf.__name__,  # can have optional -gpu suffix
                                              tf_version=tf.__version__,
                                              pillow_version=PIL.__version__))

        mlflow.pyfunc.log_model(artifact_path=artifact_path,
                                loader_module=__name__,
                                code_path=[__file__],
                                data_path=data_path,
                                conda_env=conda_env) 

def log_keras_version_info():
    import keras
    logger.info("Keras version: " + keras.__version__)
    from keras import backend as K
    try:
        backend = K.backend()
    except AttributeError:
        backend = K._BACKEND  # pylint: disable=protected-access
    if backend == 'theano':
        import theano
        logger.info("Theano version: " + theano.__version__)
        logger.warning("Using Keras' theano backend is not supported! Expect to crash...")
    elif backend == 'tensorflow':
        import tensorflow
        logger.info("Tensorflow version: " + tensorflow.__version__)  # pylint: disable=no-member 

def get_default_conda_env(include_cloudpickle=False, keras_module=None):
    """
    :return: The default Conda environment for MLflow Models produced by calls to
             :func:`save_model()` and :func:`log_model()`.
    """
    import tensorflow as tf
    conda_deps = []  # if we use tf.keras we only need to declare dependency on tensorflow
    pip_deps = []
    if keras_module is None:
        import keras
        keras_module = keras
    if keras_module.__name__ == "keras":
        # Temporary fix: the created conda environment has issues installing keras >= 2.3.1
        if LooseVersion(keras_module.__version__) < LooseVersion('2.3.1'):
            conda_deps.append("keras=={}".format(keras_module.__version__))
        else:
            pip_deps.append("keras=={}".format(keras_module.__version__))
    if include_cloudpickle:
        import cloudpickle
        pip_deps.append("cloudpickle=={}".format(cloudpickle.__version__))
    # Temporary fix: conda-forge currently does not have tensorflow > 1.14
    # The Keras pyfunc representation requires the TensorFlow
    # backend for Keras. Therefore, the conda environment must
    # include TensorFlow
    if LooseVersion(tf.__version__) <= LooseVersion('1.13.2'):
        conda_deps.append("tensorflow=={}".format(tf.__version__))
    else:
        pip_deps.append("tensorflow=={}".format(tf.__version__))

    return _mlflow_conda_env(
        additional_conda_deps=conda_deps,
        additional_pip_deps=pip_deps,
        additional_conda_channels=None)