Python numpy.float64() Examples

The following are code examples for showing how to use numpy.float64(). They are from open source Python projects. You can vote up the examples you like or vote down the ones you don't like.

Example 1
Project: wikilinks   Author: trovdimi   File: pickle_data.py    MIT License 7 votes vote down vote up
def pickle_transitions_matrix_data():
    transitions = pickle.load( open( "/ssd/ddimitrov/pickle/transitions", "rb" ) )
    vocab = pickle.load( open( "/ssd/ddimitrov/pickle/vocab", "rb" ) )

    i_indices = array.array(str("l"))
    j_indices = array.array(str("l"))
    values = array.array(str("d"))

    for s, targets in transitions.iteritems():
        for t, v in targets.iteritems():
            i_indices.append(vocab[s])
            j_indices.append(vocab[t])
            values.append(v)

    i_indices = np.frombuffer(i_indices, dtype=np.int_)
    j_indices = np.frombuffer(j_indices, dtype=np.int_)
    values = np.frombuffer(values, dtype=np.float64)
    transition_matrix=[i_indices,j_indices,values]
    pickle.dump(transition_matrix, open("/ssd/ddimitrov/pickle/transition_matrix", "wb"), protocol=pickle.HIGHEST_PROTOCOL)
    print "transition_matrix" 
Example 2
Project: cat-bbs   Author: aleju   File: common.py    MIT License 6 votes vote down vote up
def draw_heatmap(img, heatmap, alpha=0.5):
    """Draw a heatmap overlay over an image."""
    assert len(heatmap.shape) == 2 or \
        (len(heatmap.shape) == 3 and heatmap.shape[2] == 1)
    assert img.dtype in [np.uint8, np.int32, np.int64]
    assert heatmap.dtype in [np.float32, np.float64]

    if img.shape[0:2] != heatmap.shape[0:2]:
        heatmap_rs = np.clip(heatmap * 255, 0, 255).astype(np.uint8)
        heatmap_rs = ia.imresize_single_image(
            heatmap_rs[..., np.newaxis],
            img.shape[0:2],
            interpolation="nearest"
        )
        heatmap = np.squeeze(heatmap_rs) / 255.0

    cmap = plt.get_cmap('jet')
    heatmap_cmapped = cmap(heatmap)
    heatmap_cmapped = np.delete(heatmap_cmapped, 3, 2)
    heatmap_cmapped = heatmap_cmapped * 255
    mix = (1-alpha) * img + alpha * heatmap_cmapped
    mix = np.clip(mix, 0, 255).astype(np.uint8)
    return mix 
Example 3
Project: explirefit   Author: codogogo   File: data_shaper.py    Apache License 2.0 6 votes vote down vote up
def prep_labels_one_hot_encoding(labels, dist_labels = None, multilabel = False):
	if dist_labels is None:
		if multilabel:
			dist_labels = list(set([y for s in labels for y in s]))
		else:
			dist_labels = list(set(labels))
	y = []
	for i in range(len(labels)):
		lab_vec = [0] * len(dist_labels)
		if multilabel:
			for j in range(len(labels[i])):
				lab_vec[dist_labels.index(labels[i][j])] = 1.0
		else:
			lab_vec[dist_labels.index(labels[i])] = 1.0
		y.append(lab_vec)
	return np.array(y, dtype = np.float64), dist_labels 
Example 4
Project: aospy   Author: spencerahill   File: data_loader.py    Apache License 2.0 6 votes vote down vote up
def _maybe_cast_to_float64(da):
    """Cast DataArrays to np.float64 if they are of type np.float32.

    Parameters
    ----------
    da : xr.DataArray
        Input DataArray

    Returns
    -------
    DataArray

    """
    if da.dtype == np.float32:
        logging.warning('Datapoints were stored using the np.float32 datatype.'
                        'For accurate reduction operations using bottleneck, '
                        'datapoints are being cast to the np.float64 datatype.'
                        ' For more information see: https://github.com/pydata/'
                        'xarray/issues/1346')
        return da.astype(np.float64)
    else:
        return da 
Example 5
Project: kaldi-python-io   Author: funcwj   File: _io_kernel.py    Apache License 2.0 6 votes vote down vote up
def read_common_mat(fd):
    """ 
        Read common matrix(for class Matrix in kaldi setup)
        see matrix/kaldi-matrix.cc::
            void Matrix<Real>::Read(std::istream & is, bool binary, bool add)
        Return a numpy ndarray object
    """
    mat_type = read_token(fd)
    print_info('\tType of the common matrix: {}'.format(mat_type))
    if mat_type not in ["FM", "DM"]:
        raise RuntimeError("Unknown matrix type in kaldi: {}".format(mat_type))
    float_size = 4 if mat_type == 'FM' else 8
    float_type = np.float32 if mat_type == 'FM' else np.float64
    num_rows = read_int32(fd)
    num_cols = read_int32(fd)
    print_info('\tSize of the common matrix: {} x {}'.format(
        num_rows, num_cols))
    mat_data = fd.read(float_size * num_cols * num_rows)
    mat = np.fromstring(mat_data, dtype=float_type)
    return mat.reshape(num_rows, num_cols) 
Example 6
Project: kaldi-python-io   Author: funcwj   File: _io_kernel.py    Apache License 2.0 6 votes vote down vote up
def read_float_vec(fd, direct_access=False):
    """
        Read float vector(for class Vector in kaldi setup)
        see matrix/kaldi-vector.cc
    """
    if direct_access:
        expect_binary(fd)
    vec_type = read_token(fd)
    print_info('\tType of the common vector: {}'.format(vec_type))
    if vec_type not in ["FV", "DV"]:
        raise RuntimeError("Unknown matrix type in kaldi: {}".format(vec_type))
    float_size = 4 if vec_type == 'FV' else 8
    float_type = np.float32 if vec_type == 'FV' else np.float64
    dim = read_int32(fd)
    print_info('\tDim of the common vector: {}'.format(dim))
    vec_data = fd.read(float_size * dim)
    return np.fromstring(vec_data, dtype=float_type) 
Example 7
Project: dynamic-training-with-apache-mxnet-on-aws   Author: awslabs   File: profiler_ndarray.py    Apache License 2.0 6 votes vote down vote up
def test_ndarray_elementwise():
    np.random.seed(0)
    nrepeat = 10
    maxdim = 4
    all_type = [np.float32, np.float64, np.float16, np.uint8, np.int32]
    real_type = [np.float32, np.float64, np.float16]
    for repeat in range(nrepeat):
        for dim in range(1, maxdim):
            check_with_uniform(lambda x, y: x + y, 2, dim, type_list=all_type)
            check_with_uniform(lambda x, y: x - y, 2, dim, type_list=all_type)
            check_with_uniform(lambda x, y: x * y, 2, dim, type_list=all_type)
            check_with_uniform(lambda x, y: x / y, 2, dim, type_list=real_type)
            check_with_uniform(lambda x, y: x / y, 2, dim, rmin=1, type_list=all_type)
            check_with_uniform(mx.nd.sqrt, 1, dim, np.sqrt, rmin=0)
            check_with_uniform(mx.nd.square, 1, dim, np.square, rmin=0)
            check_with_uniform(lambda x: mx.nd.norm(x).asscalar(), 1, dim, np.linalg.norm) 
Example 8
Project: dynamic-training-with-apache-mxnet-on-aws   Author: awslabs   File: test_operator.py    Apache License 2.0 6 votes vote down vote up
def check_pad_with_shape(shape, xpu, pad_width, mode, dtype="float64"):
    # bind with label
    X = mx.symbol.Variable('X', dtype=dtype)
    Y = mx.symbol.Pad(data=X, mode=mode, pad_width=pad_width)
    x = mx.random.uniform(-1, 1, shape, ctx=mx.cpu(), dtype=dtype).copyto(xpu)
    # numpy result
    pad_grouped = list(zip(*[iter(list(pad_width))] * 2))
    np_out = np.pad(x.asnumpy(), pad_grouped, mode)
    # mxnet result
    grad = mx.nd.empty(shape, ctx = xpu, dtype=dtype)
    exec1 = Y.bind(xpu, args = [x], args_grad = {'X': grad})
    exec1.forward(is_train=True)
    out = exec1.outputs[0].asnumpy()
    # compare numpy + mxnet
    assert_almost_equal(out, np_out)
    # grad check
    check_numeric_gradient(Y, [x.asnumpy()], numeric_eps=1e-2, rtol=1e-2) 
Example 9
Project: dynamic-training-with-apache-mxnet-on-aws   Author: awslabs   File: test_operator.py    Apache License 2.0 6 votes vote down vote up
def test_psroipooling_with_type():
    arg_params = {
        'psroipool_rois': np.array([[0, 10, 22, 161, 173], [0, 20, 15, 154, 160]])}

    # plain psroipooling
    sym = mx.sym.contrib.PSROIPooling(spatial_scale=0.0625, output_dim=2, pooled_size=3, name='psroipool')
    ctx_list = [{'ctx': mx.cpu(0),
                 'psroipool_data': (1, 18, 14, 14),
                 'psroipool_rois': (2, 5),
                 'type_dict': {'psroipool_data': np.float64, 'psroipool_rois': np.float64}},
                {'ctx': mx.cpu(0),
                 'psroipool_data': (1, 18, 14, 14),
                 'psroipool_rois': (2, 5),
                 'type_dict': {'psroipool_data': np.float32, 'psroipool_rois': np.float32}},
                {'ctx': mx.cpu(0),
                 'psroipool_data': (1, 18, 14, 14),
                 'psroipool_rois': (2, 5),
                 'type_dict': {'psroipool_data': np.float16, 'psroipool_rois': np.float16}},
                ]

    check_consistency(sym, ctx_list, grad_req={'psroipool_data': 'write',
                                               'psroipool_rois': 'null'}, arg_params=arg_params) 
Example 10
Project: dynamic-training-with-apache-mxnet-on-aws   Author: awslabs   File: test_operator.py    Apache License 2.0 6 votes vote down vote up
def test_laop_4():
    # Currently disabled on GPU as syevd needs cuda8
    # and MxNet builds use cuda 7.5
    if not (default_context() == mx.cpu()):
        return

    rtol_fw = 1e-6
    atol_fw = 1e-6

    data1 = mx.symbol.Variable('data1')

    check_fw = lambda sym, location, expected, dtype :\
        check_symbolic_forward(sym, location, expected, rtol=rtol_fw,
                               atol=atol_fw, dtype=dtype)

    a_np = np.array([[1., 2.], [2., 4.]])
    u_np = np.array([[0.89442718, -0.44721359], [0.44721359, 0.89442718]])
    l_np = np.array([0., 5.])
    test_syevd = mx.sym.linalg.syevd(data1)
    # float64
    #print('float64')
    check_fw(test_syevd, [a_np], [u_np, l_np], np.float64)
    # float32
    #print('float32')
    check_fw(test_syevd, [a_np], [u_np, l_np], np.float32) 
Example 11
Project: dynamic-training-with-apache-mxnet-on-aws   Author: awslabs   File: test_operator.py    Apache License 2.0 6 votes vote down vote up
def finite_diff_binary_op(
        name, forward_mxnet_call, shape, input1_low, input1_high, input2_low,
        input2_high, rtol, atol, num_eps):
    # Finite difference tests are done in float64
    dtype = np.float64
    check_grad = lambda sym, location:\
        check_numeric_gradient(sym, location, numeric_eps=num_eps, rtol=rtol,
                               atol=atol, dtype=dtype)
    data1_np = np.random.uniform(input1_low, input1_high, shape).astype(dtype)
    data2_np = np.random.uniform(input2_low, input2_high, shape).astype(dtype)
    data1 = mx.symbol.Variable('data1', dtype=dtype)
    data2 = mx.symbol.Variable('data2', dtype=dtype)
    op_name = 'binary_op={}, dtype={}'.format(name, dtype)
    op_ex = mx.sym.broadcast_add(
        forward_mxnet_call(data1, data2), mx.sym.zeros_like(data1),
        name=op_name)
    check_grad(op_ex, [data1_np, data2_np])

# Tests for unary operators (basic mathematical functions):
# - Forward: Comparison to NumPy (several dtype)
# - Backward: Comparison to NumPy (several dtype)
# - Finite difference tests (only dtype = float64) 
Example 12
Project: dynamic-training-with-apache-mxnet-on-aws   Author: awslabs   File: test_random.py    Apache License 2.0 6 votes vote down vote up
def test_random_seed_setting():
    ctx = mx.context.current_context()
    seed_to_test = 1234
    num_temp_seeds = 25
    probs = [0.125, 0.25, 0.25, 0.0625, 0.125, 0.1875]
    num_samples = 100000
    for dtype in ['float16', 'float32', 'float64']:
        seed = set_seed_variously(1, num_temp_seeds, seed_to_test)
        samples1 = mx.nd.random.multinomial(data=mx.nd.array(probs, ctx=ctx, dtype=dtype),
                                            shape=num_samples)
        seed = set_seed_variously(seed, num_temp_seeds, seed_to_test)
        samples2 = mx.nd.random.multinomial(data=mx.nd.array(probs, ctx=ctx, dtype=dtype),
                                            shape=num_samples)
        samples1np = samples1.asnumpy()
        set_seed_variously(seed, num_temp_seeds, seed_to_test+1)
        samples2np = samples2.asnumpy()
        assert same(samples1np, samples2np), \
            "seed-setting test: `multinomial` should give the same result with the same seed"


# Tests that seed setting of parallel rng is synchronous w.r.t. rng use before and after. 
Example 13
Project: dynamic-training-with-apache-mxnet-on-aws   Author: awslabs   File: test_random.py    Apache License 2.0 6 votes vote down vote up
def test_normal_generator():
    ctx = mx.context.current_context()
    samples = 1000000
    # Default success rate is 0.25, so 2 successes of 8 trials will pass.
    trials = 8
    num_buckets = 5
    for dtype in ['float16', 'float32', 'float64']:
        for mu, sigma in [(0.0, 1.0), (1.0, 5.0)]:
            print("ctx=%s, dtype=%s, Mu=%g, Sigma=%g:" % (ctx, dtype, mu, sigma))
            buckets, probs = gen_buckets_probs_with_ppf(lambda x: ss.norm.ppf(x, mu, sigma), num_buckets)
            # Quantize bucket boundaries to reflect the actual dtype and adjust probs accordingly
            buckets = np.array(buckets, dtype=dtype).tolist()
            probs = [(ss.norm.cdf(buckets[i][1], mu, sigma) -
                      ss.norm.cdf(buckets[i][0], mu, sigma)) for i in range(num_buckets)]
            generator_mx = lambda x: mx.nd.random.normal(mu, sigma, shape=x, ctx=ctx, dtype=dtype).asnumpy()
            verify_generator(generator=generator_mx, buckets=buckets, probs=probs,
                             nsamples=samples, nrepeat=trials)
            generator_mx_same_seed =\
                lambda x: np.concatenate(
                    [mx.nd.random.normal(mu, sigma, shape=x // 10, ctx=ctx, dtype=dtype).asnumpy()
                     for _ in range(10)])
            verify_generator(generator=generator_mx_same_seed, buckets=buckets, probs=probs,
                             nsamples=samples, nrepeat=trials) 
Example 14
Project: multi-embedding-cws   Author: wangjksjtu   File: pw_lstm_crf_train.py    MIT License 5 votes vote down vote up
def test_evaluate(sess, unary_score, test_sequence_length, transMatrix, inp_char, inp_pinyin, inp_wubi, tX_char, tX_pinyin, tX_wubi, tY):
    totalEqual = 0
    batchSize = FLAGS.batch_size
    totalLen = tX_char.shape[0]
    numBatch = int((tX_char.shape[0] - 1) / batchSize) + 1
    correct_labels = 0
    total_labels = 0

    for i in range(numBatch):
        endOff = (i + 1) * batchSize
        if endOff > totalLen:
            endOff = totalLen

        y = tY[i * batchSize:endOff]
        feed_dict = {inp_char: tX_char[i * batchSize:endOff], inp_pinyin:tX_pinyin[i * batchSize:endOff], inp_wubi: tX_wubi[i * batchSize:endOff]}
        #feed_dict_pinyin = {inp_pinyin: tX_pinyin[i * batchSize:endOff]}
        #feed_dict_wubi = {inp_wubi: tX_wubi[i * batchSize:endOff]}
        unary_score_val, test_sequence_length_val = sess.run(
            [unary_score, test_sequence_length], feed_dict)

        for tf_unary_scores_, y_, sequence_length_ in zip(
                unary_score_val, y, test_sequence_length_val):

            tf_unary_scores_ = tf_unary_scores_[:sequence_length_]
            y_ = y_[:sequence_length_]

            viterbi_sequence, _ = tf.contrib.crf.viterbi_decode(
                tf_unary_scores_, transMatrix)

            # Evaluate word-level accuracy.
            correct_labels += np.sum(np.equal(viterbi_sequence, y_))
            total_labels += sequence_length_

    cl = np.float64(correct_labels)
    tl = np.float64(total_labels)
    accuracy = 100.0 * cl / tl
    print("Accuracy: %.3f%%" % accuracy)
    return accuracy 
Example 15
Project: multi-embedding-cws   Author: wangjksjtu   File: pw_lstm3_crf_train.py    MIT License 5 votes vote down vote up
def test_evaluate(sess, unary_score, test_sequence_length, transMatrix, inp_char, inp_pinyin, inp_wubi, tX_char, tX_pinyin, tX_wubi, tY):
    totalEqual = 0
    batchSize = FLAGS.batch_size
    totalLen = tX_char.shape[0]
    numBatch = int((tX_char.shape[0] - 1) / batchSize) + 1
    correct_labels = 0
    total_labels = 0

    for i in range(numBatch):
        endOff = (i + 1) * batchSize
        if endOff > totalLen:
            endOff = totalLen

        y = tY[i * batchSize:endOff]
        feed_dict = {inp_char: tX_char[i * batchSize:endOff], inp_pinyin:tX_pinyin[i * batchSize:endOff], inp_wubi: tX_wubi[i * batchSize:endOff]}
        #feed_dict_pinyin = {inp_pinyin: tX_pinyin[i * batchSize:endOff]}
        #feed_dict_wubi = {inp_wubi: tX_wubi[i * batchSize:endOff]}
        unary_score_val, test_sequence_length_val = sess.run(
            [unary_score, test_sequence_length], feed_dict)

        for tf_unary_scores_, y_, sequence_length_ in zip(
                unary_score_val, y, test_sequence_length_val):

            tf_unary_scores_ = tf_unary_scores_[:sequence_length_]
            y_ = y_[:sequence_length_]

            viterbi_sequence, _ = tf.contrib.crf.viterbi_decode(
                tf_unary_scores_, transMatrix)

            # Evaluate word-level accuracy.
            correct_labels += np.sum(np.equal(viterbi_sequence, y_))
            total_labels += sequence_length_

    cl = np.float64(correct_labels)
    tl = np.float64(total_labels)
    accuracy = 100.0 * cl / tl
    print("Accuracy: %.3f%%" % accuracy)
    return accuracy 
Example 16
Project: multi-embedding-cws   Author: wangjksjtu   File: lstm_cnn_train.py    MIT License 5 votes vote down vote up
def test_evaluate(sess, unary_score, test_sequence_length, inp, tX, tY):
    totalEqual = 0
    batchSize = FLAGS.batch_size
    totalLen = tX.shape[0]
    numBatch = int((tX.shape[0] - 1) / batchSize) + 1
    correct_labels = 0
    total_labels = 0

    for i in range(numBatch):
        endOff = (i + 1) * batchSize
        if endOff > totalLen:
            endOff = totalLen

        y = tY[i * batchSize:endOff]
        feed_dict = {inp: tX[i * batchSize:endOff]}
        unary_score_val, test_sequence_length_val = sess.run(
            [unary_score, test_sequence_length], feed_dict)

        for tf_unary_scores_, y_, sequence_length_ in zip(
                unary_score_val, y, test_sequence_length_val):

            best_sequence = tf_unary_scores_[:sequence_length_]
            y_ = y_[:sequence_length_]

            # Evaluate word-level accuracy.
            correct_labels += np.sum(np.equal(best_sequence, y_))
            total_labels += sequence_length_

    cl = np.float64(correct_labels)
    tl = np.float64(total_labels)
    accuracy = 100.0 * cl / tl

    print("Accuracy: %.3f%%" % accuracy)
    return accuracy 
Example 17
Project: multi-embedding-cws   Author: wangjksjtu   File: share_lstm3_crf_train.py    MIT License 5 votes vote down vote up
def test_evaluate(sess, unary_score, test_sequence_length, transMatrix, inp_char, inp_pinyin, inp_wubi, tX_char, tX_pinyin, tX_wubi, tY):
    totalEqual = 0
    batchSize = FLAGS.batch_size
    totalLen = tX_char.shape[0]
    numBatch = int((tX_char.shape[0] - 1) / batchSize) + 1
    correct_labels = 0
    total_labels = 0

    for i in range(numBatch):
        endOff = (i + 1) * batchSize
        if endOff > totalLen:
            endOff = totalLen

        y = tY[i * batchSize:endOff]
        feed_dict = {inp_char: tX_char[i * batchSize:endOff], inp_pinyin:tX_pinyin[i * batchSize:endOff], inp_wubi: tX_wubi[i * batchSize:endOff]}
        #feed_dict_pinyin = {inp_pinyin: tX_pinyin[i * batchSize:endOff]}
        #feed_dict_wubi = {inp_wubi: tX_wubi[i * batchSize:endOff]}
        unary_score_val, test_sequence_length_val = sess.run(
            [unary_score, test_sequence_length], feed_dict)

        for tf_unary_scores_, y_, sequence_length_ in zip(
                unary_score_val, y, test_sequence_length_val):

            tf_unary_scores_ = tf_unary_scores_[:sequence_length_]
            y_ = y_[:sequence_length_]

            viterbi_sequence, _ = tf.contrib.crf.viterbi_decode(
                tf_unary_scores_, transMatrix)

            # Evaluate word-level accuracy.
            correct_labels += np.sum(np.equal(viterbi_sequence, y_))
            total_labels += sequence_length_

    cl = np.float64(correct_labels)
    tl = np.float64(total_labels)
    accuracy = 100.0 * cl / tl
    print("Accuracy: %.3f%%" % accuracy)
    return accuracy 
Example 18
Project: multi-embedding-cws   Author: wangjksjtu   File: lstm4_crf_train.py    MIT License 5 votes vote down vote up
def test_evaluate(sess, unary_score, test_sequence_length, transMatrix, inp, tX, tY):
    totalEqual = 0
    batchSize = FLAGS.batch_size
    totalLen = tX.shape[0]
    numBatch = int((tX.shape[0] - 1) / batchSize) + 1
    correct_labels = 0
    total_labels = 0

    for i in range(numBatch):
        endOff = (i + 1) * batchSize
        if endOff > totalLen:
            endOff = totalLen

        y = tY[i * batchSize:endOff]
        feed_dict = {inp: tX[i * batchSize:endOff]}
        unary_score_val, test_sequence_length_val = sess.run(
            [unary_score, test_sequence_length], feed_dict)

        for tf_unary_scores_, y_, sequence_length_ in zip(
                unary_score_val, y, test_sequence_length_val):

            tf_unary_scores_ = tf_unary_scores_[:sequence_length_]
            y_ = y_[:sequence_length_]

            viterbi_sequence, _ = tf.contrib.crf.viterbi_decode(
                tf_unary_scores_, transMatrix)

            # Evaluate word-level accuracy.
            correct_labels += np.sum(np.equal(viterbi_sequence, y_))
            total_labels += sequence_length_

    cl = np.float64(correct_labels)
    tl = np.float64(total_labels)
    accuracy = 100.0 * cl / tl
    print("Accuracy: %.3f%%" % accuracy)
    return accuracy 
Example 19
Project: multi-embedding-cws   Author: wangjksjtu   File: share_lstm3_crf_time_paper.py    MIT License 5 votes vote down vote up
def test_evaluate(sess, unary_score, test_sequence_length, transMatrix, inp_char, inp_pinyin, inp_wubi, tX_char, tX_pinyin, tX_wubi, tY):
    totalEqual = 0
    batchSize = FLAGS.batch_size
    totalLen = tX_char.shape[0]
    numBatch = int((tX_char.shape[0] - 1) / batchSize) + 1
    correct_labels = 0
    total_labels = 0

    for i in range(numBatch):
        endOff = (i + 1) * batchSize
        if endOff > totalLen:
            endOff = totalLen

        y = tY[i * batchSize:endOff]
        feed_dict = {inp_char: tX_char[i * batchSize:endOff], inp_pinyin:tX_pinyin[i * batchSize:endOff], inp_wubi: tX_wubi[i * batchSize:endOff]}
        #feed_dict_pinyin = {inp_pinyin: tX_pinyin[i * batchSize:endOff]}
        #feed_dict_wubi = {inp_wubi: tX_wubi[i * batchSize:endOff]}
        unary_score_val, test_sequence_length_val = sess.run(
            [unary_score, test_sequence_length], feed_dict)

        for tf_unary_scores_, y_, sequence_length_ in zip(
                unary_score_val, y, test_sequence_length_val):

            tf_unary_scores_ = tf_unary_scores_[:sequence_length_]
            y_ = y_[:sequence_length_]

            viterbi_sequence, _ = tf.contrib.crf.viterbi_decode(
                tf_unary_scores_, transMatrix)

            # Evaluate word-level accuracy.
            correct_labels += np.sum(np.equal(viterbi_sequence, y_))
            total_labels += sequence_length_

    cl = np.float64(correct_labels)
    tl = np.float64(total_labels)
    accuracy = 100.0 * cl / tl
    print("Accuracy: %.3f%%" % accuracy)
    return accuracy 
Example 20
Project: multi-embedding-cws   Author: wangjksjtu   File: lstm_crf_train.py    MIT License 5 votes vote down vote up
def test_evaluate(sess, unary_score, test_sequence_length, transMatrix, inp, tX, tY):
    totalEqual = 0
    batchSize = FLAGS.batch_size
    totalLen = tX.shape[0]
    numBatch = int((tX.shape[0] - 1) / batchSize) + 1
    correct_labels = 0
    total_labels = 0

    for i in range(numBatch):
        endOff = (i + 1) * batchSize
        if endOff > totalLen:
            endOff = totalLen

        y = tY[i * batchSize:endOff]
        feed_dict = {inp: tX[i * batchSize:endOff]}
        unary_score_val, test_sequence_length_val = sess.run(
            [unary_score, test_sequence_length], feed_dict)

        for tf_unary_scores_, y_, sequence_length_ in zip(
                unary_score_val, y, test_sequence_length_val):

            tf_unary_scores_ = tf_unary_scores_[:sequence_length_]
            y_ = y_[:sequence_length_]

            viterbi_sequence, _ = tf.contrib.crf.viterbi_decode(
                tf_unary_scores_, transMatrix)

            # Evaluate word-level accuracy.
            correct_labels += np.sum(np.equal(viterbi_sequence, y_))
            total_labels += sequence_length_

    cl = np.float64(correct_labels)
    tl = np.float64(total_labels)
    accuracy = 100.0 * cl / tl
    print("Accuracy: %.3f%%" % accuracy)
    return accuracy 
Example 21
Project: multi-embedding-cws   Author: wangjksjtu   File: nopy_fc_lstm3_crf_train.py    MIT License 5 votes vote down vote up
def test_evaluate(sess, unary_score, test_sequence_length, transMatrix, inp_char, inp_wubi,tX_char, tX_wubi, tY):
    totalEqual = 0
    batchSize = FLAGS.batch_size
    totalLen = tX_char.shape[0]
    numBatch = int((tX_char.shape[0] - 1) / batchSize) + 1
    correct_labels = 0
    total_labels = 0

    for i in range(numBatch):
        endOff = (i + 1) * batchSize
        if endOff > totalLen:
            endOff = totalLen

        y = tY[i * batchSize:endOff]
        feed_dict = {inp_char: tX_char[i * batchSize:endOff], inp_wubi: tX_wubi[i * batchSize:endOff]}
        unary_score_val, test_sequence_length_val = sess.run(
            [unary_score, test_sequence_length], feed_dict)

        for tf_unary_scores_, y_, sequence_length_ in zip(
                unary_score_val, y, test_sequence_length_val):

            tf_unary_scores_ = tf_unary_scores_[:sequence_length_]
            y_ = y_[:sequence_length_]

            viterbi_sequence, _ = tf.contrib.crf.viterbi_decode(
                tf_unary_scores_, transMatrix)

            # Evaluate word-level accuracy.
            correct_labels += np.sum(np.equal(viterbi_sequence, y_))
            total_labels += sequence_length_

    cl = np.float64(correct_labels)
    tl = np.float64(total_labels)
    accuracy = 100.0 * cl / tl
    print("Accuracy: %.3f%%" % accuracy)
    return accuracy 
Example 22
Project: multi-embedding-cws   Author: wangjksjtu   File: share_lstm_crf_train_paper.py    MIT License 5 votes vote down vote up
def test_evaluate(sess, unary_score, test_sequence_length, transMatrix, inp_char, inp_pinyin, inp_wubi, tX_char, tX_pinyin, tX_wubi, tY):
    totalEqual = 0
    batchSize = FLAGS.batch_size
    totalLen = tX_char.shape[0]
    numBatch = int((tX_char.shape[0] - 1) / batchSize) + 1
    correct_labels = 0
    total_labels = 0

    for i in range(numBatch):
        endOff = (i + 1) * batchSize
        if endOff > totalLen:
            endOff = totalLen

        y = tY[i * batchSize:endOff]
        feed_dict = {inp_char: tX_char[i * batchSize:endOff], inp_pinyin:tX_pinyin[i * batchSize:endOff], inp_wubi: tX_wubi[i * batchSize:endOff]}
        #feed_dict_pinyin = {inp_pinyin: tX_pinyin[i * batchSize:endOff]}
        #feed_dict_wubi = {inp_wubi: tX_wubi[i * batchSize:endOff]}
        unary_score_val, test_sequence_length_val = sess.run(
            [unary_score, test_sequence_length], feed_dict)

        for tf_unary_scores_, y_, sequence_length_ in zip(
                unary_score_val, y, test_sequence_length_val):

            tf_unary_scores_ = tf_unary_scores_[:sequence_length_]
            y_ = y_[:sequence_length_]

            viterbi_sequence, _ = tf.contrib.crf.viterbi_decode(
                tf_unary_scores_, transMatrix)

            # Evaluate word-level accuracy.
            correct_labels += np.sum(np.equal(viterbi_sequence, y_))
            total_labels += sequence_length_

    cl = np.float64(correct_labels)
    tl = np.float64(total_labels)
    accuracy = 100.0 * cl / tl
    print("Accuracy: %.3f%%" % accuracy)
    return accuracy 
Example 23
Project: multi-embedding-cws   Author: wangjksjtu   File: share_lstm_crf_train.py    MIT License 5 votes vote down vote up
def test_evaluate(sess, unary_score, test_sequence_length, transMatrix, inp_char, inp_pinyin, inp_wubi, tX_char, tX_pinyin, tX_wubi, tY):
    totalEqual = 0
    batchSize = FLAGS.batch_size
    totalLen = tX_char.shape[0]
    numBatch = int((tX_char.shape[0] - 1) / batchSize) + 1
    correct_labels = 0
    total_labels = 0

    for i in range(numBatch):
        endOff = (i + 1) * batchSize
        if endOff > totalLen:
            endOff = totalLen

        y = tY[i * batchSize:endOff]
        feed_dict = {inp_char: tX_char[i * batchSize:endOff], inp_pinyin:tX_pinyin[i * batchSize:endOff], inp_wubi: tX_wubi[i * batchSize:endOff]}
        #feed_dict_pinyin = {inp_pinyin: tX_pinyin[i * batchSize:endOff]}
        #feed_dict_wubi = {inp_wubi: tX_wubi[i * batchSize:endOff]}
        unary_score_val, test_sequence_length_val = sess.run(
            [unary_score, test_sequence_length], feed_dict)

        for tf_unary_scores_, y_, sequence_length_ in zip(
                unary_score_val, y, test_sequence_length_val):

            tf_unary_scores_ = tf_unary_scores_[:sequence_length_]
            y_ = y_[:sequence_length_]

            viterbi_sequence, _ = tf.contrib.crf.viterbi_decode(
                tf_unary_scores_, transMatrix)

            # Evaluate word-level accuracy.
            correct_labels += np.sum(np.equal(viterbi_sequence, y_))
            total_labels += sequence_length_

    cl = np.float64(correct_labels)
    tl = np.float64(total_labels)
    accuracy = 100.0 * cl / tl
    print("Accuracy: %.3f%%" % accuracy)
    return accuracy 
Example 24
Project: multi-embedding-cws   Author: wangjksjtu   File: share_lstm3_crf_train_paper.py    MIT License 5 votes vote down vote up
def test_evaluate(sess, unary_score, test_sequence_length, transMatrix, inp_char, inp_pinyin, inp_wubi, tX_char, tX_pinyin, tX_wubi, tY):
    totalEqual = 0
    batchSize = FLAGS.batch_size
    totalLen = tX_char.shape[0]
    numBatch = int((tX_char.shape[0] - 1) / batchSize) + 1
    correct_labels = 0
    total_labels = 0

    for i in range(numBatch):
        endOff = (i + 1) * batchSize
        if endOff > totalLen:
            endOff = totalLen

        y = tY[i * batchSize:endOff]
        feed_dict = {inp_char: tX_char[i * batchSize:endOff], inp_pinyin:tX_pinyin[i * batchSize:endOff], inp_wubi: tX_wubi[i * batchSize:endOff]}
        #feed_dict_pinyin = {inp_pinyin: tX_pinyin[i * batchSize:endOff]}
        #feed_dict_wubi = {inp_wubi: tX_wubi[i * batchSize:endOff]}
        unary_score_val, test_sequence_length_val = sess.run(
            [unary_score, test_sequence_length], feed_dict)

        for tf_unary_scores_, y_, sequence_length_ in zip(
                unary_score_val, y, test_sequence_length_val):

            tf_unary_scores_ = tf_unary_scores_[:sequence_length_]
            y_ = y_[:sequence_length_]

            viterbi_sequence, _ = tf.contrib.crf.viterbi_decode(
                tf_unary_scores_, transMatrix)

            # Evaluate word-level accuracy.
            correct_labels += np.sum(np.equal(viterbi_sequence, y_))
            total_labels += sequence_length_

    cl = np.float64(correct_labels)
    tl = np.float64(total_labels)
    accuracy = 100.0 * cl / tl
    print("Accuracy: %.3f%%" % accuracy)
    return accuracy 
Example 25
Project: multi-embedding-cws   Author: wangjksjtu   File: fc_lstm3_crf_train.py    MIT License 5 votes vote down vote up
def test_evaluate(sess, unary_score, test_sequence_length, transMatrix, inp_char, inp_pinyin, inp_wubi,tX_char, tX_pinyin, tX_wubi, tY):
    totalEqual = 0
    batchSize = FLAGS.batch_size
    totalLen = tX_char.shape[0]
    numBatch = int((tX_char.shape[0] - 1) / batchSize) + 1
    correct_labels = 0
    total_labels = 0

    for i in range(numBatch):
        endOff = (i + 1) * batchSize
        if endOff > totalLen:
            endOff = totalLen

        y = tY[i * batchSize:endOff]
        feed_dict = {inp_char: tX_char[i * batchSize:endOff], inp_pinyin:tX_pinyin[i * batchSize:endOff], inp_wubi: tX_wubi[i * batchSize:endOff]}
        unary_score_val, test_sequence_length_val = sess.run(
            [unary_score, test_sequence_length], feed_dict)

        for tf_unary_scores_, y_, sequence_length_ in zip(
                unary_score_val, y, test_sequence_length_val):

            tf_unary_scores_ = tf_unary_scores_[:sequence_length_]
            y_ = y_[:sequence_length_]

            viterbi_sequence, _ = tf.contrib.crf.viterbi_decode(
                tf_unary_scores_, transMatrix)

            # Evaluate word-level accuracy.
            correct_labels += np.sum(np.equal(viterbi_sequence, y_))
            total_labels += sequence_length_

    cl = np.float64(correct_labels)
    tl = np.float64(total_labels)
    accuracy = 100.0 * cl / tl
    print("Accuracy: %.3f%%" % accuracy)
    return accuracy 
Example 26
Project: multi-embedding-cws   Author: wangjksjtu   File: lstm3_crf_train.py    MIT License 5 votes vote down vote up
def test_evaluate(sess, unary_score, test_sequence_length, transMatrix, inp, tX, tY):
    totalEqual = 0
    batchSize = FLAGS.batch_size
    totalLen = tX.shape[0]
    numBatch = int((tX.shape[0] - 1) / batchSize) + 1
    correct_labels = 0
    total_labels = 0

    for i in range(numBatch):
        endOff = (i + 1) * batchSize
        if endOff > totalLen:
            endOff = totalLen

        y = tY[i * batchSize:endOff]
        feed_dict = {inp: tX[i * batchSize:endOff]}
        unary_score_val, test_sequence_length_val = sess.run(
            [unary_score, test_sequence_length], feed_dict)

        for tf_unary_scores_, y_, sequence_length_ in zip(
                unary_score_val, y, test_sequence_length_val):

            tf_unary_scores_ = tf_unary_scores_[:sequence_length_]
            y_ = y_[:sequence_length_]

            viterbi_sequence, _ = tf.contrib.crf.viterbi_decode(
                tf_unary_scores_, transMatrix)

            # Evaluate word-level accuracy.
            correct_labels += np.sum(np.equal(viterbi_sequence, y_))
            total_labels += sequence_length_

    cl = np.float64(correct_labels)
    tl = np.float64(total_labels)
    accuracy = 100.0 * cl / tl
    print("Accuracy: %.3f%%" % accuracy)
    return accuracy 
Example 27
Project: multi-embedding-cws   Author: wangjksjtu   File: fc_lstm_crf_train.py    MIT License 5 votes vote down vote up
def test_evaluate(sess, unary_score, test_sequence_length, transMatrix, inp_char, inp_pinyin, inp_wubi, tX_char, tX_pinyin, tX_wubi, tY):
    totalEqual = 0
    batchSize = FLAGS.batch_size
    totalLen = tX_char.shape[0]
    numBatch = int((tX_char.shape[0] - 1) / batchSize) + 1
    correct_labels = 0
    total_labels = 0

    for i in range(numBatch):
        endOff = (i + 1) * batchSize
        if endOff > totalLen:
            endOff = totalLen

        y = tY[i * batchSize:endOff]
        feed_dict = {inp_char: tX_char[i * batchSize:endOff], inp_pinyin:tX_pinyin[i * batchSize:endOff], inp_wubi: tX_wubi[i * batchSize:endOff]}
        #feed_dict_pinyin = {inp_pinyin: tX_pinyin[i * batchSize:endOff]}
        #feed_dict_wubi = {inp_wubi: tX_wubi[i * batchSize:endOff]}
        unary_score_val, test_sequence_length_val = sess.run(
            [unary_score, test_sequence_length], feed_dict)

        for tf_unary_scores_, y_, sequence_length_ in zip(
                unary_score_val, y, test_sequence_length_val):

            tf_unary_scores_ = tf_unary_scores_[:sequence_length_]
            y_ = y_[:sequence_length_]

            viterbi_sequence, _ = tf.contrib.crf.viterbi_decode(
                tf_unary_scores_, transMatrix)

            # Evaluate word-level accuracy.
            correct_labels += np.sum(np.equal(viterbi_sequence, y_))
            total_labels += sequence_length_

    cl = np.float64(correct_labels)
    tl = np.float64(total_labels)
    accuracy = 100.0 * cl / tl
    print("Accuracy: %.3f%%" % accuracy)
    return accuracy 
Example 28
Project: multi-embedding-cws   Author: wangjksjtu   File: pw_lstm3_crf_time.py    MIT License 5 votes vote down vote up
def test_evaluate(sess, unary_score, test_sequence_length, transMatrix, inp_char, inp_pinyin, inp_wubi, tX_char, tX_pinyin, tX_wubi, tY):
    totalEqual = 0
    batchSize = FLAGS.batch_size
    totalLen = tX_char.shape[0]
    numBatch = int((tX_char.shape[0] - 1) / batchSize) + 1
    correct_labels = 0
    total_labels = 0

    for i in range(numBatch):
        endOff = (i + 1) * batchSize
        if endOff > totalLen:
            endOff = totalLen

        y = tY[i * batchSize:endOff]
        feed_dict = {inp_char: tX_char[i * batchSize:endOff], inp_pinyin:tX_pinyin[i * batchSize:endOff], inp_wubi: tX_wubi[i * batchSize:endOff]}
        #feed_dict_pinyin = {inp_pinyin: tX_pinyin[i * batchSize:endOff]}
        #feed_dict_wubi = {inp_wubi: tX_wubi[i * batchSize:endOff]}
        unary_score_val, test_sequence_length_val = sess.run(
            [unary_score, test_sequence_length], feed_dict)

        for tf_unary_scores_, y_, sequence_length_ in zip(
                unary_score_val, y, test_sequence_length_val):

            tf_unary_scores_ = tf_unary_scores_[:sequence_length_]
            y_ = y_[:sequence_length_]

            viterbi_sequence, _ = tf.contrib.crf.viterbi_decode(
                tf_unary_scores_, transMatrix)

            # Evaluate word-level accuracy.
            correct_labels += np.sum(np.equal(viterbi_sequence, y_))
            total_labels += sequence_length_

    cl = np.float64(correct_labels)
    tl = np.float64(total_labels)
    accuracy = 100.0 * cl / tl
    print("Accuracy: %.3f%%" % accuracy)
    return accuracy 
Example 29
Project: multi-embedding-cws   Author: wangjksjtu   File: nopy_share_lstm3_crf_train.py    MIT License 5 votes vote down vote up
def test_evaluate(sess, unary_score, test_sequence_length, transMatrix, inp_char, inp_wubi, tX_char, tX_wubi, tY):
    totalEqual = 0
    batchSize = FLAGS.batch_size
    totalLen = tX_char.shape[0]
    numBatch = int((tX_char.shape[0] - 1) / batchSize) + 1
    correct_labels = 0
    total_labels = 0

    for i in range(numBatch):
        endOff = (i + 1) * batchSize
        if endOff > totalLen:
            endOff = totalLen

        y = tY[i * batchSize:endOff]
        feed_dict = {inp_char: tX_char[i * batchSize:endOff], inp_wubi: tX_wubi[i * batchSize:endOff]}
        #feed_dict_pinyin = {inp_pinyin: tX_pinyin[i * batchSize:endOff]}
        #feed_dict_wubi = {inp_wubi: tX_wubi[i * batchSize:endOff]}
        unary_score_val, test_sequence_length_val = sess.run(
            [unary_score, test_sequence_length], feed_dict)

        for tf_unary_scores_, y_, sequence_length_ in zip(
                unary_score_val, y, test_sequence_length_val):

            tf_unary_scores_ = tf_unary_scores_[:sequence_length_]
            y_ = y_[:sequence_length_]

            viterbi_sequence, _ = tf.contrib.crf.viterbi_decode(
                tf_unary_scores_, transMatrix)

            # Evaluate word-level accuracy.
            correct_labels += np.sum(np.equal(viterbi_sequence, y_))
            total_labels += sequence_length_

    cl = np.float64(correct_labels)
    tl = np.float64(total_labels)
    accuracy = 100.0 * cl / tl
    print("Accuracy: %.3f%%" % accuracy)
    return accuracy 
Example 30
Project: multi-embedding-cws   Author: wangjksjtu   File: fc_lstm3_crf_time.py    MIT License 5 votes vote down vote up
def test_evaluate(sess, unary_score, test_sequence_length, transMatrix, inp_char, inp_pinyin, inp_wubi,tX_char, tX_pinyin, tX_wubi, tY):
    totalEqual = 0
    batchSize = FLAGS.batch_size
    totalLen = tX_char.shape[0]
    numBatch = int((tX_char.shape[0] - 1) / batchSize) + 1
    correct_labels = 0
    total_labels = 0

    for i in range(numBatch):
        endOff = (i + 1) * batchSize
        if endOff > totalLen:
            endOff = totalLen

        y = tY[i * batchSize:endOff]
        feed_dict = {inp_char: tX_char[i * batchSize:endOff], inp_pinyin:tX_pinyin[i * batchSize:endOff], inp_wubi: tX_wubi[i * batchSize:endOff]}
        unary_score_val, test_sequence_length_val = sess.run(
            [unary_score, test_sequence_length], feed_dict)

        for tf_unary_scores_, y_, sequence_length_ in zip(
                unary_score_val, y, test_sequence_length_val):

            tf_unary_scores_ = tf_unary_scores_[:sequence_length_]
            y_ = y_[:sequence_length_]

            viterbi_sequence, _ = tf.contrib.crf.viterbi_decode(
                tf_unary_scores_, transMatrix)

            # Evaluate word-level accuracy.
            correct_labels += np.sum(np.equal(viterbi_sequence, y_))
            total_labels += sequence_length_

    cl = np.float64(correct_labels)
    tl = np.float64(total_labels)
    accuracy = 100.0 * cl / tl
    print("Accuracy: %.3f%%" % accuracy)
    return accuracy 
Example 31
Project: aospy   Author: spencerahill   File: data_loader.py    Apache License 2.0 5 votes vote down vote up
def _sel_var(ds, var, upcast_float32=True):
    """Select the specified variable by trying all possible alternative names.

    Parameters
    ----------
    ds : Dataset
        Dataset possibly containing var
    var : aospy.Var
        Variable to find data for
    upcast_float32 : bool (default True)
        Whether to cast a float32 DataArray up to float64

    Returns
    -------
    DataArray

    Raises
    ------
    KeyError
        If the variable is not in the Dataset

    """
    for name in var.names:
        try:
            da = ds[name].rename(var.name)
            if upcast_float32:
                return _maybe_cast_to_float64(da)
            else:
                return da
        except KeyError:
            pass
    msg = '{0} not found among names: {1} in\n{2}'.format(var, var.names, ds)
    raise LookupError(msg) 
Example 32
Project: aospy   Author: spencerahill   File: test_utils_vertcoord.py    Apache License 2.0 5 votes vote down vote up
def setUp(self):
        self.p_in_hpa = np.array([1000, 925, 850, 775, 700, 600, 500, 400, 300,
                                  200, 150, 100, 70, 50, 30, 20, 10],
                                 dtype=np.float64)
        self.p_in_pa = self.p_in_hpa*1e2
        self.p_top = 0
        self.p_bot = 1.1e5
        self.p_edges = 0.5*(self.p_in_pa[1:] + 0.5*self.p_in_pa[:-1])
        self.phalf = np.concatenate(([self.p_bot], self.p_edges, [self.p_top])) 
Example 33
Project: aospy   Author: spencerahill   File: test_data_loader.py    Apache License 2.0 5 votes vote down vote up
def test_load_variable_float32_to_float64(load_variable_data_loader,
                                          start_date, end_date):
    def preprocess(ds, **kwargs):
        # This function converts testing data to the float32 datatype
        return ds.astype(np.float32)
    load_variable_data_loader.upcast_float32 = True
    load_variable_data_loader.preprocess_func = preprocess
    result = load_variable_data_loader.load_variable(
        condensation_rain, start_date,
        end_date,
        intvl_in='monthly').dtype
    expected = np.float64
    assert result == expected 
Example 34
Project: prediction-constrained-topic-models   Author: dtak   File: calc_coherence_metrics.py    MIT License 5 votes vote down vote up
def calc_pairwise_cooccurance_counts(
        x_csr_DV=None,
        dataset=None,
        ):
    """ Calculate word cooccurances across a corpus of D documents

    Returns
    -------
    ndocs_V : 1D array, size V
        entry v counts the number of documents that contain v at least once
    ndocs_csc_VV : 2D csc sparse matrix, V x V
        entry v,w counts the number of documents which contain
        the word pair (v, w) at least once

    Examples
    --------
    >>> x_DV = np.arange(6)[:,np.newaxis] * np.hstack([np.eye(6), np.zeros((6, 3))])
    >>> x_DV[:3, :3] += 1
    >>> x_DV[4, 5] += 17
    >>> ndocs_V, ndocs_csc_VV = calc_pairwise_cooccurance_counts(x_csr_DV=x_DV)
    >>> ndocs_V.astype(np.int32).tolist()
    [3, 3, 3, 1, 1, 2, 0, 0, 0]
    >>> ndocs_csc_VV.toarray()[:3, :3]
    array([[ 3.,  3.,  3.],
           [ 3.,  3.,  3.],
           [ 3.,  3.,  3.]])
    """
    if x_csr_DV is None:
        x_csr_DV = dataset['x_csr_DV']
    x_csr_DV = scipy.sparse.csr_matrix(x_csr_DV, dtype=np.float64)

    binx_csr_DV = x_csr_DV.copy()
    binx_csr_DV.data[:] = 1.0

    ndocs_V = np.squeeze(np.asarray(binx_csr_DV.sum(axis=0)))

    ndocs_csc_VV = (binx_csr_DV.T * binx_csr_DV).tocsc()
    return ndocs_V, ndocs_csc_VV 
Example 35
Project: prediction-constrained-topic-models   Author: dtak   File: util_io_csr.py    MIT License 5 votes vote down vote up
def process_ldac_line_into_preallocated_arrays(line, word_id, word_ct, start):
    """

    Returns
    -------

    Examples
    --------
    >>> word_id = np.zeros(5, dtype=np.int32)
    >>> word_ct = np.zeros(5, dtype=np.float64)
    >>> a = process_ldac_line_into_preallocated_arrays(
    ...    '5 66:6 77:7 88:8',
    ...    word_id, word_ct, 0)
    >>> a
    5
    >>> word_id.tolist()
    [66, 77, 88, 0, 0]
    >>> word_ct.tolist()
    [ 6.,7., 8., 0., 0.]
    """
    line = line.replace(':', ' ')
    data = np.fromstring(line, sep=' ', dtype=np.int32)
    stop = start + (len(data) - 1) // 2
    if stop >= word_id.size:
        raise IndexError("Provided array not large enough")    
    word_id[start:stop] = data[1::2]
    word_ct[start:stop] = data[2::2]
    return data[0] 
Example 36
Project: prediction-constrained-topic-models   Author: dtak   File: util_array.py    MIT License 5 votes vote down vote up
def toCArray(X, dtype=np.float64):
    """ Convert input into numpy array of C-contiguous order.

    Ensures returned array is aligned and owns its own data,
    not a view of another array.

    Returns
    -------
    X : ND array

    Examples
    -------
    >>> Q = np.zeros(10, dtype=np.int32, order='F')
    >>> toCArray(Q).flags.c_contiguous
    True
    >>> toCArray(Q).dtype.byteorder
    '='
    """
    X = np.asarray_chkfinite(X, dtype=dtype, order='C')
    if X.dtype.byteorder != '=':
        X = X.newbyteorder('=').copy()
    if not X.flags.owndata or X.flags.aligned:
        X = X.copy()
    assert X.flags.owndata
    assert X.flags.aligned
    return X 
Example 37
Project: CIM2Matpower   Author: kkgerasimov   File: PostProcess_mpc.py    MIT License 5 votes vote down vote up
def _make_bus_list(case):
    """Creates a list ob bus arrays for PYPOWER."""
    # Needs *case* to create/update the mapping bus: bus_id mapping
    # PYPOWER bus description has 13 (or +4) entries
    bus_list = np.zeros((len(case.buses), 13), dtype=np.float64)

    case.bus_ids = {}

    for i, bus in enumerate(case.buses):
        case.bus_ids[bus] = bus.id
        _fill_bus_array(bus_list[i], bus, bus.id)

    return bus_list 
Example 38
Project: CIM2Matpower   Author: kkgerasimov   File: PostProcess_mpc.py    MIT License 5 votes vote down vote up
def _make_gen_list(generators, bus_ids):
    """Creates a list of generators for PYPOWER."""
    # PYPOWER gen description has 21 (+4) entries
    gen_list = np.zeros((len(generators), 21), dtype=np.float64)

    for i, gen in enumerate(generators):
        _fill_gen_array(gen_list[i], gen, bus_ids)

    return gen_list 
Example 39
Project: CIM2Matpower   Author: kkgerasimov   File: PostProcess_mpc.py    MIT License 5 votes vote down vote up
def _make_branch_list(branches, bus_ids):
    """Creates a list of branches for PYPOWER."""
    # PYPOWER branch description has 17 (+4) entries
    branch_list = np.zeros((len(branches), 17), dtype=np.float64)

    for i, branch in enumerate(branches):
        _fill_branch_array(branch_list[i], branch, bus_ids)

    return branch_list 
Example 40
Project: DataHack2018   Author: InnovizTech   File: vis_utils.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def assign_points(self, point_cloud, color):
        assert color.shape[1] == 4
        assert color.shape[0] == point_cloud.shape[0]
        load_point_cloud(self._pos_writer, self._color_writer, point_cloud.astype(np.float64), color.astype(np.float64),
                         self._point_cloud_size, self._prev_point_cloud_size)
        self._prev_point_cloud_size = point_cloud.shape[0] 
Example 41
Project: kaldi-python-io   Author: funcwj   File: _io_kernel.py    Apache License 2.0 5 votes vote down vote up
def write_common_mat(fd, mat):
    """
        Write a common matrix
    """
    if mat.dtype not in [np.float32, np.float64]:
        raise RuntimeError("Unsupported numpy dtype: {}".format(mat.dtype))
    mat_type = 'FM' if mat.dtype == np.float32 else 'DM'
    write_token(fd, mat_type)
    num_rows, num_cols = mat.shape
    write_int32(fd, num_rows)
    write_int32(fd, num_cols)
    fd.write(mat.tobytes()) 
Example 42
Project: kaldi-python-io   Author: funcwj   File: _io_kernel.py    Apache License 2.0 5 votes vote down vote up
def write_float_vec(fd, vec):
    """
        Write a float vector
    """
    if vec.dtype not in [np.float32, np.float64]:
        raise RuntimeError("Unsupported numpy dtype: {}".format(vec.dtype))
    vec_type = 'FV' if vec.dtype == np.float32 else 'DV'
    write_token(fd, vec_type)
    if vec.ndim != 1:
        raise RuntimeError("write_float_vec accept 1D-vector only")
    dim = vec.size
    write_int32(fd, dim)
    fd.write(vec.tobytes()) 
Example 43
Project: mmdetection   Author: open-mmlab   File: test_nms.py    Apache License 2.0 5 votes vote down vote up
def test_nms_device_and_dtypes_cpu():
    """
    CommandLine:
        xdoctest -m tests/test_nms.py test_nms_device_and_dtypes_cpu
    """
    iou_thr = 0.7
    base_dets = np.array([[49.1, 32.4, 51.0, 35.9, 0.9],
                          [49.3, 32.9, 51.0, 35.3, 0.9],
                          [35.3, 11.5, 39.9, 14.5, 0.4],
                          [35.2, 11.7, 39.7, 15.7, 0.3]])

    # CPU can handle float32 and float64
    dets = base_dets.astype(np.float32)
    supressed, inds = nms(dets, iou_thr)
    assert dets.dtype == supressed.dtype
    assert len(inds) == len(supressed) == 3

    dets = torch.FloatTensor(base_dets)
    surpressed, inds = nms(dets, iou_thr)
    assert dets.dtype == surpressed.dtype
    assert len(inds) == len(surpressed) == 3

    dets = base_dets.astype(np.float64)
    supressed, inds = nms(dets, iou_thr)
    assert dets.dtype == supressed.dtype
    assert len(inds) == len(supressed) == 3

    dets = torch.DoubleTensor(base_dets)
    surpressed, inds = nms(dets, iou_thr)
    assert dets.dtype == surpressed.dtype
    assert len(inds) == len(surpressed) == 3 
Example 44
Project: mmdetection   Author: open-mmlab   File: test_nms.py    Apache License 2.0 5 votes vote down vote up
def test_nms_device_and_dtypes_gpu():
    """
    CommandLine:
        xdoctest -m tests/test_nms.py test_nms_device_and_dtypes_gpu
    """
    if not torch.cuda.is_available():
        import pytest
        pytest.skip('test requires GPU and torch+cuda')

    iou_thr = 0.7
    base_dets = np.array([[49.1, 32.4, 51.0, 35.9, 0.9],
                          [49.3, 32.9, 51.0, 35.3, 0.9],
                          [35.3, 11.5, 39.9, 14.5, 0.4],
                          [35.2, 11.7, 39.7, 15.7, 0.3]])

    for device_id in range(torch.cuda.device_count()):
        print('Run NMS on device_id = {!r}'.format(device_id))
        # GPU can handle float32 but not float64
        dets = base_dets.astype(np.float32)
        supressed, inds = nms(dets, iou_thr, device_id)
        assert dets.dtype == supressed.dtype
        assert len(inds) == len(supressed) == 3

        dets = torch.FloatTensor(base_dets).to(device_id)
        surpressed, inds = nms(dets, iou_thr)
        assert dets.dtype == surpressed.dtype
        assert len(inds) == len(surpressed) == 3 
Example 45
Project: deep-learning-note   Author: wdxtub   File: 1_linear_basic.py    MIT License 5 votes vote down vote up
def evaluateModel(model, testData, features, labels):
    """
    计算线性模型的均方差和决定系数

    参数
    ----
    model : LinearRegression, 训练完成的线性模型

    testData : DataFrame,测试数据

    features : list[str],特征名列表

    labels : list[str],标签名列表

    返回
    ----
    error : np.float64,均方差

    score : np.float64,决定系数
    """
    # 均方差(The mean squared error),均方差越小越好
    error = np.mean(
        (model.predict(testData[features]) - testData[labels]) ** 2)
    # 决定系数(Coefficient of determination),决定系数越接近1越好
    score = model.score(testData[features], testData[labels])
    return error, score 
Example 46
Project: dynamic-training-with-apache-mxnet-on-aws   Author: awslabs   File: custom_softmax_rtc.py    Apache License 2.0 5 votes vote down vote up
def forward(self, is_train, req, in_data, out_data, aux):
        if req[0] == "null":
            return
        x = in_data[0]  # input
        y = out_data[0] # output

        if y.dtype == np.float64:
            # args, ctx, grid_shape, block_shape, shared_mem = 0
            self.fwd_double_kernel.launch((x, y, x.shape[1], self._reqCode(req[0])), mx.gpu(0), (1, 1, 1), (x.shape[0], 1, 1))
        else:
            # args, ctx, grid_shape, block_shape, shared_mem = 0
            self.fwd_float_kernel.launch((x, y, x.shape[1], self._reqCode(req[0])), mx.gpu(0), (1, 1, 1), (x.shape[0], 1, 1)) 
Example 47
Project: dynamic-training-with-apache-mxnet-on-aws   Author: awslabs   File: custom_softmax_rtc.py    Apache License 2.0 5 votes vote down vote up
def backward(self, req, out_grad, in_data, out_data, in_grad, aux):
        if req[0] == "null":
            return
        l = in_data[1]  # label
        y = out_data[0] # output from the forward pass
        dx = in_grad[0] # the storage for the gradient

        if dx.dtype == np.float64:
            # args, ctx, grid_shape, block_shape, shared_mem = 0
            self.bwd_double_kernel.launch((l, y, dx, self._reqCode(req[0])), mx.gpu(0), (y.shape[0], 1, 1), (y.shape[1], 1, 1))
        else:
            # args, ctx, grid_shape, block_shape, shared_mem = 0
            self.bwd_float_kernel.launch((l, y, dx, self._reqCode(req[0])), mx.gpu(0), (y.shape[0], 1, 1), (y.shape[1], 1, 1)) 
Example 48
Project: dynamic-training-with-apache-mxnet-on-aws   Author: awslabs   File: test_optimizer.py    Apache License 2.0 5 votes vote down vote up
def test_nag():
    opt1 = PyNAG
    opt2 = mx.optimizer.NAG
    shape = (3, 4, 5)
    mom_options = [{}, {'momentum': 0.9}]
    cg_options = [{}, {'clip_gradient': 0.4}, {'clip_gradient': 0.5}]
    rg_options = [{}, {'rescale_grad': 0.14}, {'rescale_grad': 0.8}]
    wd_options = [{}, {'wd': 0.03}, {'wd': 0.05}, {'wd': 0.07}]
    mp_options = [{}, {'multi_precision': False}, {'multi_precision': True}]
    for dtype in [np.float16, np.float32, np.float64]:
        for mom_option in mom_options:
            for cg_option in cg_options:
                for rg_option in rg_options:
                    for wd_option in wd_options:
                        for mp_option in mp_options:
                            kwarg = {}
                            kwarg.update(mom_option)
                            kwarg.update(cg_option)
                            kwarg.update(rg_option)
                            kwarg.update(wd_option)
                            kwarg.update(mp_option)
                            if (dtype == np.float16 and
                                    ('multi_precision' not in kwarg or
                                        not kwarg['multi_precision'])):
                                continue
                            compare_optimizer(opt1(**kwarg), opt2(**kwarg), shape, dtype)



# FTML 
Example 49
Project: dynamic-training-with-apache-mxnet-on-aws   Author: awslabs   File: test_optimizer.py    Apache License 2.0 5 votes vote down vote up
def test_adam():
    opt1 = PyAdam
    opt2 = mx.optimizer.Adam
    shape = (3, 4, 5)
    cg_options = [{}, {'clip_gradient': 0.4}, {'clip_gradient': 0.5}]
    rg_options = [{}, {'rescale_grad': 0.14}, {'rescale_grad': 0.8}]
    wd_options = [{}, {'wd': 0.03}, {'wd': 0.05}, {'wd': 0.07}]
    mp_options = [{}, {'multi_precision': False}, {'multi_precision': True}]
    for dtype in [np.float16, np.float32, np.float64]:
        for cg_option in cg_options:
            for rg_option in rg_options:
                for wd_option in wd_options:
                    for mp_option in mp_options:
                        kwarg = {}
                        kwarg.update(cg_option)
                        kwarg.update(rg_option)
                        kwarg.update(wd_option)
                        kwarg.update(mp_option)
                        if (dtype == np.float16 and
                                ('multi_precision' not in kwarg or
                                    not kwarg['multi_precision'])):
                            continue
                        # atol 2e-5 needed to pass with seed 1248389097
                        compare_optimizer(opt1(lazy_update=False, **kwarg), opt2(**kwarg), shape, dtype,
                                          rtol=1e-4, atol=2e-5)
                        # atol 2e-5 needed to pass with seed 781809840
                        compare_optimizer(opt1(**kwarg), opt2(**kwarg), shape,
                                          dtype, w_stype='row_sparse', g_stype='row_sparse',
                                          rtol=1e-4, atol=2e-5)
                        compare_optimizer(opt1(lazy_update=False, **kwarg), opt2(lazy_update=False, **kwarg), shape,
                                          dtype, w_stype='row_sparse', g_stype='row_sparse',
                                          rtol=1e-4, atol=2e-5)
                        compare_optimizer(opt1(**kwarg), opt2(**kwarg), shape,
                                          dtype, w_stype='default', g_stype='row_sparse',
                                          rtol=1e-4, atol=2e-5)
                        compare_optimizer(opt1(lazy_update=False, **kwarg), opt2(lazy_update=False, **kwarg), shape,
                                          dtype, w_stype='default', g_stype='row_sparse',
                                          rtol=1e-4, atol=2e-5)

# Signum 
Example 50
Project: dynamic-training-with-apache-mxnet-on-aws   Author: awslabs   File: test_optimizer.py    Apache License 2.0 5 votes vote down vote up
def test_signum():
    opt1 = PySignum
    opt2 = mx.optimizer.Signum
    shape = (3, 4, 5)
    cg_options = [{}, {'clip_gradient': 0.4}, {'clip_gradient': 0.5}]
    rg_options = [{}, {'rescale_grad': 0.14}, {'rescale_grad': 0.8}]
    wd_options = [{}, {'wd': 0.03}, {'wd': 0.05}, {'wd': 0.07}]
    wd_lh_options = [{}, {'wd_lh': 0.015}, {'wd_lh': 0.0}]
    mom_options = [{}, {'momentum': 0.9}]
    lr_options = [{'learning_rate': 0.05},{'learning_rate': 0.01}]
    for dtype in [np.float32, np.float64]:
        for cg_option in cg_options:
            for rg_option in rg_options:
                for wd_option in wd_options:
                    for mp_option in wd_lh_options:
                        for lr_option in lr_options:
                            for mom_option in mom_options:
                                kwarg = {}
                                kwarg.update(cg_option)
                                kwarg.update(rg_option)
                                kwarg.update(wd_option)
                                kwarg.update(mp_option)
                                kwarg.update(lr_option)
                                kwarg.update(mom_option)
                                compare_optimizer(opt1(**kwarg), opt2(**kwarg), shape, dtype)


# RMSProp