Python numpy.int32() Examples
The following are code examples for showing how to use numpy.int32(). They are from open source Python projects. You can vote up the examples you like or vote down the ones you don't like.
Example 1
| Project: chainer-openai-transformer-lm Author: soskek File: datasets.py MIT License | 8 votes |
|
def rocstories(data_dir, n_train=1497, n_valid=374):
storys, comps1, comps2, ys = _rocstories(os.path.join(
data_dir, 'cloze_test_val__spring2016 - cloze_test_ALL_val.csv'))
teX1, teX2, teX3, _ = _rocstories(os.path.join(
data_dir, 'cloze_test_test__spring2016 - cloze_test_ALL_test.csv'))
tr_storys, va_storys, tr_comps1, va_comps1, tr_comps2, va_comps2, tr_ys, va_ys = train_test_split(
storys, comps1, comps2, ys, test_size=n_valid, random_state=seed)
trX1, trX2, trX3 = [], [], []
trY = []
for s, c1, c2, y in zip(tr_storys, tr_comps1, tr_comps2, tr_ys):
trX1.append(s)
trX2.append(c1)
trX3.append(c2)
trY.append(y)
vaX1, vaX2, vaX3 = [], [], []
vaY = []
for s, c1, c2, y in zip(va_storys, va_comps1, va_comps2, va_ys):
vaX1.append(s)
vaX2.append(c1)
vaX3.append(c2)
vaY.append(y)
trY = np.asarray(trY, dtype=np.int32)
vaY = np.asarray(vaY, dtype=np.int32)
return (trX1, trX2, trX3, trY), (vaX1, vaX2, vaX3, vaY), (teX1, teX2, teX3)
Example 2
| Project: cat-bbs Author: aleju File: create_dataset.py MIT License | 7 votes |
|
def load_keypoints(image_filepath, image_height, image_width):
"""Load facial keypoints of one image."""
fp_keypoints = "%s.cat" % (image_filepath,)
if not os.path.isfile(fp_keypoints):
raise Exception("Could not find keypoint coordinates for image '%s'." \
% (image_filepath,))
else:
coords_raw = open(fp_keypoints, "r").readlines()[0].strip().split(" ")
coords_raw = [abs(int(coord)) for coord in coords_raw]
keypoints = []
#keypoints_arr = np.zeros((9*2,), dtype=np.int32)
for i in range(1, len(coords_raw), 2): # first element is the number of coords
x = np.clip(coords_raw[i], 0, image_width-1)
y = np.clip(coords_raw[i+1], 0, image_height-1)
keypoints.append((x, y))
return keypoints
Example 3
| Project: chainer-openai-transformer-lm Author: soskek File: datasets.py MIT License | 6 votes |
|
def sst():
train_url = 'https://raw.githubusercontent.com/harvardnlp/sent-conv-torch/master/data/stsa.binary.train'
valid_url = 'https://raw.githubusercontent.com/harvardnlp/sent-conv-torch/master/data/stsa.binary.dev'
test_url = 'https://raw.githubusercontent.com/harvardnlp/sent-conv-torch/master/data/stsa.binary.test'
path = chainer.dataset.cached_download(train_url)
trX, trY = _sst(path)
sys.stderr.write('train data is {}\n'.format(path))
path = chainer.dataset.cached_download(valid_url)
vaX, vaY = _sst(path)
sys.stderr.write('valid data is {}\n'.format(path))
path = chainer.dataset.cached_download(test_url)
teX, teY = _sst(path)
sys.stderr.write('test data is {}\n'.format(path))
trY = np.asarray(trY, dtype=np.int32)
vaY = np.asarray(vaY, dtype=np.int32)
teY = np.asarray(teY, dtype=np.int32)
return (trX, trY), (vaX, vaY), (teX, teY)
Example 4
| Project: explirefit Author: codogogo File: batcher.py Apache License 2.0 | 6 votes |
|
def batch_iter(data, batch_size, num_epochs, shuffle = True): """ Generates a batch iterator for a dataset. """ #data = np.array(data, dtype = np.int32) data_size = len(data) num_batches_per_epoch = int(data_size/batch_size) + 1 for epoch in range(num_epochs): # Shuffle the data at each epoch if shuffle: #shuffle_indices = np.random.permutation(np.arange(data_size)) #shuffled_data = data[shuffle_indices] random.shuffle(data) #else: # shuffled_data = data for batch_num in range(num_batches_per_epoch): start_index = batch_num * batch_size end_index = min((batch_num + 1) * batch_size, data_size) yield data[start_index:end_index]
Example 5
| Project: Att-ChemdNER Author: lingluodlut File: theano_backend.py Apache License 2.0 | 6 votes |
|
def in_top_k(predictions, targets, k):
'''Returns whether the `targets` are in the top `k` `predictions`
# Arguments
predictions: A tensor of shape batch_size x classess and type float32.
targets: A tensor of shape batch_size and type int32 or int64.
k: An int, number of top elements to consider.
# Returns
A tensor of shape batch_size and type int. output_i is 1 if
targets_i is within top-k values of predictions_i
'''
predictions_top_k = T.argsort(predictions)[:, -k:]
result, _ = theano.map(lambda prediction, target: any(equal(prediction, target)), sequences=[predictions_top_k, targets])
return result
# CONVOLUTIONS
Example 6
| Project: Att-ChemdNER Author: lingluodlut File: theano_backend.py Apache License 2.0 | 6 votes |
|
def ctc_path_probs(predict, Y, alpha=1e-4):
smoothed_predict = (1 - alpha) * predict[:, Y] + alpha * np.float32(1.) / Y.shape[0]
L = T.log(smoothed_predict)
zeros = T.zeros_like(L[0])
log_first = zeros
f_skip_idxs = ctc_create_skip_idxs(Y)
b_skip_idxs = ctc_create_skip_idxs(Y[::-1]) # there should be a shortcut to calculating this
def step(log_f_curr, log_b_curr, f_active, log_f_prev, b_active, log_b_prev):
f_active_next, log_f_next = ctc_update_log_p(f_skip_idxs, zeros, f_active, log_f_curr, log_f_prev)
b_active_next, log_b_next = ctc_update_log_p(b_skip_idxs, zeros, b_active, log_b_curr, log_b_prev)
return f_active_next, log_f_next, b_active_next, log_b_next
[f_active, log_f_probs, b_active, log_b_probs], _ = theano.scan(
step, sequences=[L, L[::-1, ::-1]], outputs_info=[np.int32(1), log_first, np.int32(1), log_first])
idxs = T.arange(L.shape[1]).dimshuffle('x', 0)
mask = (idxs < f_active.dimshuffle(0, 'x')) & (idxs < b_active.dimshuffle(0, 'x'))[::-1, ::-1]
log_probs = log_f_probs + log_b_probs[::-1, ::-1] - L
return log_probs, mask
Example 7
| Project: Collaborative-Learning-for-Weakly-Supervised-Object-Detection Author: Sunarker File: imdb.py MIT License | 6 votes |
|
def create_roidb_from_box_list(self, box_list, gt_roidb):
assert len(box_list) == self.num_images, \
'Number of boxes must match number of ground-truth images'
roidb = []
if gt_roidb is not None:
for i in range(self.num_images):
boxes = box_list[i]
real_label = gt_roidb[i]['labels']
roidb.append({'boxes' : boxes,
'labels' : np.array([real_label], dtype=np.int32),
'flipped' : False})
else:
for i in range(self.num_images):
boxes = box_list[i]
roidb.append({'boxes' : boxes,
'labels' : np.zeros((1, 0), dtype=np.int32),
'flipped' : False})
return roidb
Example 8
| Project: Collaborative-Learning-for-Weakly-Supervised-Object-Detection Author: Sunarker File: snippets.py MIT License | 6 votes |
|
def generate_anchors_pre(height, width, feat_stride, anchor_scales=(8,16,32), anchor_ratios=(0.5,1,2)):
""" A wrapper function to generate anchors given different scales
Also return the number of anchors in variable 'length'
"""
anchors = generate_anchors(ratios=np.array(anchor_ratios), scales=np.array(anchor_scales))
A = anchors.shape[0]
shift_x = np.arange(0, width) * feat_stride
shift_y = np.arange(0, height) * feat_stride
shift_x, shift_y = np.meshgrid(shift_x, shift_y)
shifts = np.vstack((shift_x.ravel(), shift_y.ravel(), shift_x.ravel(), shift_y.ravel())).transpose()
K = shifts.shape[0]
# width changes faster, so here it is H, W, C
anchors = anchors.reshape((1, A, 4)) + shifts.reshape((1, K, 4)).transpose((1, 0, 2))
anchors = anchors.reshape((K * A, 4)).astype(np.float32, copy=False)
length = np.int32(anchors.shape[0])
return anchors, length
Example 9
| Project: prediction-constrained-topic-models Author: dtak File: slda_utils__init_manager.py MIT License | 6 votes |
|
def init_topics_KV__rand_active_words(
n_states=10,
frac_words_active=0.5,
blend_frac_active=0.5,
n_vocabs=144,
seed=0):
prng = np.random.RandomState(int(seed))
unif_topics_KV = np.ones((n_states, n_vocabs)) / float(n_vocabs)
active_topics_KV = np.zeros((n_states, n_vocabs))
for k in xrange(n_states):
active_words_U = prng.choice(
np.arange(n_vocabs, dtype=np.int32),
int(frac_words_active * n_vocabs),
replace=False)
active_topics_KV[k, active_words_U] = 1.0 / active_words_U.size
topics_KV = (1 - blend_frac_active) * unif_topics_KV \
+ blend_frac_active * active_topics_KV
topics_KV /= topics_KV.sum(axis=1)[:,np.newaxis]
return topics_KV
Example 10
| Project: prediction-constrained-topic-models Author: dtak File: slda_utils__init_manager.py MIT License | 6 votes |
|
def init_topics_KV__rand_docs(
dataset=None,
n_states=10,
n_vocabs=144,
blend_frac_doc=0.5,
seed=0):
prng = np.random.RandomState(int(seed))
unif_topics_KV = np.ones((n_states, n_vocabs)) / float(n_vocabs)
doc_KV = np.zeros((n_states, n_vocabs))
chosen_doc_ids = prng.choice(
np.arange(dataset['n_docs'], dtype=np.int32),
n_states,
replace=False)
for k in xrange(n_states):
start_d = dataset['doc_indptr_Dp1'][chosen_doc_ids[k]]
stop_d = dataset['doc_indptr_Dp1'][chosen_doc_ids[k] + 1]
active_words_U = dataset['word_id_U'][start_d:stop_d]
doc_KV[k, active_words_U] = dataset['word_ct_U'][start_d:stop_d]
doc_KV /= doc_KV.sum(axis=1)[:,np.newaxis]
topics_KV = (1 - blend_frac_doc) * unif_topics_KV \
+ blend_frac_doc * doc_KV
topics_KV /= topics_KV.sum(axis=1)[:,np.newaxis]
return topics_KV
Example 11
| Project: FasterRCNN_TF_Py3 Author: upojzsb File: snippets.py MIT License | 6 votes |
|
def generate_anchors_pre(height, width, feat_stride, anchor_scales=(8, 16, 32), anchor_ratios=(0.5, 1, 2)):
""" A wrapper function to generate anchors given different scales
Also return the number of anchors in variable 'length'
"""
anchors = generate_anchors(ratios=np.array(anchor_ratios), scales=np.array(anchor_scales))
A = anchors.shape[0]
shift_x = np.arange(0, width) * feat_stride
shift_y = np.arange(0, height) * feat_stride
shift_x, shift_y = np.meshgrid(shift_x, shift_y)
shifts = np.vstack((shift_x.ravel(), shift_y.ravel(), shift_x.ravel(), shift_y.ravel())).transpose()
K = shifts.shape[0]
# width changes faster, so here it is H, W, C
anchors = anchors.reshape((1, A, 4)) + shifts.reshape((1, K, 4)).transpose((1, 0, 2))
anchors = anchors.reshape((K * A, 4)).astype(np.float32, copy=False)
length = np.int32(anchors.shape[0])
return anchors, length
Example 12
| Project: disentangling_conditional_gans Author: zalandoresearch File: dataset_tool.py MIT License | 6 votes |
|
def create_cifar100(tfrecord_dir, cifar100_dir):
print('Loading CIFAR-100 from "%s"' % cifar100_dir)
import pickle
with open(os.path.join(cifar100_dir, 'train'), 'rb') as file:
data = pickle.load(file, encoding='latin1')
images = data['data'].reshape(-1, 3, 32, 32)
labels = np.array(data['fine_labels'])
assert images.shape == (50000, 3, 32, 32) and images.dtype == np.uint8
assert labels.shape == (50000,) and labels.dtype == np.int32
assert np.min(images) == 0 and np.max(images) == 255
assert np.min(labels) == 0 and np.max(labels) == 99
onehot = np.zeros((labels.size, np.max(labels) + 1), dtype=np.float32)
onehot[np.arange(labels.size), labels] = 1.0
with TFRecordExporter(tfrecord_dir, images.shape[0]) as tfr:
order = tfr.choose_shuffled_order()
for idx in range(order.size):
tfr.add_image(images[order[idx]])
tfr.add_labels(onehot[order])
#----------------------------------------------------------------------------
Example 13
| Project: disentangling_conditional_gans Author: zalandoresearch File: dataset.py MIT License | 6 votes |
|
def __init__(self, resolution=1024, num_channels=3, dtype='uint8', dynamic_range=[0,255], label_size=0, label_dtype='float32'):
self.resolution = resolution
self.resolution_log2 = int(np.log2(resolution))
self.shape = [num_channels, resolution, resolution]
self.dtype = dtype
self.dynamic_range = dynamic_range
self.label_size = label_size
self.label_dtype = label_dtype
self._tf_minibatch_var = None
self._tf_lod_var = None
self._tf_minibatch_np = None
self._tf_labels_np = None
assert self.resolution == 2 ** self.resolution_log2
with tf.name_scope('Dataset'):
self._tf_minibatch_var = tf.Variable(np.int32(0), name='minibatch_var')
self._tf_lod_var = tf.Variable(np.int32(0), name='lod_var')
Example 14
| Project: mmdetection Author: open-mmlab File: maskiou_head.py Apache License 2.0 | 6 votes |
|
def _get_area_ratio(self, pos_proposals, pos_assigned_gt_inds, gt_masks):
"""Compute area ratio of the gt mask inside the proposal and the gt
mask of the corresponding instance"""
num_pos = pos_proposals.size(0)
if num_pos > 0:
area_ratios = []
proposals_np = pos_proposals.cpu().numpy()
pos_assigned_gt_inds = pos_assigned_gt_inds.cpu().numpy()
# compute mask areas of gt instances (batch processing for speedup)
gt_instance_mask_area = gt_masks.sum((-1, -2))
for i in range(num_pos):
gt_mask = gt_masks[pos_assigned_gt_inds[i]]
# crop the gt mask inside the proposal
x1, y1, x2, y2 = proposals_np[i, :].astype(np.int32)
gt_mask_in_proposal = gt_mask[y1:y2 + 1, x1:x2 + 1]
ratio = gt_mask_in_proposal.sum() / (
gt_instance_mask_area[pos_assigned_gt_inds[i]] + 1e-7)
area_ratios.append(ratio)
area_ratios = torch.from_numpy(np.stack(area_ratios)).float().to(
pos_proposals.device)
else:
area_ratios = pos_proposals.new_zeros((0, ))
return area_ratios
Example 15
| Project: mmdetection Author: open-mmlab File: mean_ap.py Apache License 2.0 | 6 votes |
|
def get_cls_results(det_results, gt_bboxes, gt_labels, gt_ignore, class_id):
"""Get det results and gt information of a certain class."""
cls_dets = [det[class_id]
for det in det_results] # det bboxes of this class
cls_gts = [] # gt bboxes of this class
cls_gt_ignore = []
for j in range(len(gt_bboxes)):
gt_bbox = gt_bboxes[j]
cls_inds = (gt_labels[j] == class_id + 1)
cls_gt = gt_bbox[cls_inds, :] if gt_bbox.shape[0] > 0 else gt_bbox
cls_gts.append(cls_gt)
if gt_ignore is None:
cls_gt_ignore.append(np.zeros(cls_gt.shape[0], dtype=np.int32))
else:
cls_gt_ignore.append(gt_ignore[j][cls_inds])
return cls_dets, cls_gts, cls_gt_ignore
Example 16
| Project: neural-fingerprinting Author: StephanZheng File: run_attacks_and_defenses.py BSD 3-Clause "New" or "Revised" License | 6 votes |
|
def _load_dataset_clipping(self, dataset_dir, epsilon):
"""Helper method which loads dataset and determines clipping range.
Args:
dataset_dir: location of the dataset.
epsilon: maximum allowed size of adversarial perturbation.
"""
self.dataset_max_clip = {}
self.dataset_min_clip = {}
self._dataset_image_count = 0
for fname in os.listdir(dataset_dir):
if not fname.endswith('.png'):
continue
image_id = fname[:-4]
image = np.array(
Image.open(os.path.join(dataset_dir, fname)).convert('RGB'))
image = image.astype('int32')
self._dataset_image_count += 1
self.dataset_max_clip[image_id] = np.clip(image + epsilon,
0,
255).astype('uint8')
self.dataset_min_clip[image_id] = np.clip(image - epsilon,
0,
255).astype('uint8')
Example 17
| Project: neural-fingerprinting Author: StephanZheng File: test_imagenet_attacks.py BSD 3-Clause "New" or "Revised" License | 6 votes |
|
def load_images(input_dir, metadata_file_path, batch_shape):
"""Retrieve numpy arrays of images and labels, read from a directory."""
num_images = batch_shape[0]
with open(metadata_file_path) as input_file:
reader = csv.reader(input_file)
header_row = next(reader)
rows = list(reader)
row_idx_image_id = header_row.index('ImageId')
row_idx_true_label = header_row.index('TrueLabel')
images = np.zeros(batch_shape)
labels = np.zeros(num_images, dtype=np.int32)
for idx in xrange(num_images):
row = rows[idx]
filepath = os.path.join(input_dir, row[row_idx_image_id] + '.png')
with tf.gfile.Open(filepath, 'rb') as f:
image = np.array(
Image.open(f).convert('RGB')).astype(np.float) / 255.0
images[idx, :, :, :] = image
labels[idx] = int(row[row_idx_true_label])
return images, labels
Example 18
| Project: Black-Box-Audio Author: rtaori File: run_audio_attack.py MIT License | 5 votes |
|
def setup_graph(self, input_audio_batch, target_phrase):
batch_size = input_audio_batch.shape[0]
weird = (input_audio_batch.shape[1] - 1) // 320
logits_arg2 = np.tile(weird, batch_size)
dense_arg1 = np.array(np.tile(target_phrase, (batch_size, 1)), dtype=np.int32)
dense_arg2 = np.array(np.tile(target_phrase.shape[0], batch_size), dtype=np.int32)
pass_in = np.clip(input_audio_batch, -2**15, 2**15-1)
seq_len = np.tile(weird, batch_size).astype(np.int32)
with tf.variable_scope('', reuse=tf.AUTO_REUSE):
inputs = tf.placeholder(tf.float32, shape=pass_in.shape, name='a')
len_batch = tf.placeholder(tf.float32, name='b')
arg2_logits = tf.placeholder(tf.int32, shape=logits_arg2.shape, name='c')
arg1_dense = tf.placeholder(tf.float32, shape=dense_arg1.shape, name='d')
arg2_dense = tf.placeholder(tf.int32, shape=dense_arg2.shape, name='e')
len_seq = tf.placeholder(tf.int32, shape=seq_len.shape, name='f')
logits = get_logits(inputs, arg2_logits)
target = ctc_label_dense_to_sparse(arg1_dense, arg2_dense, len_batch)
ctcloss = tf.nn.ctc_loss(labels=tf.cast(target, tf.int32), inputs=logits, sequence_length=len_seq)
decoded, _ = tf.nn.ctc_greedy_decoder(logits, arg2_logits, merge_repeated=True)
sess = tf.Session()
saver = tf.train.Saver(tf.global_variables())
saver.restore(sess, "models/session_dump")
func1 = lambda a, b, c, d, e, f: sess.run(ctcloss,
feed_dict={inputs: a, len_batch: b, arg2_logits: c, arg1_dense: d, arg2_dense: e, len_seq: f})
func2 = lambda a, b, c, d, e, f: sess.run([ctcloss, decoded],
feed_dict={inputs: a, len_batch: b, arg2_logits: c, arg1_dense: d, arg2_dense: e, len_seq: f})
return (func1, func2)
Example 19
| Project: Black-Box-Audio Author: rtaori File: run_audio_attack.py MIT License | 5 votes |
|
def getctcloss(self, input_audio_batch, target_phrase, decode=False):
batch_size = input_audio_batch.shape[0]
weird = (input_audio_batch.shape[1] - 1) // 320
logits_arg2 = np.tile(weird, batch_size)
dense_arg1 = np.array(np.tile(target_phrase, (batch_size, 1)), dtype=np.int32)
dense_arg2 = np.array(np.tile(target_phrase.shape[0], batch_size), dtype=np.int32)
pass_in = np.clip(input_audio_batch, -2**15, 2**15-1)
seq_len = np.tile(weird, batch_size).astype(np.int32)
if decode:
return self.funcs[1](pass_in, batch_size, logits_arg2, dense_arg1, dense_arg2, seq_len)
else:
return self.funcs[0](pass_in, batch_size, logits_arg2, dense_arg1, dense_arg2, seq_len)
Example 20
| Project: SyNEThesia Author: RunOrVeith File: feature_creators.py MIT License | 5 votes |
|
def _split_into_chunks(signal, chunks_per_second=24):
# TODO currently broken
raise NotImplemented("Splitting to chunks is currently broken.")
window_length_ms = 1/chunks_per_second * 1000
intervals = np.arange(window_length_ms, signal.shape[0], window_length_ms, dtype=np.int32)
chunks = np.array_split(signal, intervals, axis=0)
pad_to = _next_power_of_two(np.max([chunk.shape[0] for chunk in chunks]))
padded_chunks = np.stack(np.concatenate([chunk, np.zeros((pad_to - chunk.shape[0],))]) for chunk in chunks)
return padded_chunks
Example 21
| Project: speed_estimation Author: NeilNie File: helper.py MIT License | 5 votes |
|
def udacity_val_gen(data, batch_size):
"""
Generate training image give image paths and associated steering angles
"""
images = np.empty([batch_size, configs.LENGTH, configs.IMG_HEIGHT, configs.IMG_WIDTH, 3], dtype=np.int32)
labels = np.empty([batch_size])
while True:
c = 0
for index in np.random.permutation(data.shape[0]):
imgs = np.empty([configs.LENGTH, configs.IMG_HEIGHT, configs.IMG_WIDTH, 3], dtype=np.int32)
if index < configs.LENGTH:
start = 0
end = configs.LENGTH
elif index + configs.LENGTH >= len(data):
start = len(data) - configs.LENGTH - 1
end = len(data) - 1
else:
start = index
end = index + configs.LENGTH
for i in range(start, end):
center_path = '/home/neil/dataset/steering/test/center/' + str(data['frame_id'].loc[i]) + ".jpg"
image = load_image(center_path)
imgs[i - start] = image
images[c] = imgs
labels[c] = data['steering_angle'].loc[end]
c += 1
if c == batch_size:
break
yield images, labels
Example 22
| Project: chainer-openai-transformer-lm Author: soskek File: train.py MIT License | 5 votes |
|
def transform_roc(X1, X2, X3):
n_batch = len(X1)
xmb = np.zeros((n_batch, 2, n_ctx, 2), dtype=np.int32)
mmb = np.zeros((n_batch, 2, n_ctx), dtype=np.float32)
start = encoder['_start_']
delimiter = encoder['_delimiter_']
for i, (x1, x2, x3), in enumerate(zip(X1, X2, X3)):
x12 = [start] + x1[:max_len] + [delimiter] + x2[:max_len] + [clf_token]
x13 = [start] + x1[:max_len] + [delimiter] + x3[:max_len] + [clf_token]
l12 = len(x12)
l13 = len(x13)
xmb[i, 0, :l12, 0] = x12
xmb[i, 1, :l13, 0] = x13
mmb[i, 0, :l12] = 1
mmb[i, 1, :l13] = 1
xmb[:, :, :, 1] = np.arange(
n_vocab + n_special, n_vocab + n_special + n_ctx)
return xmb, mmb
Example 23
| Project: chainer-openai-transformer-lm Author: soskek File: train.py MIT License | 5 votes |
|
def transform_sst(X1):
n_batch = len(X1)
xmb = np.zeros((n_batch, 1, n_ctx, 2), dtype=np.int32)
mmb = np.zeros((n_batch, 1, n_ctx), dtype=np.float32)
start = encoder['_start_']
delimiter = encoder['_delimiter_']
for i, x1, in enumerate(X1):
x1 = [start] + x1[:max_len] + [clf_token]
l1 = len(x1)
xmb[i, 0, :l1, 0] = x1
mmb[i, 0, :l1] = 1
xmb[:, :, :, 1] = np.arange(
n_vocab + n_special, n_vocab + n_special + n_ctx)
return xmb, mmb
Example 24
| Project: cat-bbs Author: aleju File: bbs.py MIT License | 5 votes |
|
def draw_on_image(self, img, color=[0, 255, 0], alpha=1.0, thickness=1, copy=copy):
assert img.dtype in [np.uint8, np.float32, np.int32, np.int64]
result = np.copy(img) if copy else img
for i in range(thickness):
y = [self.y1-i, self.y1-i, self.y2+i, self.y2+i]
x = [self.x1-i, self.x2+i, self.x2+i, self.x1-i]
rr, cc = draw.polygon_perimeter(y, x, shape=img.shape)
if alpha >= 0.99:
result[rr, cc, 0] = color[0]
result[rr, cc, 1] = color[1]
result[rr, cc, 2] = color[2]
else:
if result.dtype == np.float32:
result[rr, cc, 0] = (1 - alpha) * result[rr, cc, 0] + alpha * color[0]
result[rr, cc, 1] = (1 - alpha) * result[rr, cc, 1] + alpha * color[1]
result[rr, cc, 2] = (1 - alpha) * result[rr, cc, 2] + alpha * color[2]
result = np.clip(result, 0, 255)
else:
result = result.astype(np.float32)
result[rr, cc, 0] = (1 - alpha) * result[rr, cc, 0] + alpha * color[0]
result[rr, cc, 1] = (1 - alpha) * result[rr, cc, 1] + alpha * color[1]
result[rr, cc, 2] = (1 - alpha) * result[rr, cc, 2] + alpha * color[2]
result = np.clip(result, 0, 255).astype(np.uint8)
return result
Example 25
| Project: cat-bbs Author: aleju File: common.py MIT License | 5 votes |
|
def draw_heatmap(img, heatmap, alpha=0.5):
"""Draw a heatmap overlay over an image."""
assert len(heatmap.shape) == 2 or \
(len(heatmap.shape) == 3 and heatmap.shape[2] == 1)
assert img.dtype in [np.uint8, np.int32, np.int64]
assert heatmap.dtype in [np.float32, np.float64]
if img.shape[0:2] != heatmap.shape[0:2]:
heatmap_rs = np.clip(heatmap * 255, 0, 255).astype(np.uint8)
heatmap_rs = ia.imresize_single_image(
heatmap_rs[..., np.newaxis],
img.shape[0:2],
interpolation="nearest"
)
heatmap = np.squeeze(heatmap_rs) / 255.0
cmap = plt.get_cmap('jet')
heatmap_cmapped = cmap(heatmap)
heatmap_cmapped = np.delete(heatmap_cmapped, 3, 2)
heatmap_cmapped = heatmap_cmapped * 255
mix = (1-alpha) * img + alpha * heatmap_cmapped
mix = np.clip(mix, 0, 255).astype(np.uint8)
return mix
Example 26
| Project: rhodonite Author: nestauk File: test_cumulative.py MIT License | 5 votes |
|
def test_steps_cumulative_cooccurrence_graph(self, simple_cumco_graph):
g, o, o_cumsum, co, co_cumsum = simple_cumco_graph
steps_expected = np.array([0, 1, 2], dtype=np.int32)
assert_array_equal(g.gp['steps'], steps_expected)
Example 27
| Project: Att-ChemdNER Author: lingluodlut File: utils.py Apache License 2.0 | 5 votes |
|
def evaluate(parameters, f_eval, raw_sentences, parsed_sentences,
id_to_tag, dictionary_tags,filename,
useAttend=True):
#{{{
"""
Evaluate current model using CoNLL script.
"""
n_tags = len(id_to_tag)
predictions = []
count = np.zeros((n_tags, n_tags), dtype=np.int32)
for raw_sentence, data in zip(raw_sentences, parsed_sentences):
input = create_input(data, parameters, False,useAttend=useAttend)
if parameters['crf']:
y_preds = np.array(f_eval(*input))
else:
y_preds = f_eval(*input).argmax(axis=1)
y_reals = np.array(data['tags']).astype(np.int32)
assert len(y_preds) == len(y_reals)
p_tags = [id_to_tag[y_pred] for y_pred in y_preds]
r_tags = [id_to_tag[y_real] for y_real in y_reals]
if parameters['tag_scheme'] == 'iobes':
p_tags = iobes_iob(p_tags)
r_tags = iobes_iob(r_tags)
for i, (y_pred, y_real) in enumerate(zip(y_preds, y_reals)):
new_line = " ".join(raw_sentence[i][:-1] + [r_tags[i], p_tags[i]])
predictions.append(new_line)
count[y_real, y_pred] += 1
predictions.append("")
#write to file
with codecs.open(filename, 'w', 'utf8') as f:
f.write("\n".join(predictions))
return get_perf(filename)
#}}}
Example 28
| Project: Att-ChemdNER Author: lingluodlut File: model.py Apache License 2.0 | 5 votes |
|
def modelScore(self,tag_ids,scores,s_len):
#{{{
"""
ATTENTATION THIS FUNCTION IS SYMBOL PROGRAMMING
this function is to return the score of our model at a fixed sentence label
@param:
scores: the scores matrix ,the output of our model
tag: a numpy array, which represent one sentence label
sent_lens: a scalar number, the length of sentence.
because our sentence label will be expand to max sentence length,
so we will use this to get the original sentence label.
@return:
a scalar number ,the score;
"""
#{{{
n_tags=self.output_dim;
transitions=self.transitions;
#score from tags_scores
real_path_score = scores[T.arange(s_len), tag_ids].sum()
# Score from transitions
b_id = theano.shared(value=np.array([n_tags], dtype=np.int32))
e_id = theano.shared(value=np.array([n_tags + 1], dtype=np.int32))
padded_tags_ids = T.concatenate([b_id, tag_ids, e_id], axis=0)
real_path_score += transitions[
padded_tags_ids[T.arange(s_len + 1)],
padded_tags_ids[T.arange(s_len + 1) + 1]
].sum()
#to prevent T.exp(real_path_score) to be inf
#return real_path_score;
return real_path_score/s_len;
#}}}
#}}}
Example 29
| Project: Att-ChemdNER Author: lingluodlut File: theano_backend.py Apache License 2.0 | 5 votes |
|
def arange(start, stop=None, step=1, dtype='int32'):
'''Creates a 1-D tensor containing a sequence of integers.
The function arguments use the same convention as
Theano's arange: if only one argument is provided,
it is in fact the "stop" argument.
The default type of the returned tensor is 'int32' to
match TensorFlow's default.
'''
return T.arange(start, stop=stop, step=step, dtype=dtype)
Example 30
| Project: Att-ChemdNER Author: lingluodlut File: theano_backend.py Apache License 2.0 | 5 votes |
|
def sparse_categorical_crossentropy(output, target, from_logits=False):
target = T.cast(T.flatten(target), 'int32')
target = T.extra_ops.to_one_hot(target, nb_class=output.shape[-1])
target = reshape(target, shape(output))
return categorical_crossentropy(output, target, from_logits)
Example 31
| Project: TensorFlow-TransX Author: thunlp File: transE.py MIT License | 5 votes |
|
def __init__(self, config):
entity_total = config.entity
relation_total = config.relation
batch_size = config.batch_size
size = config.hidden_size
margin = config.margin
self.pos_h = tf.placeholder(tf.int32, [None])
self.pos_t = tf.placeholder(tf.int32, [None])
self.pos_r = tf.placeholder(tf.int32, [None])
self.neg_h = tf.placeholder(tf.int32, [None])
self.neg_t = tf.placeholder(tf.int32, [None])
self.neg_r = tf.placeholder(tf.int32, [None])
with tf.name_scope("embedding"):
self.ent_embeddings = tf.get_variable(name = "ent_embedding", shape = [entity_total, size], initializer = tf.contrib.layers.xavier_initializer(uniform = False))
self.rel_embeddings = tf.get_variable(name = "rel_embedding", shape = [relation_total, size], initializer = tf.contrib.layers.xavier_initializer(uniform = False))
pos_h_e = tf.nn.embedding_lookup(self.ent_embeddings, self.pos_h)
pos_t_e = tf.nn.embedding_lookup(self.ent_embeddings, self.pos_t)
pos_r_e = tf.nn.embedding_lookup(self.rel_embeddings, self.pos_r)
neg_h_e = tf.nn.embedding_lookup(self.ent_embeddings, self.neg_h)
neg_t_e = tf.nn.embedding_lookup(self.ent_embeddings, self.neg_t)
neg_r_e = tf.nn.embedding_lookup(self.rel_embeddings, self.neg_r)
if config.L1_flag:
pos = tf.reduce_sum(abs(pos_h_e + pos_r_e - pos_t_e), 1, keep_dims = True)
neg = tf.reduce_sum(abs(neg_h_e + neg_r_e - neg_t_e), 1, keep_dims = True)
self.predict = pos
else:
pos = tf.reduce_sum((pos_h_e + pos_r_e - pos_t_e) ** 2, 1, keep_dims = True)
neg = tf.reduce_sum((neg_h_e + neg_r_e - neg_t_e) ** 2, 1, keep_dims = True)
self.predict = pos
with tf.name_scope("output"):
self.loss = tf.reduce_sum(tf.maximum(pos - neg + margin, 0))
Example 32
| Project: Collaborative-Learning-for-Weakly-Supervised-Object-Detection Author: Sunarker File: pascal_voc.py MIT License | 5 votes |
|
def _load_pascal_labels(self, index):
"""
Load image labels.
"""
filename = os.path.join(self._data_path, 'Annotations', index + '.xml')
tree = ET.parse(filename)
objs = tree.findall('object')
if not self.config['use_diff']:
# Exclude the samples labeled as difficult
non_diff_objs = [
obj for obj in objs if int(obj.find('difficult').text) == 0]
# if len(non_diff_objs) != len(objs):
# print 'Removed {} difficult objects'.format(
# len(objs) - len(non_diff_objs))
objs = non_diff_objs
num_objs = len(objs)
gt_classes = np.zeros((num_objs), dtype=np.int32)
# Load object bounding boxes into a data frame.
for ix, obj in enumerate(objs):
cls = self._class_to_ind[obj.find('name').text.lower().strip()]
gt_classes[ix] = cls
real_label = np.zeros(self.num_classes).astype(np.float32)
for label in gt_classes:
real_label[label] = 1
return {'labels' : real_label}
Example 33
| Project: prediction-constrained-topic-models Author: dtak File: slda_utils__dataset_manager.py MIT License | 5 votes |
|
def make_dataset_subset(doc_ids=None, dataset=None, **kwargs):
''' Make dataset using subset of the examples in a larger dataset.
Returns
-------
subset : dict
'''
doc_ids = np.asarray(doc_ids, dtype=np.int32)
n_vocabs = dataset['n_vocabs']
doc_indptr_Dp1 = dataset['doc_indptr_Dp1']
word_id_U = dataset['word_id_U']
word_ct_U = dataset['word_ct_U']
subset_wid_list = list()
subset_wct_list = list()
doclen_list = list()
for d in doc_ids:
start_d = doc_indptr_Dp1[d]
stop_d = doc_indptr_Dp1[d+1]
subset_wid_list.append(word_id_U[start_d:stop_d])
subset_wct_list.append(word_ct_U[start_d:stop_d])
doclen_list.append(stop_d - start_d)
subset = dict(
word_id_U=np.hstack(subset_wid_list),
word_ct_U=np.hstack(subset_wct_list),
doc_indptr_Dp1=np.hstack([0,np.cumsum(doclen_list)]),
n_docs=len(doc_ids),
n_vocabs=n_vocabs)
subset['x_csr_DV'] = scipy.sparse.csr_matrix(
(subset['word_ct_U'],
subset['word_id_U'],
subset['doc_indptr_Dp1']),
shape=(subset['n_docs'], subset['n_vocabs']))
if 'y_DC' in dataset:
subset['y_DC'] = dataset['y_DC'][doc_ids]
subset['n_labels'] = dataset['n_labels']
if 'y_rowmask' in dataset:
subset['y_rowmask'] = dataset['y_rowmask'][doc_ids]
return subset
Example 34
| Project: prediction-constrained-topic-models Author: dtak File: util_io_csr.py MIT License | 5 votes |
|
def process_ldac_line_into_preallocated_arrays(line, word_id, word_ct, start):
"""
Returns
-------
Examples
--------
>>> word_id = np.zeros(5, dtype=np.int32)
>>> word_ct = np.zeros(5, dtype=np.float64)
>>> a = process_ldac_line_into_preallocated_arrays(
... '5 66:6 77:7 88:8',
... word_id, word_ct, 0)
>>> a
5
>>> word_id.tolist()
[66, 77, 88, 0, 0]
>>> word_ct.tolist()
[ 6.,7., 8., 0., 0.]
"""
line = line.replace(':', ' ')
data = np.fromstring(line, sep=' ', dtype=np.int32)
stop = start + (len(data) - 1) // 2
if stop >= word_id.size:
raise IndexError("Provided array not large enough")
word_id[start:stop] = data[1::2]
word_ct[start:stop] = data[2::2]
return data[0]
Example 35
| Project: prediction-constrained-topic-models Author: dtak File: util_array.py MIT License | 5 votes |
|
def toCArray(X, dtype=np.float64):
""" Convert input into numpy array of C-contiguous order.
Ensures returned array is aligned and owns its own data,
not a view of another array.
Returns
-------
X : ND array
Examples
-------
>>> Q = np.zeros(10, dtype=np.int32, order='F')
>>> toCArray(Q).flags.c_contiguous
True
>>> toCArray(Q).dtype.byteorder
'='
"""
X = np.asarray_chkfinite(X, dtype=dtype, order='C')
if X.dtype.byteorder != '=':
X = X.newbyteorder('=').copy()
if not X.flags.owndata or X.flags.aligned:
X = X.copy()
assert X.flags.owndata
assert X.flags.aligned
return X
Example 36
| Project: prediction-constrained-topic-models Author: dtak File: select_best_runs_and_snapshots.py MIT License | 5 votes |
|
def make_best_task_df(
df,
target_query="SPLIT_NAME == 'VALID' and LAP > 50",
score_colname='Y_ERROR_RATE',
score_ranking_func=np.argmin,
default_score=None,
verbose=False):
''' Find best task for each unique job in provided df.
Returns
-------
best_df : dataframe of best tasks for each unique job
'''
if default_score is None:
default_score = fetch_default_score(score_ranking_func.__name__)
best_task_df_list = list()
job_paths = np.unique(df['JOB_PATH'].values)
for job_path in job_paths:
if job_path is None:
continue
job_df = df.query("JOB_PATH == '%s'" % job_path)
taskids = np.unique(job_df['TASKID'].values)
best_score_idx = np.zeros_like(taskids, dtype=np.int32)
best_score = default_score * np.ones_like(taskids, dtype=np.float64)
for tt, taskidstr in enumerate(taskids):
task_df = job_df.query(target_query + " and TASKID == '%s'" % taskidstr)
if task_df.shape[0] < 1:
continue
if not np.all(np.isfinite(task_df[score_colname].values)):
pprint(task_df[score_colname].values)
best_score_idx[tt] = score_ranking_func(task_df[score_colname].values)
best_score[tt] = task_df[score_colname].values[best_score_idx[tt]]
best_task_idx = score_ranking_func(best_score)
best_task_df = job_df.query("TASKID == '%s'" % taskids[best_task_idx])
best_task_df_list.append(best_task_df)
if verbose:
pprint(job_path)
pprint("best task: %s" % best_task_idx)
return pd.concat(best_task_df_list)
Example 37
| Project: AutoDL Author: tanguofu File: test_custom_pyop.py BSD 3-Clause "New" or "Revised" License | 5 votes |
|
def struct_unpack(bytes):
lists = struct.unpack('IIII', bytes)
shape = tf.TensorShape(lists)
return np.array(shape.as_list(), np.int32)
Example 38
| Project: AutoDL Author: tanguofu File: test_custom_pyop.py BSD 3-Clause "New" or "Revised" License | 5 votes |
|
def convert_shape_to_tensor(shape):
return tf.convert_to_tensor(shape.as_list(), dtype=tf.int32)
Example 39
| Project: AutoDL Author: tanguofu File: test_custom_pyop.py BSD 3-Clause "New" or "Revised" License | 5 votes |
|
def test_pack():
x = tf.placeholder(tf.int32, shape=(4,))
shapepack = tf.py_func(struct_pack, [x],tf.string)
with tf.Session() as sess:
v = sess.run(shapepack, feed_dict={x:[1,2,3,4]})
print((type(v),v))
Example 40
| Project: AutoDL Author: tanguofu File: test_custom_pyop.py BSD 3-Clause "New" or "Revised" License | 5 votes |
|
def test_unpack():
z = tf.placeholder(tf.string, shape=())
shapeunpack = tf.py_func(struct_unpack, [z], tf.int32)
v = b'\x01\x00\x00\x00\x02\x00\x00\x00\x03\x00\x00\x00\x04\x00\x00\x00'
with tf.Session() as sess:
v = sess.run([shapeunpack], feed_dict={z:v})
print(v)
Example 41
| Project: ieml Author: IEMLdev File: _test_thesaurus.py GNU General Public License v3.0 | 5 votes |
|
def test_paradigm_citation_dist(self):
paradigm = sc("E:O:O:.")
cells = [
Term("E:U:U:."),
Term("E:U:A:."),
Term("E:A:U:."),
Term("E:A:A:.")
]
headers = [Term("E:O:U:."), Term("E:O:A:."), Term("E:U:O:."), Term("E:A:O:.")]
usl_collection = [
usl(Word(Morpheme([cells[0], cells[1]]), Morpheme([cells[3], cells[1], cells[2]]))),
usl(Word(Morpheme([cells[0], cells[2]]), Morpheme([cells[3]]))),
usl(Word(Morpheme([headers[0], cells[3]]), Morpheme([headers[3], headers[2]]))),
usl(Word(Morpheme([cells[1]]), Morpheme([cells[1]])))
]
result = paradigm_usl_distribution(paradigm, usl_collection)
correct_result = np.zeros((2, 2), dtype=np.int32)
correct_result[0][0] = 4
correct_result[0][1] = 5
correct_result[1][0] = 4
correct_result[1][1] = 4
self.assertEqual(len(result), 1, "The paradigm has one table so we should have one distribution table")
self.assertTrue(np.array_equal(result[0], correct_result))
Example 42
| Project: FasterRCNN_TF_Py3 Author: upojzsb File: imdb.py MIT License | 5 votes |
|
def create_roidb_from_box_list(self, box_list, gt_roidb):
assert len(box_list) == self.num_images, \
'Number of boxes must match number of ground-truth images'
roidb = []
for i in range(self.num_images):
boxes = box_list[i]
num_boxes = boxes.shape[0]
overlaps = np.zeros((num_boxes, self.num_classes), dtype=np.float32)
if gt_roidb is not None and gt_roidb[i]['boxes'].size > 0:
gt_boxes = gt_roidb[i]['boxes']
gt_classes = gt_roidb[i]['gt_classes']
gt_overlaps = bbox_overlaps(boxes.astype(np.float),
gt_boxes.astype(np.float))
argmaxes = gt_overlaps.argmax(axis=1)
maxes = gt_overlaps.max(axis=1)
I = np.where(maxes > 0)[0]
overlaps[I, gt_classes[argmaxes[I]]] = maxes[I]
overlaps = scipy.sparse.csr_matrix(overlaps)
roidb.append({
'boxes': boxes,
'gt_classes': np.zeros((num_boxes,), dtype=np.int32),
'gt_overlaps': overlaps,
'flipped': False,
'seg_areas': np.zeros((num_boxes,), dtype=np.float32),
})
return roidb
Example 43
| Project: yolov3-tf2 Author: zzh8829 File: utils.py MIT License | 5 votes |
|
def draw_outputs(img, outputs, class_names):
boxes, objectness, classes, nums = outputs
boxes, objectness, classes, nums = boxes[0], objectness[0], classes[0], nums[0]
wh = np.flip(img.shape[0:2])
for i in range(nums):
x1y1 = tuple((np.array(boxes[i][0:2]) * wh).astype(np.int32))
x2y2 = tuple((np.array(boxes[i][2:4]) * wh).astype(np.int32))
img = cv2.rectangle(img, x1y1, x2y2, (255, 0, 0), 2)
img = cv2.putText(img, '{} {:.4f}'.format(
class_names[int(classes[i])], objectness[i]),
x1y1, cv2.FONT_HERSHEY_COMPLEX_SMALL, 1, (0, 0, 255), 2)
return img
Example 44
| Project: yolov3-tf2 Author: zzh8829 File: utils.py MIT License | 5 votes |
|
def draw_labels(x, y, class_names):
img = x.numpy()
boxes, classes = tf.split(y, (4, 1), axis=-1)
classes = classes[..., 0]
wh = np.flip(img.shape[0:2])
for i in range(len(boxes)):
x1y1 = tuple((np.array(boxes[i][0:2]) * wh).astype(np.int32))
x2y2 = tuple((np.array(boxes[i][2:4]) * wh).astype(np.int32))
img = cv2.rectangle(img, x1y1, x2y2, (255, 0, 0), 2)
img = cv2.putText(img, class_names[classes[i]],
x1y1, cv2.FONT_HERSHEY_COMPLEX_SMALL,
1, (0, 0, 255), 2)
return img
Example 45
| Project: disentangling_conditional_gans Author: zalandoresearch File: dataset_tool.py MIT License | 5 votes |
|
def create_cifar10(tfrecord_dir, cifar10_dir):
print('Loading CIFAR-10 from "%s"' % cifar10_dir)
import pickle
images = []
labels = []
for batch in range(1, 6):
with open(os.path.join(cifar10_dir, 'data_batch_%d' % batch), 'rb') as file:
data = pickle.load(file, encoding='latin1')
images.append(data['data'].reshape(-1, 3, 32, 32))
labels.append(data['labels'])
images = np.concatenate(images)
labels = np.concatenate(labels)
assert images.shape == (50000, 3, 32, 32) and images.dtype == np.uint8
assert labels.shape == (50000,) and labels.dtype == np.int32
assert np.min(images) == 0 and np.max(images) == 255
assert np.min(labels) == 0 and np.max(labels) == 9
onehot = np.zeros((labels.size, np.max(labels) + 1), dtype=np.float32)
onehot[np.arange(labels.size), labels] = 1.0
with TFRecordExporter(tfrecord_dir, images.shape[0]) as tfr:
order = tfr.choose_shuffled_order()
for idx in range(order.size):
tfr.add_image(images[order[idx]])
tfr.add_labels(onehot[order])
#----------------------------------------------------------------------------
Example 46
| Project: disentangling_conditional_gans Author: zalandoresearch File: dataset.py MIT License | 5 votes |
|
def get_random_labels_tf(self, minibatch_size): # => labels
if self.label_size > 0:
return tf.gather(self._tf_labels_var, tf.random_uniform([minibatch_size], 0, self._np_labels.shape[0], dtype=tf.int32))
else:
return tf.zeros([minibatch_size, 0], self.label_dtype)
# Get random labels as NumPy array.
Example 47
| Project: disentangling_conditional_gans Author: zalandoresearch File: dataset.py MIT License | 5 votes |
|
def get_minibatch_tf(self): # => images, labels
with tf.name_scope('SyntheticDataset'):
shrink = tf.cast(2.0 ** tf.cast(self._tf_lod_var, tf.float32), tf.int32)
shape = [self.shape[0], self.shape[1] // shrink, self.shape[2] // shrink]
images = self._generate_images(self._tf_minibatch_var, self._tf_lod_var, shape)
labels = self._generate_labels(self._tf_minibatch_var)
return images, labels
Example 48
| Project: Electrolyte_Analysis_FTIR Author: Samuel-Buteau File: Constant_run.py MIT License | 5 votes |
|
def __init__(self, n_samples=None, list_of_indecies=None):
if not n_samples is None:
self.GetFresh_list = numpy.arange(n_samples, dtype=numpy.int32)
self.get_fresh_count = n_samples
elif not list_of_indecies is None:
self.GetFresh_list = numpy.array(copy.deepcopy(list_of_indecies))
self.get_fresh_count = len(self.GetFresh_list)
else:
raise Exception('Invalid Input')
numpy.random.shuffle(self.GetFresh_list)
self.get_fresh_pos = 0
Example 49
| Project: Electrolyte_Analysis_FTIR Author: Samuel-Buteau File: Constant_run.py MIT License | 5 votes |
|
def get(self, n):
"""
will return a list of n random numbers in self.GetFresh_list
- Samuel Buteau, October 2018
"""
if n >= self.get_fresh_count:
return numpy.concatenate((self.get(int(n/2)),self.get(n- int(n/2))))
reshuffle_flag = False
n_immediate_fulfill = min(n, self.get_fresh_count - self.get_fresh_pos)
batch_of_indecies = numpy.empty([n], dtype=numpy.int32)
for i in range(0, n_immediate_fulfill):
batch_of_indecies[i] = self.GetFresh_list[i + self.get_fresh_pos]
self.get_fresh_pos += n_immediate_fulfill
if self.get_fresh_pos >= self.get_fresh_count:
self.get_fresh_pos -= self.get_fresh_count
reshuffle_flag = True
# Now, the orders that needed to be satisfied are satisfied.
n_delayed_fulfill = max(0, n - n_immediate_fulfill)
if reshuffle_flag:
numpy.random.shuffle(self.GetFresh_list)
if n_delayed_fulfill > 0:
for i in range(0, n_delayed_fulfill):
batch_of_indecies[i + n_immediate_fulfill] = self.GetFresh_list[i]
self.get_fresh_pos = n_delayed_fulfill
return batch_of_indecies
Example 50
| Project: Tacotron Author: ElwynWang File: data.py GNU General Public License v3.0 | 5 votes |
|
def parser(transcript, ref_path):
transcript = tf.decode_raw(transcript, tf.int32) # decode string to int
transcript = tf.pad(transcript, ([0, Hp.num_charac], ))[:Hp.num_charac] # pad charac[0] to fill in the whole max length Tx
def load_audio(audio_path):
sub_paths = audio_path.strip().split('/')
main_path = '/'.join(sub_paths[:-2])
fname = os.path.basename(audio_path)
mel_path = main_path + "/mels/{}".format(sub_paths[-1].replace('wav', 'npy')) # mel spectrogrom
mag_path = main_path + "/mags/{}".format(sub_paths[-1].replace('wav', 'npy')) # wavform
mel = np.load(mel_path)
mag = np.load(mag_path)
ref_len = mel.shape[0] if mel.shape[1]==Hp.num_mels*Hp.reduction_factor \
else mel.shape[0]*mel.shape[1]//Hp.num_mels*Hp.reduction_factor
ref_len = np.array(ref_len, dtype = np.int32)
return mel, mag, ref_len
spectrogrom, wavform, ref_len = tf.py_func(load_audio, [ref_path], [tf.float32, tf.float32, tf.int32])
transcript = tf.reshape(transcript, [Hp.num_charac,])
spectrogrom = tf.reshape(spectrogrom, [-1, Hp.num_mels*Hp.reduction_factor])
#ref_len = tf.cast(spectrogrom.get_shape()[0], tf.int32)
wavform = tf.reshape(wavform, [-1, Hp.num_fft//2+1])
inputs = list((transcript, spectrogrom, tf.reshape(ref_len, [1]), tf.reshape(tf.constant(0), [1]), spectrogrom, wavform))
#inputs = {'transcript':transcript, 'reference':spectrogrom, 'ref_len':tf.reshape(ref_len,[1]),
#'speaker':tf.reshape(tf.constant(0), [1]), 'decoder':spectrogrom, 'labels':wavform}
return inputs
Example 51
| Project: mmdetection Author: open-mmlab File: recall.py Apache License 2.0 | 5 votes |
|
def print_recall_summary(recalls,
proposal_nums,
iou_thrs,
row_idxs=None,
col_idxs=None):
"""Print recalls in a table.
Args:
recalls(ndarray): calculated from `bbox_recalls`
proposal_nums(ndarray or list): top N proposals
iou_thrs(ndarray or list): iou thresholds
row_idxs(ndarray): which rows(proposal nums) to print
col_idxs(ndarray): which cols(iou thresholds) to print
"""
proposal_nums = np.array(proposal_nums, dtype=np.int32)
iou_thrs = np.array(iou_thrs)
if row_idxs is None:
row_idxs = np.arange(proposal_nums.size)
if col_idxs is None:
col_idxs = np.arange(iou_thrs.size)
row_header = [''] + iou_thrs[col_idxs].tolist()
table_data = [row_header]
for i, num in enumerate(proposal_nums[row_idxs]):
row = [
'{:.3f}'.format(val)
for val in recalls[row_idxs[i], col_idxs].tolist()
]
row.insert(0, num)
table_data.append(row)
table = AsciiTable(table_data)
print(table.table)
Example 52
| Project: mmdetection Author: open-mmlab File: mask_target.py Apache License 2.0 | 5 votes |
|
def mask_target_single(pos_proposals, pos_assigned_gt_inds, gt_masks, cfg):
mask_size = _pair(cfg.mask_size)
num_pos = pos_proposals.size(0)
mask_targets = []
if num_pos > 0:
proposals_np = pos_proposals.cpu().numpy()
_, maxh, maxw = gt_masks.shape
proposals_np[:, [0, 2]] = np.clip(proposals_np[:, [0, 2]], 0, maxw - 1)
proposals_np[:, [1, 3]] = np.clip(proposals_np[:, [1, 3]], 0, maxh - 1)
pos_assigned_gt_inds = pos_assigned_gt_inds.cpu().numpy()
for i in range(num_pos):
gt_mask = gt_masks[pos_assigned_gt_inds[i]]
bbox = proposals_np[i, :].astype(np.int32)
x1, y1, x2, y2 = bbox
w = np.maximum(x2 - x1 + 1, 1)
h = np.maximum(y2 - y1 + 1, 1)
# mask is uint8 both before and after resizing
# mask_size (h, w) to (w, h)
target = mmcv.imresize(gt_mask[y1:y1 + h, x1:x1 + w],
mask_size[::-1])
mask_targets.append(target)
mask_targets = torch.from_numpy(np.stack(mask_targets)).float().to(
pos_proposals.device)
else:
mask_targets = pos_proposals.new_zeros((0, ) + mask_size)
return mask_targets
Example 53
| Project: speed_estimation Author: NeilNie File: helper.py MIT License | 4 votes |
|
def comma_batch_generator(data, batch_size, augment):
"""
Generate training images given image paths and associated steering angles
:param data : (numpy.array) the loaded data (converted to list from pandas format)
:param batch_size : (int) batch size for training
:param training : (boolean): whether to use augmentation or not.
:rtype: Iterator[images, angles] images for training
the corresponding steering angles
"""
images = np.empty([batch_size, configs.LENGTH, configs.IMG_HEIGHT, configs.IMG_WIDTH, configs.CHANNELS], dtype=np.int32)
labels = np.empty([batch_size])
while True:
c = 0
for index in np.random.permutation(data.shape[0]):
imgs = np.empty([configs.LENGTH, configs.IMG_HEIGHT, configs.IMG_WIDTH, configs.CHANNELS], dtype=np.int32)
if index < configs.LENGTH:
start = 0
end = configs.LENGTH
elif index + configs.LENGTH >= len(data):
start = len(data) - configs.LENGTH - 1
end = len(data) - 1
else:
start = index
end = index + configs.LENGTH
for i in range(start, end):
center_path = "/home/neil/dataset/speedchallenge/data/train/" + str(data[i][1])
image = load_image(center_path)
imgs[i - start] = image
# augmentaion if needed
if augment and bool(random.getrandbits(1)):
imgs = augument(imgs)
angle = data[end][2]
images[c] = imgs
labels[c] = angle
c += 1
if c == batch_size:
break
yield images, labels
Example 54
| Project: speed_estimation Author: NeilNie File: helper.py MIT License | 4 votes |
|
def comma_accel_batch_generator(data, batch_size, augment=False):
"""
Generate training images given image paths and associated acceleration commands
:param data : (numpy.array) the loaded data (converted to list from pandas format)
:param batch_size : (int) batch size for training
:param training : (boolean): whether to use augmentation or not.
:rtype: Iterator[images, angles] images for training
the corresponding steering angles
"""
images = np.empty([batch_size, configs.LENGTH, configs.IMG_HEIGHT, configs.IMG_WIDTH, configs.CHANNELS], dtype=np.int32)
labels = np.empty([batch_size])
while True:
c = 0
for index in np.random.permutation(data.shape[0]):
imgs = np.empty([configs.LENGTH, configs.IMG_HEIGHT, configs.IMG_WIDTH, configs.CHANNELS], dtype=np.int32)
if index < configs.LENGTH:
start = 0
end = configs.LENGTH
elif index + configs.LENGTH >= len(data):
start = len(data) - configs.LENGTH - 1
end = len(data) - 1
else:
start = index
end = index + configs.LENGTH
for i in range(start, end):
center_path = "/home/neil/dataset/speedchallenge/data/train/" + str(data[i][1])
image = load_image(center_path)
imgs[i - start] = image
# augmentaion if needed (not recommended)
if augment and bool(random.getrandbits(1)):
imgs = augument(imgs)
speed = data[end][2]
pre_speed = data[end-1][2]
images[c] = imgs
labels[c] = (speed - pre_speed) * 20 # frame rate
c += 1
if c == batch_size:
break
yield images, labels
Example 55
| Project: speed_estimation Author: NeilNie File: helper.py MIT License | 4 votes |
|
def comma_flow_accel_batch_gen(batch_size, data):
"""
Generate training images given image paths and associated steering angles
:param data : (numpy.array) the loaded data (converted to list from pandas format)
:param batch_size : (int) batch size for training
:rtype: Iterator[images, angles] images for training the corresponding steering angles
"""
images = np.empty([batch_size, configs.LENGTH, configs.IMG_HEIGHT, configs.IMG_WIDTH, 2], dtype=np.int32)
labels = np.empty([batch_size])
while True:
c = 0
for index in np.random.permutation(data.shape[0]):
imgs = np.empty([configs.LENGTH, configs.IMG_HEIGHT, configs.IMG_WIDTH, 2], dtype=np.int32)
if index < configs.LENGTH + 1:
start = 0
end = configs.LENGTH + 1
elif index + configs.LENGTH + 1 >= len(data):
start = len(data) - configs.LENGTH - 1
end = len(data) - 1
else:
start = index
end = index + configs.LENGTH + 1
grays = []
for i in range(start, end):
path = "/home/neil/dataset/speedchallenge/data/train/" + str(data[i][1])
gray = load_gray_image(path)
grays.append(gray)
current = grays[0]
for i in range(1, len(grays)):
flow = cv2.calcOpticalFlowFarneback(current, grays[i], None, 0.5, 3, 15, 3, 5, 1.5, 0)
current = grays[i]
imgs[i - 1] = flow
accel = (data[end][2] - data[end-1][2]) * 20
images[c] = imgs
labels[c] = accel
c += 1
if c == batch_size:
break
yield images, labels
Example 56
| Project: speed_estimation Author: NeilNie File: helper.py MIT License | 4 votes |
|
def comma_flow_batch_gen(data, batch_size):
"""
Generate training images given image paths and associated steering angles
:param data : (numpy.array) the loaded data (converted to list from pandas format)
:param batch_size : (int) batch size for training
:rtype: Iterator[images, angles] images for training
the corresponding steering angles
"""
images = np.empty([batch_size, configs.LENGTH, configs.IMG_HEIGHT, configs.IMG_WIDTH, 2], dtype=np.int32)
labels = np.empty([batch_size])
while True:
c = 0
for index in np.random.permutation(data.shape[0]):
imgs = np.empty([configs.LENGTH, configs.IMG_HEIGHT, configs.IMG_WIDTH, 2], dtype=np.int32)
if index < configs.LENGTH + 1:
start = 0
end = configs.LENGTH + 1
elif index + configs.LENGTH + 1 >= len(data):
start = len(data) - configs.LENGTH - 1
end = len(data) - 1
else:
start = index
end = index + configs.LENGTH + 1
grays = []
for i in range(start, end):
path = "/home/neil/dataset/speedchallenge/data/train/" + str(data[i][1])
gray = load_gray_image(path)
grays.append(gray)
current = grays[0]
for i in range(1, len(grays)):
flow = cv2.calcOpticalFlowFarneback(current, grays[i], None, 0.5, 3, 15, 3, 5, 1.5, 0)
current = grays[i]
imgs[i-1] = flow
speed = data[end][2]
images[c] = imgs
labels[c] = speed
c += 1
if c == batch_size:
break
yield images, labels
Example 57
| Project: speed_estimation Author: NeilNie File: helper.py MIT License | 4 votes |
|
def udacity_train_gen(data, batch_size, augment):
"""
Generate training images given image paths and associated steering angles
Args:
data (numpy.array) : the loaded data (converted to list from pandas format)
batch_size (int) : batch size for training
training: (boolean): whether to use augmentation or not.
Yields:
images ([tensor]) : images for training
angles ([float]) : the corresponding steering angles
"""
images = np.empty([batch_size, configs.LENGTH, configs.IMG_HEIGHT, configs.IMG_WIDTH, 3], dtype=np.int32)
labels = np.empty([batch_size])
while True:
c = 0
for index in np.random.permutation(data.shape[0]):
imgs = np.empty([configs.LENGTH, configs.IMG_HEIGHT, configs.IMG_WIDTH, 3], dtype=np.int32)
if index < configs.LENGTH:
start = 0
end = configs.LENGTH
elif index + configs.LENGTH >= len(data):
start = len(data) - configs.LENGTH - 1
end = len(data) - 1
else:
start = index
end = index + configs.LENGTH
for i in range(start, end):
center_path = str(data[i][5])
image = load_image(center_path)
imgs[i - start] = image
# augmentaion if needed
if augment and bool(random.getrandbits(1)):
imgs, angle = augument(imgs, data[end][8])
else:
angle = data[end][8]
images[c] = imgs
labels[c] = angle
c += 1
if c == batch_size:
break
yield images, labels
Example 58
| Project: explirefit Author: codogogo File: data_shaper.py Apache License 2.0 | 4 votes |
|
def prep_classification(texts, labels, embeddings, stopwords = None, embeddings_language = 'en', multilingual_langs = None, lowercase = False, pad = True, pad_token = '<PAD/>', numbers_token = None, punct_token = None, dist_labels = None, max_seq_len = None, add_out_of_vocabulary_terms = False): x = [] y = [] for i in range(len(texts)): tok_list = [] lab_list = [] language = embeddings_language if multilingual_langs is None else multilingual_langs[i] for j in range(len(texts[i])): token_clean = texts[i][j].lower() if lowercase else texts[i][j] token = token_clean if multilingual_langs is None else multilingual_langs[i] + "__" + token_clean if token_clean.strip() in punctuation() and punct_token is not None: token = punct_token if is_number(token_clean) and numbers_token is not None: token = numbers_token if stopwords is not None and (token_clean in stopwords[language] or token_clean.lower() in stopwords[language]): continue if token not in embeddings.lang_vocabularies[embeddings_language] and token.lower() not in embeddings.lang_vocabularies[embeddings_language]: if add_out_of_vocabulary_terms: embeddings.add_word(embeddings_language, token) else: continue if max_seq_len is None or len(tok_list) < max_seq_len: tok_list.append(embeddings.lang_vocabularies[embeddings_language][token] if token in embeddings.lang_vocabularies[embeddings_language] else embeddings.lang_vocabularies[embeddings_language][token.lower()]) else: break x.append(tok_list) if labels is not None: if dist_labels is None: dist_labels = list(set([l for txt_labs in labels for l in txt_labs])) for i in range(len(labels)): lab_vec = [0] * len(dist_labels) for j in range(len(labels[i])): lab_vec[dist_labels.index(labels[i][j])] = 1.0 y.append(lab_vec) if pad: ind_pad = embeddings.lang_vocabularies[embeddings_language][pad_token] max_len = max([len(t) for t in x]) if max_seq_len is None else max_seq_len x = [t + [ind_pad] * (max_len - len(t)) for t in x] if labels is not None: x_ret = np.array(x, dtype = np.int32) y_ret = np.array(y, dtype = np.float64) return x_ret, y_ret, dist_labels else: return np.array(x, dtype = np.int32)
Example 59
| Project: TensorFlow-TransX Author: thunlp File: transR.py MIT License | 4 votes |
|
def __init__(self, config, ent_init = None, rel_init = None):
entity_total = config.entity
relation_total = config.relation
batch_size = config.batch_size
sizeE = config.hidden_sizeE
sizeR = config.hidden_sizeR
margin = config.margin
with tf.name_scope("read_inputs"):
self.pos_h = tf.placeholder(tf.int32, [batch_size])
self.pos_t = tf.placeholder(tf.int32, [batch_size])
self.pos_r = tf.placeholder(tf.int32, [batch_size])
self.neg_h = tf.placeholder(tf.int32, [batch_size])
self.neg_t = tf.placeholder(tf.int32, [batch_size])
self.neg_r = tf.placeholder(tf.int32, [batch_size])
with tf.name_scope("embedding"):
if ent_init != None:
self.ent_embeddings = tf.Variable(np.loadtxt(ent_init), name = "ent_embedding", dtype = np.float32)
else:
self.ent_embeddings = tf.get_variable(name = "ent_embedding", shape = [entity_total, sizeE], initializer = tf.contrib.layers.xavier_initializer(uniform = False))
if rel_init != None:
self.rel_embeddings = tf.Variable(np.loadtxt(rel_init), name = "rel_embedding", dtype = np.float32)
else:
self.rel_embeddings = tf.get_variable(name = "rel_embedding", shape = [relation_total, sizeR], initializer = tf.contrib.layers.xavier_initializer(uniform = False))
rel_matrix = np.zeros([relation_total, sizeR * sizeE], dtype = np.float32)
for i in range(relation_total):
for j in range(sizeR):
for k in range(sizeE):
if j == k:
rel_matrix[i][j * sizeE + k] = 1.0
self.rel_matrix = tf.Variable(rel_matrix, name = "rel_matrix")
with tf.name_scope('lookup_embeddings'):
pos_h_e = tf.reshape(tf.nn.embedding_lookup(self.ent_embeddings, self.pos_h), [-1, sizeE, 1])
pos_t_e = tf.reshape(tf.nn.embedding_lookup(self.ent_embeddings, self.pos_t), [-1, sizeE, 1])
pos_r_e = tf.reshape(tf.nn.embedding_lookup(self.rel_embeddings, self.pos_r), [-1, sizeR])
neg_h_e = tf.reshape(tf.nn.embedding_lookup(self.ent_embeddings, self.neg_h), [-1, sizeE, 1])
neg_t_e = tf.reshape(tf.nn.embedding_lookup(self.ent_embeddings, self.neg_t), [-1, sizeE, 1])
neg_r_e = tf.reshape(tf.nn.embedding_lookup(self.rel_embeddings, self.neg_r), [-1, sizeR])
pos_matrix = tf.reshape(tf.nn.embedding_lookup(self.rel_matrix, self.pos_r), [-1, sizeR, sizeE])
neg_matrix = tf.reshape(tf.nn.embedding_lookup(self.rel_matrix, self.neg_r), [-1, sizeR, sizeE])
pos_h_e = tf.nn.l2_normalize(tf.reshape(tf.matmul(pos_matrix, pos_h_e), [-1, sizeR]), 1)
pos_t_e = tf.nn.l2_normalize(tf.reshape(tf.matmul(pos_matrix, pos_t_e), [-1, sizeR]), 1)
neg_h_e = tf.nn.l2_normalize(tf.reshape(tf.matmul(neg_matrix, neg_h_e), [-1, sizeR]), 1)
neg_t_e = tf.nn.l2_normalize(tf.reshape(tf.matmul(neg_matrix, neg_t_e), [-1, sizeR]), 1)
if config.L1_flag:
pos = tf.reduce_sum(abs(pos_h_e + pos_r_e - pos_t_e), 1, keep_dims = True)
neg = tf.reduce_sum(abs(neg_h_e + neg_r_e - neg_t_e), 1, keep_dims = True)
self.predict = pos
else:
pos = tf.reduce_sum((pos_h_e + pos_r_e - pos_t_e) ** 2, 1, keep_dims = True)
neg = tf.reduce_sum((neg_h_e + neg_r_e - neg_t_e) ** 2, 1, keep_dims = True)
self.predict = pos
with tf.name_scope("output"):
self.loss = tf.reduce_sum(tf.maximum(pos - neg + margin, 0))
Example 60
| Project: TensorFlow-TransX Author: thunlp File: transD.py MIT License | 4 votes |
|
def __init__(self, config):
entity_total = config.entity
relation_total = config.relation
batch_size = config.batch_size
size = config.hidden_size
margin = config.margin
self.pos_h = tf.placeholder(tf.int32, [None])
self.pos_t = tf.placeholder(tf.int32, [None])
self.pos_r = tf.placeholder(tf.int32, [None])
self.neg_h = tf.placeholder(tf.int32, [None])
self.neg_t = tf.placeholder(tf.int32, [None])
self.neg_r = tf.placeholder(tf.int32, [None])
with tf.name_scope("embedding"):
self.ent_embeddings = tf.get_variable(name = "ent_embedding", shape = [entity_total, size], initializer = tf.contrib.layers.xavier_initializer(uniform = False))
self.rel_embeddings = tf.get_variable(name = "rel_embedding", shape = [relation_total, size], initializer = tf.contrib.layers.xavier_initializer(uniform = False))
self.ent_transfer = tf.get_variable(name = "ent_transfer", shape = [entity_total, size], initializer = tf.contrib.layers.xavier_initializer(uniform = False))
self.rel_transfer = tf.get_variable(name = "rel_transfer", shape = [relation_total, size], initializer = tf.contrib.layers.xavier_initializer(uniform = False))
pos_h_e = tf.nn.embedding_lookup(self.ent_embeddings, self.pos_h)
pos_t_e = tf.nn.embedding_lookup(self.ent_embeddings, self.pos_t)
pos_r_e = tf.nn.embedding_lookup(self.rel_embeddings, self.pos_r)
pos_h_t = tf.nn.embedding_lookup(self.ent_transfer, self.pos_h)
pos_t_t = tf.nn.embedding_lookup(self.ent_transfer, self.pos_t)
pos_r_t = tf.nn.embedding_lookup(self.rel_transfer, self.pos_r)
neg_h_e = tf.nn.embedding_lookup(self.ent_embeddings, self.neg_h)
neg_t_e = tf.nn.embedding_lookup(self.ent_embeddings, self.neg_t)
neg_r_e = tf.nn.embedding_lookup(self.rel_embeddings, self.neg_r)
neg_h_t = tf.nn.embedding_lookup(self.ent_transfer, self.neg_h)
neg_t_t = tf.nn.embedding_lookup(self.ent_transfer, self.neg_t)
neg_r_t = tf.nn.embedding_lookup(self.rel_transfer, self.neg_r)
pos_h_e = self.calc(pos_h_e, pos_h_t, pos_r_t)
pos_t_e = self.calc(pos_t_e, pos_t_t, pos_r_t)
neg_h_e = self.calc(neg_h_e, neg_h_t, neg_r_t)
neg_t_e = self.calc(neg_t_e, neg_t_t, neg_r_t)
if config.L1_flag:
pos = tf.reduce_sum(abs(pos_h_e + pos_r_e - pos_t_e), 1, keep_dims = True)
neg = tf.reduce_sum(abs(neg_h_e + neg_r_e - neg_t_e), 1, keep_dims = True)
self.predict = pos
else:
pos = tf.reduce_sum((pos_h_e + pos_r_e - pos_t_e) ** 2, 1, keep_dims = True)
neg = tf.reduce_sum((neg_h_e + neg_r_e - neg_t_e) ** 2, 1, keep_dims = True)
self.predict = pos
with tf.name_scope("output"):
self.loss = tf.reduce_sum(tf.maximum(pos - neg + margin, 0))
Example 61
| Project: TensorFlow-TransX Author: thunlp File: transH.py MIT License | 4 votes |
|
def __init__(self, config):
entity_total = config.entity
relation_total = config.relation
batch_size = config.batch_size
size = config.hidden_size
margin = config.margin
self.pos_h = tf.placeholder(tf.int32, [None])
self.pos_t = tf.placeholder(tf.int32, [None])
self.pos_r = tf.placeholder(tf.int32, [None])
self.neg_h = tf.placeholder(tf.int32, [None])
self.neg_t = tf.placeholder(tf.int32, [None])
self.neg_r = tf.placeholder(tf.int32, [None])
with tf.name_scope("embedding"):
self.ent_embeddings = tf.get_variable(name = "ent_embedding", shape = [entity_total, size], initializer = tf.contrib.layers.xavier_initializer(uniform = False))
self.rel_embeddings = tf.get_variable(name = "rel_embedding", shape = [relation_total, size], initializer = tf.contrib.layers.xavier_initializer(uniform = False))
self.normal_vector = tf.get_variable(name = "normal_vector", shape = [relation_total, size], initializer = tf.contrib.layers.xavier_initializer(uniform = False))
pos_h_e = tf.nn.embedding_lookup(self.ent_embeddings, self.pos_h)
pos_t_e = tf.nn.embedding_lookup(self.ent_embeddings, self.pos_t)
pos_r_e = tf.nn.embedding_lookup(self.rel_embeddings, self.pos_r)
neg_h_e = tf.nn.embedding_lookup(self.ent_embeddings, self.neg_h)
neg_t_e = tf.nn.embedding_lookup(self.ent_embeddings, self.neg_t)
neg_r_e = tf.nn.embedding_lookup(self.rel_embeddings, self.neg_r)
pos_norm = tf.nn.embedding_lookup(self.normal_vector, self.pos_r)
neg_norm = tf.nn.embedding_lookup(self.normal_vector, self.neg_r)
pos_h_e = self.calc(pos_h_e, pos_norm)
pos_t_e = self.calc(pos_t_e, pos_norm)
neg_h_e = self.calc(neg_h_e, neg_norm)
neg_t_e = self.calc(neg_t_e, neg_norm)
if config.L1_flag:
pos = tf.reduce_sum(abs(pos_h_e + pos_r_e - pos_t_e), 1, keep_dims = True)
neg = tf.reduce_sum(abs(neg_h_e + neg_r_e - neg_t_e), 1, keep_dims = True)
self.predict = pos
else:
pos = tf.reduce_sum((pos_h_e + pos_r_e - pos_t_e) ** 2, 1, keep_dims = True)
neg = tf.reduce_sum((neg_h_e + neg_r_e - neg_t_e) ** 2, 1, keep_dims = True)
self.predict = pos
with tf.name_scope("output"):
self.loss = tf.reduce_sum(tf.maximum(pos - neg + margin, 0))
Example 62
| Project: Collaborative-Learning-for-Weakly-Supervised-Object-Detection Author: Sunarker File: pascal_voc.py MIT License | 4 votes |
|
def _load_pascal_annotation(self, index):
"""
Load image and bounding boxes info from XML file in the PASCAL VOC
format.
"""
filename = os.path.join(self._data_path, 'Annotations', index + '.xml')
tree = ET.parse(filename)
objs = tree.findall('object')
if not self.config['use_diff']:
# Exclude the samples labeled as difficult
non_diff_objs = [
obj for obj in objs if int(obj.find('difficult').text) == 0]
# if len(non_diff_objs) != len(objs):
# print 'Removed {} difficult objects'.format(
# len(objs) - len(non_diff_objs))
objs = non_diff_objs
num_objs = len(objs)
boxes = np.zeros((num_objs, 4), dtype=np.uint16)
gt_classes = np.zeros((num_objs), dtype=np.int32)
overlaps = np.zeros((num_objs, self.num_classes), dtype=np.float32)
# "Seg" area for pascal is just the box area
seg_areas = np.zeros((num_objs), dtype=np.float32)
# Load object bounding boxes into a data frame.
for ix, obj in enumerate(objs):
bbox = obj.find('bndbox')
# Make pixel indexes 0-based
x1 = float(bbox.find('xmin').text) - 1
y1 = float(bbox.find('ymin').text) - 1
x2 = float(bbox.find('xmax').text) - 1
y2 = float(bbox.find('ymax').text) - 1
cls = self._class_to_ind[obj.find('name').text.lower().strip()]
boxes[ix, :] = [x1, y1, x2, y2]
gt_classes[ix] = cls
overlaps[ix, cls] = 1.0
seg_areas[ix] = (x2 - x1 + 1) * (y2 - y1 + 1)
overlaps = scipy.sparse.csr_matrix(overlaps)
return {'boxes': boxes,
'gt_classes': gt_classes,
'gt_overlaps': overlaps,
'flipped': False,
'seg_areas': seg_areas}
Example 63
| Project: Collaborative-Learning-for-Weakly-Supervised-Object-Detection Author: Sunarker File: coco.py MIT License | 4 votes |
|
def _load_coco_annotation(self, index):
"""
Loads COCO bounding-box instance annotations. Crowd instances are
handled by marking their overlaps (with all categories) to -1. This
overlap value means that crowd "instances" are excluded from training.
"""
im_ann = self._COCO.loadImgs(index)[0]
width = im_ann['width']
height = im_ann['height']
annIds = self._COCO.getAnnIds(imgIds=index, iscrowd=None)
objs = self._COCO.loadAnns(annIds)
# Sanitize bboxes -- some are invalid
valid_objs = []
for obj in objs:
x1 = np.max((0, obj['bbox'][0]))
y1 = np.max((0, obj['bbox'][1]))
x2 = np.min((width - 1, x1 + np.max((0, obj['bbox'][2] - 1))))
y2 = np.min((height - 1, y1 + np.max((0, obj['bbox'][3] - 1))))
if obj['area'] > 0 and x2 >= x1 and y2 >= y1:
obj['clean_bbox'] = [x1, y1, x2, y2]
valid_objs.append(obj)
objs = valid_objs
num_objs = len(objs)
boxes = np.zeros((num_objs, 4), dtype=np.uint16)
gt_classes = np.zeros((num_objs), dtype=np.int32)
overlaps = np.zeros((num_objs, self.num_classes), dtype=np.float32)
seg_areas = np.zeros((num_objs), dtype=np.float32)
# Lookup table to map from COCO category ids to our internal class
# indices
coco_cat_id_to_class_ind = dict([(self._class_to_coco_cat_id[cls],
self._class_to_ind[cls])
for cls in self._classes[1:]])
for ix, obj in enumerate(objs):
cls = coco_cat_id_to_class_ind[obj['category_id']]
boxes[ix, :] = obj['clean_bbox']
gt_classes[ix] = cls
seg_areas[ix] = obj['area']
if obj['iscrowd']:
# Set overlap to -1 for all classes for crowd objects
# so they will be excluded during training
overlaps[ix, :] = -1.0
else:
overlaps[ix, cls] = 1.0
ds_utils.validate_boxes(boxes, width=width, height=height)
overlaps = scipy.sparse.csr_matrix(overlaps)
return {'width': width,
'height': height,
'boxes': boxes,
'gt_classes': gt_classes,
'gt_overlaps': overlaps,
'flipped': False,
'seg_areas': seg_areas}
Example 64
| Project: yolov3-detector Author: ccerhan File: darknet.py MIT License | 4 votes |
|
def load_weights(self, weights_path):
"""
Parses and loads the weights stored in 'weights_path'.
:param weights_path: YOLOv3 weights file path.
"""
with open(weights_path, 'rb') as fp:
header = np.fromfile(fp, dtype=np.int32, count=5)
weights = np.fromfile(fp, dtype=np.float32)
ptr = 0
for i, (module_def, module) in enumerate(zip(self.module_defs, self.module_list)):
if module_def['type'] == 'convolutional':
conv_layer = module[0]
if module_def['batch_normalize']:
# Load BN bias, weights, running mean and running variance
bn_layer = module[1]
num_b = bn_layer.bias.numel() # Number of biases
# Bias
bn_b = torch.from_numpy(weights[ptr : ptr + num_b]).view_as(bn_layer.bias)
bn_layer.bias.data.copy_(bn_b)
ptr += num_b
# Weight
bn_w = torch.from_numpy(weights[ptr : ptr + num_b]).view_as(bn_layer.weight)
bn_layer.weight.data.copy_(bn_w)
ptr += num_b
# Running Mean
bn_rm = torch.from_numpy(weights[ptr : ptr + num_b]).view_as(bn_layer.running_mean)
bn_layer.running_mean.data.copy_(bn_rm)
ptr += num_b
# Running Var
bn_rv = torch.from_numpy(weights[ptr : ptr + num_b]).view_as(bn_layer.running_var)
bn_layer.running_var.data.copy_(bn_rv)
ptr += num_b
else:
# Load conv. bias
num_b = conv_layer.bias.numel()
conv_b = torch.from_numpy(weights[ptr : ptr + num_b]).view_as(conv_layer.bias)
conv_layer.bias.data.copy_(conv_b)
ptr += num_b
# Load conv. weights
num_w = conv_layer.weight.numel()
conv_w = torch.from_numpy(weights[ptr : ptr + num_w]).view_as(conv_layer.weight)
conv_layer.weight.data.copy_(conv_w)
ptr += num_w
Example 65
| Project: prediction-constrained-topic-models Author: dtak File: slda_utils__dataset_manager.py MIT License | 4 votes |
|
def slice_dataset(
cur_slice=None, dataset=None,
max_n_examples_per_slice=np.inf,
include_rowmask=True,
**kwargs):
''' Create slice of provided dataset.
Returns
-------
slice_dict : dict of subset of data
'''
if cur_slice is None:
cur_slice = slice(0, dataset['n_docs'])
if cur_slice.stop - cur_slice.start > max_n_examples_per_slice:
cur_slice = slice(
cur_slice.start,
cur_slice.start + max_n_examples_per_slice)
n_vocabs = dataset['n_vocabs']
doc_indptr_Dp1 = dataset['doc_indptr_Dp1']
word_id_U = dataset['word_id_U']
word_ct_U = dataset['word_ct_U']
u_start = doc_indptr_Dp1[cur_slice.start]
u_stop = doc_indptr_Dp1[cur_slice.stop]
d_start = cur_slice.start
d_stop = cur_slice.stop
n_docs = cur_slice.stop - cur_slice.start
word_id_U = word_id_U[u_start:u_stop]
word_ct_U = word_ct_U[u_start:u_stop]
doc_indptr_Dp1 = doc_indptr_Dp1[d_start:(d_stop + 1)].copy()
doc_indptr_Dp1 = doc_indptr_Dp1 - doc_indptr_Dp1[0]
slice_dict = dict(
n_docs=n_docs,
n_vocabs=n_vocabs,
word_id_U=word_id_U,
word_ct_U=word_ct_U,
doc_indptr_Dp1=doc_indptr_Dp1,
x_csr_DV=scipy.sparse.csr_matrix(
(word_ct_U, word_id_U, doc_indptr_Dp1),
shape=(n_docs, n_vocabs)),
)
if 'y_DC' in dataset:
slice_dict['y_DC'] = dataset['y_DC'][cur_slice]
slice_dict['n_labels'] = slice_dict['y_DC'].shape[1]
if include_rowmask and 'y_rowmask' in dataset:
slice_dict['y_rowmask'] = dataset['y_rowmask'][cur_slice]
elif 'y_rowmask' in dataset:
# handle case where some things are real nan values
y_rowmask = np.all(np.isfinite(dataset['y_DC'][cur_slice]), axis=1).astype(np.int32)
if np.sum(y_rowmask) < y_rowmask.size:
slice_dict['y_rowmask'] = y_rowmask
return slice_dict
Example 66
| Project: prediction-constrained-topic-models Author: dtak File: calc_coherence_metrics.py MIT License | 4 votes |
|
def calc_umass_coherence_for_top_ranked_terms_in_topic(
top_vocab_ids=None,
ndocs_V=None,
ndocs_csc_VV=None,
topics_KV=None,
k=None,
dataset=None,
pair_smooth_eps=0.1,
marg_smooth_eps=1e-9,
):
""" Compute Coherence metric for given topic's top-ranked terms.
Returns
-------
coherence_score : float
Larger values indicate more coherent topics.
Examples
--------
>>> x_DV = np.arange(6)[:,np.newaxis] * np.hstack([np.eye(6), np.zeros((6, 3))])
>>> x_DV[:3, :3] += 1
>>> x_DV[4, 5] += 17
>>> ndocs_V, ndocs_csc_VV = calc_pairwise_cooccurance_counts(x_csr_DV=x_DV)
>>> coh = calc_umass_coherence_for_top_ranked_terms_in_topic([0, 8], ndocs_V, ndocs_csc_VV)
>>> coh2 = np.log(0.1 / 3.0)
>>> np.allclose(coh, coh2)
True
>>> coh_good = calc_umass_coherence_for_top_ranked_terms_in_topic([0, 1, 2], ndocs_V, ndocs_csc_VV)
>>> coh_bad = calc_umass_coherence_for_top_ranked_terms_in_topic([0, 4, 5], ndocs_V, ndocs_csc_VV)
>>> coh_worst = calc_umass_coherence_for_top_ranked_terms_in_topic([0, 3, 7], ndocs_V, ndocs_csc_VV)
>>> coh_good > coh_bad
True
>>> coh_bad > coh_worst
True
"""
V = ndocs_V.size
top_vocab_ids = np.asarray(top_vocab_ids, dtype=np.int32)
M = top_vocab_ids.size
coherence_score = 0.0
for mm, v in enumerate(top_vocab_ids[:-1]):
Mrem = M - mm - 1
counts_Mrem = ndocs_csc_VV[v, top_vocab_ids[mm+1:]]
try:
counts_Mrem = counts_Mrem.toarray()
except AttributeError:
pass
assert counts_Mrem.size == Mrem
coherence_score += (
np.sum(np.log(counts_Mrem + pair_smooth_eps))
- Mrem * np.log(ndocs_V[v] + marg_smooth_eps)
)
return coherence_score
Example 67
| Project: prediction-constrained-topic-models Author: dtak File: util_io_csr.py MIT License | 4 votes |
|
def load_csr_matrix_from_ldac_txtfile(
filepath=None,
shape=None,
n_vocabs=None,
index_dtype=np.int32,
data_dtype=np.float64,
):
''' Creates csr_matrix from a .ldac formatted plain-text file.
Returns
-------
x_DV : scipy.sparse.csr_matrix
'''
assert n_vocabs is not None or shape is not None
# Estimate num tokens in the file
fileSize_bytes = os.path.getsize(filepath)
nTokensPerByte = 1.0 / 5
estimate_nUniqueTokens = int(nTokensPerByte * fileSize_bytes)
# Preallocate space
word_id_U = np.zeros(estimate_nUniqueTokens, dtype=index_dtype)
word_ct_U = np.zeros(estimate_nUniqueTokens, dtype=data_dtype)
nSeen = 0
doc_sizes = []
with open(filepath, 'r') as f:
# Simple case: read the whole file
for line in f.readlines():
nUnique_d = -1
while nUnique_d < 0:
try:
nUnique_d = process_ldac_line_into_preallocated_arrays(
line, word_id, word_ct, nSeen)
assert nUnique_d >= 0
except IndexError as e:
# Preallocated arrays not large enough
# Double our preallocation, then try again
extra_word_id = np.zeros(word_id.size, dtype=word_id.dtype)
extra_word_ct = np.zeros(word_ct.size, dtype=word_ct.dtype)
word_id = np.hstack([word_id, extra_word_id])
word_ct = np.hstack([word_ct, extra_word_ct])
doc_sizes.append(nUnique_d)
nSeen += nUnique_d
word_id = word_id[:nSeen]
word_ct = word_ct[:nSeen]
n_docs = len(doc_sizes)
doc_range = np.hstack([0, np.cumsum(doc_sizes)], dtype=index_dtype)
if shape is None:
assert n_vocabs is not None
shape = (n_docs, n_vocabs)
x_csr_DV = scipy.csr_matrix(
(word_ct, word_id, doc_range),
shape=shape)
return x_csr_DV
Example 68
| Project: prediction-constrained-topic-models Author: dtak File: train_and_eval_sklearn_binary_classifier.py MIT License | 4 votes |
|
def __init__(self, clf=None, proba_thr_for_class1=0.5, classes=[0,1]):
""" Make hard predictions at custom-tuned thresholds
Args
----
clf : sklearn ClassifierMixin
threshold : float within (0.0, 1.0)
Provides value at which we call labels of second category
classes : list
Provides numeric/string values of class labels
Examples
--------
# Create toy dataset with 80% label=0, 20% label=1
>>> prng = np.random.RandomState(0)
>>> x_N = prng.randn(100, 1)
>>> y_N = np.asarray(prng.rand(100) > 0.8, dtype=np.int32)
# 'Train' neighbor classifier
>>> clf = KNeighborsClassifier(n_neighbors=100);
>>> clf = clf.fit(x_N, y_N)
>>> clf.classes_
array([0, 1], dtype=int32)
# A classifier with 0.5 threshold calls all 0
>>> thr050 = ThresholdClassifier(clf, 0.5)
>>> thr050.predict(x_N).min()
0
# A classifier with 0.15 threshold calls all 1
>>> thr015 = ThresholdClassifier(clf, 0.15)
>>> thr015.predict(x_N).min()
1
# A classifier with 0.95 threshold calls all 0
>>> thr015 = ThresholdClassifier(clf, 0.95)
>>> thr015.predict(x_N).min()
0
"""
self.clf = clf
self.proba_thr_for_class1 = proba_thr_for_class1
try:
self.classes_ = clf.classes_
except AttributeError:
self.classes_ = classes
assert len(self.classes_) == 2
Example 69
| Project: AutoDL Author: tanguofu File: nccl_ops_test.py BSD 3-Clause "New" or "Revised" License | 4 votes |
|
def _Test(self,
nccl_reduce,
numpy_fn,
device_sets=(['/device:GPU:1', '/device:GPU:2', '/device:GPU:0'],
['/device:GPU:1', '/device:GPU:0'])):
"""Tests that nccl_reduce does the same as reduction with numpy_fn.
Args:
nccl_reduce: A function taking a list of tensors and a list of devices,
and returns a list of reduced tensors and a list of ops to perform the
reduction.
numpy_fn: A function taking two tensors and returning the reduction of the
two.
device_sets: Tuple of virtual devices to run test on.
"""
for dtype in [np.float16, np.float32, np.int32, np.int64, np.float64]:
# Create session inside outer loop to test use of
# same communicator across multiple sessions.
with self.test_session(use_gpu=True) as sess:
for devices in device_sets:
shape = (3, 4)
random = (np.random.random_sample(shape) - .5) * 1024
tensors = []
for _ in devices:
tensors.append(random.astype(dtype))
np_ans = tensors[0]
for t in tensors[1:]:
np_ans = numpy_fn(np_ans, t)
reduce_tensors = nccl_reduce(tensors, devices)
self.assertNotEmpty(reduce_tensors)
# Test shape inference.
for r in reduce_tensors:
self.assertEqual(shape, r.get_shape())
result_tensors = [array_ops.identity(t) for t in reduce_tensors]
# Check GPU availability *after* creating session, see b/68975239.
if not test.is_gpu_available():
# If no GPU is available, only test graph construction.
continue
# Test execution and results.
for t in sess.run(result_tensors):
self.assertAllClose(t, np_ans)
Example 70
| Project: FasterRCNN_TF_Py3 Author: upojzsb File: pascal_voc.py MIT License | 4 votes |
|
def _load_pascal_annotation(self, index):
"""
Load image and bounding boxes info from XML file in the PASCAL VOC
format.
"""
filename = os.path.join(self._data_path, 'Annotations', index + '.xml')
tree = ET.parse(filename)
objs = tree.findall('object')
if not self.config['use_diff']:
# Exclude the samples labeled as difficult
non_diff_objs = [
obj for obj in objs if int(obj.find('difficult').text) == 0]
# if len(non_diff_objs) != len(objs):
# print 'Removed {} difficult objects'.format(
# len(objs) - len(non_diff_objs))
objs = non_diff_objs
num_objs = len(objs)
boxes = np.zeros((num_objs, 4), dtype=np.uint16)
gt_classes = np.zeros((num_objs), dtype=np.int32)
overlaps = np.zeros((num_objs, self.num_classes), dtype=np.float32)
# "Seg" area for pascal is just the box area
seg_areas = np.zeros((num_objs), dtype=np.float32)
# Load object bounding boxes into a data frame.
for ix, obj in enumerate(objs):
bbox = obj.find('bndbox')
# Make pixel indexes 0-based
x1 = float(bbox.find('xmin').text) - 1
y1 = float(bbox.find('ymin').text) - 1
x2 = float(bbox.find('xmax').text) - 1
y2 = float(bbox.find('ymax').text) - 1
cls = self._class_to_ind[obj.find('name').text.lower().strip()]
boxes[ix, :] = [x1, y1, x2, y2]
gt_classes[ix] = cls
overlaps[ix, cls] = 1.0
seg_areas[ix] = (x2 - x1 + 1) * (y2 - y1 + 1)
overlaps = scipy.sparse.csr_matrix(overlaps)
return {'boxes': boxes,
'gt_classes': gt_classes,
'gt_overlaps': overlaps,
'flipped': False,
'seg_areas': seg_areas}
Example 71
| Project: FasterRCNN_TF_Py3 Author: upojzsb File: coco.py MIT License | 4 votes |
|
def _load_coco_annotation(self, index):
"""
Loads COCO bounding-box instance annotations. Crowd instances are
handled by marking their overlaps (with all categories) to -1. This
overlap value means that crowd "instances" are excluded from training.
"""
im_ann = self._COCO.loadImgs(index)[0]
width = im_ann['width']
height = im_ann['height']
annIds = self._COCO.getAnnIds(imgIds=index, iscrowd=None)
objs = self._COCO.loadAnns(annIds)
# Sanitize bboxes -- some are invalid
valid_objs = []
for obj in objs:
x1 = np.max((0, obj['bbox'][0]))
y1 = np.max((0, obj['bbox'][1]))
x2 = np.min((width - 1, x1 + np.max((0, obj['bbox'][2] - 1))))
y2 = np.min((height - 1, y1 + np.max((0, obj['bbox'][3] - 1))))
if obj['area'] > 0 and x2 >= x1 and y2 >= y1:
obj['clean_bbox'] = [x1, y1, x2, y2]
valid_objs.append(obj)
objs = valid_objs
num_objs = len(objs)
boxes = np.zeros((num_objs, 4), dtype=np.uint16)
gt_classes = np.zeros((num_objs), dtype=np.int32)
overlaps = np.zeros((num_objs, self.num_classes), dtype=np.float32)
seg_areas = np.zeros((num_objs), dtype=np.float32)
# Lookup table to map from COCO category ids to our internal class
# indices
coco_cat_id_to_class_ind = dict([(self._class_to_coco_cat_id[cls],
self._class_to_ind[cls])
for cls in self._classes[1:]])
for ix, obj in enumerate(objs):
cls = coco_cat_id_to_class_ind[obj['category_id']]
boxes[ix, :] = obj['clean_bbox']
gt_classes[ix] = cls
seg_areas[ix] = obj['area']
if obj['iscrowd']:
# Set overlap to -1 for all classes for crowd objects
# so they will be excluded during training
overlaps[ix, :] = -1.0
else:
overlaps[ix, cls] = 1.0
ds_utils.validate_boxes(boxes, width=width, height=height)
overlaps = scipy.sparse.csr_matrix(overlaps)
return {'width': width,
'height': height,
'boxes': boxes,
'gt_classes': gt_classes,
'gt_overlaps': overlaps,
'flipped': False,
'seg_areas': seg_areas}
Example 72
| Project: yolov3-tf2 Author: zzh8829 File: utils.py MIT License | 4 votes |
|
def load_darknet_weights(model, weights_file, tiny=False):
wf = open(weights_file, 'rb')
major, minor, revision, seen, _ = np.fromfile(wf, dtype=np.int32, count=5)
if tiny:
layers = YOLOV3_TINY_LAYER_LIST
else:
layers = YOLOV3_LAYER_LIST
for layer_name in layers:
sub_model = model.get_layer(layer_name)
for i, layer in enumerate(sub_model.layers):
if not layer.name.startswith('conv2d'):
continue
batch_norm = None
if i + 1 < len(sub_model.layers) and \
sub_model.layers[i + 1].name.startswith('batch_norm'):
batch_norm = sub_model.layers[i + 1]
logging.info("{}/{} {}".format(
sub_model.name, layer.name, 'bn' if batch_norm else 'bias'))
filters = layer.filters
size = layer.kernel_size[0]
in_dim = layer.input_shape[-1]
if batch_norm is None:
conv_bias = np.fromfile(wf, dtype=np.float32, count=filters)
else:
# darknet [beta, gamma, mean, variance]
bn_weights = np.fromfile(
wf, dtype=np.float32, count=4 * filters)
# tf [gamma, beta, mean, variance]
bn_weights = bn_weights.reshape((4, filters))[[1, 0, 2, 3]]
# darknet shape (out_dim, in_dim, height, width)
conv_shape = (filters, in_dim, size, size)
conv_weights = np.fromfile(
wf, dtype=np.float32, count=np.product(conv_shape))
# tf shape (height, width, in_dim, out_dim)
conv_weights = conv_weights.reshape(
conv_shape).transpose([2, 3, 1, 0])
if batch_norm is None:
layer.set_weights([conv_weights, conv_bias])
else:
layer.set_weights([conv_weights])
batch_norm.set_weights(bn_weights)
assert len(wf.read()) == 0, 'failed to read all data'
wf.close()
Example 73
| Project: Tacotron Author: ElwynWang File: data.py GNU General Public License v3.0 | 4 votes |
|
def input_fn(mode):
data_dir = Hp.train_data_dir if mode == "train" else Hp.eval_data_dir
batch_size = Hp.train_batch_size if mode == "train" else Hp.eval_batch_size
char2idx, idx2char = load_vocab()
lines = codecs.open(os.path.join(data_dir, 'metadata.csv'), 'r', 'utf-8').readlines()
transcripts = []
ref_paths = []
for line in lines:
fname, _, transcript = line.strip().split('|')
ref_path = os.path.join(data_dir, 'wavs', fname+'.wav')
ref_paths.append(ref_path)
transcript = text_normalize(transcript)+u"␃" # EOS
transcript = [char2idx[char] for char in transcript]
transcripts.append(np.array(transcript, np.int32).tostring())
transcripts = tf.convert_to_tensor(transcripts)
ref_paths = tf.convert_to_tensor(ref_paths)
dataset = tf.data.Dataset.from_tensor_slices((transcripts,ref_paths))
if mode == "train":
dataset = dataset.shuffle(buffer_size = 100000)
dataset = dataset.repeat()
dataset = dataset.map(parser, num_parallel_calls = Hp.data_num_parallel_calls)
#dataset = dataset.batch(batch_size)
dataset = dataset.padded_batch(batch_size = batch_size, padded_shapes=(
[Hp.num_charac], [None, Hp.num_mels*Hp.reduction_factor], [1], [1],
[None, Hp.num_mels*Hp.reduction_factor], [None, Hp.num_fft//2+1]))
dataset = dataset.prefetch(1)
iterator = dataset.make_one_shot_iterator() #one_shot_iterator can automatically initialize
#return iterator
batch_inputs = iterator.get_next()
names = ['transcript', 'reference', 'ref_len', 'speaker', 'decoder', 'labels']
batch_inputs = {name:inp for name,inp in zip(names, batch_inputs)}
return batch_inputs
Example 74
| Project: dc_tts Author: Kyubyong File: synthesize.py Apache License 2.0 | 4 votes |
|
def synthesize():
# Load data
L = load_data("synthesize")
# Load graph
g = Graph(mode="synthesize"); print("Graph loaded")
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
# Restore parameters
var_list = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, 'Text2Mel')
saver1 = tf.train.Saver(var_list=var_list)
saver1.restore(sess, tf.train.latest_checkpoint(hp.logdir + "-1"))
print("Text2Mel Restored!")
var_list = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, 'SSRN') + \
tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, 'gs')
saver2 = tf.train.Saver(var_list=var_list)
saver2.restore(sess, tf.train.latest_checkpoint(hp.logdir + "-2"))
print("SSRN Restored!")
# Feed Forward
## mel
Y = np.zeros((len(L), hp.max_T, hp.n_mels), np.float32)
prev_max_attentions = np.zeros((len(L),), np.int32)
for j in tqdm(range(hp.max_T)):
_gs, _Y, _max_attentions, _alignments = \
sess.run([g.global_step, g.Y, g.max_attentions, g.alignments],
{g.L: L,
g.mels: Y,
g.prev_max_attentions: prev_max_attentions})
Y[:, j, :] = _Y[:, j, :]
prev_max_attentions = _max_attentions[:, j]
# Get magnitude
Z = sess.run(g.Z, {g.Y: Y})
# Generate wav files
if not os.path.exists(hp.sampledir): os.makedirs(hp.sampledir)
for i, mag in enumerate(Z):
print("Working on file", i+1)
wav = spectrogram2wav(mag)
write(hp.sampledir + "/{}.wav".format(i+1), hp.sr, wav)
Example 75
| Project: dc_tts Author: Kyubyong File: data_load.py Apache License 2.0 | 4 votes |
|
def load_data(mode="train"):
'''Loads data
Args:
mode: "train" or "synthesize".
'''
# Load vocabulary
char2idx, idx2char = load_vocab()
if mode=="train":
if "LJ" in hp.data:
# Parse
fpaths, text_lengths, texts = [], [], []
transcript = os.path.join(hp.data, 'transcript.csv')
lines = codecs.open(transcript, 'r', 'utf-8').readlines()
for line in lines:
fname, _, text = line.strip().split("|")
fpath = os.path.join(hp.data, "wavs", fname + ".wav")
fpaths.append(fpath)
text = text_normalize(text) + "E" # E: EOS
text = [char2idx[char] for char in text]
text_lengths.append(len(text))
texts.append(np.array(text, np.int32).tostring())
return fpaths, text_lengths, texts
else: # nick or kate
# Parse
fpaths, text_lengths, texts = [], [], []
transcript = os.path.join(hp.data, 'transcript.csv')
lines = codecs.open(transcript, 'r', 'utf-8').readlines()
for line in lines:
fname, _, text, is_inside_quotes, duration = line.strip().split("|")
duration = float(duration)
if duration > 10. : continue
fpath = os.path.join(hp.data, fname)
fpaths.append(fpath)
text += "E" # E: EOS
text = [char2idx[char] for char in text]
text_lengths.append(len(text))
texts.append(np.array(text, np.int32).tostring())
return fpaths, text_lengths, texts
else: # synthesize on unseen test text.
# Parse
lines = codecs.open(hp.test_data, 'r', 'utf-8').readlines()[1:]
sents = [text_normalize(line.split(" ", 1)[-1]).strip() + "E" for line in lines] # text normalization, E: EOS
texts = np.zeros((len(sents), hp.max_N), np.int32)
for i, sent in enumerate(sents):
texts[i, :len(sent)] = [char2idx[char] for char in sent]
return texts
Example 76
| Project: dc_tts Author: Kyubyong File: data_load.py Apache License 2.0 | 4 votes |
|
def get_batch():
"""Loads training data and put them in queues"""
with tf.device('/cpu:0'):
# Load data
fpaths, text_lengths, texts = load_data() # list
maxlen, minlen = max(text_lengths), min(text_lengths)
# Calc total batch count
num_batch = len(fpaths) // hp.B
# Create Queues
fpath, text_length, text = tf.train.slice_input_producer([fpaths, text_lengths, texts], shuffle=True)
# Parse
text = tf.decode_raw(text, tf.int32) # (None,)
if hp.prepro:
def _load_spectrograms(fpath):
fname = os.path.basename(fpath)
mel = "mels/{}".format(fname.replace("wav", "npy"))
mag = "mags/{}".format(fname.replace("wav", "npy"))
return fname, np.load(mel), np.load(mag)
fname, mel, mag = tf.py_func(_load_spectrograms, [fpath], [tf.string, tf.float32, tf.float32])
else:
fname, mel, mag = tf.py_func(load_spectrograms, [fpath], [tf.string, tf.float32, tf.float32]) # (None, n_mels)
# Add shape information
fname.set_shape(())
text.set_shape((None,))
mel.set_shape((None, hp.n_mels))
mag.set_shape((None, hp.n_fft//2+1))
# Batching
_, (texts, mels, mags, fnames) = tf.contrib.training.bucket_by_sequence_length(
input_length=text_length,
tensors=[text, mel, mag, fname],
batch_size=hp.B,
bucket_boundaries=[i for i in range(minlen + 1, maxlen - 1, 20)],
num_threads=8,
capacity=hp.B*4,
dynamic_pad=True)
return texts, mels, mags, fnames, num_batch
Example 77
| Project: mmdetection Author: open-mmlab File: inference.py Apache License 2.0 | 4 votes |
|
def show_result(img,
result,
class_names,
score_thr=0.3,
wait_time=0,
show=True,
out_file=None):
"""Visualize the detection results on the image.
Args:
img (str or np.ndarray): Image filename or loaded image.
result (tuple[list] or list): The detection result, can be either
(bbox, segm) or just bbox.
class_names (list[str] or tuple[str]): A list of class names.
score_thr (float): The threshold to visualize the bboxes and masks.
wait_time (int): Value of waitKey param.
show (bool, optional): Whether to show the image with opencv or not.
out_file (str, optional): If specified, the visualization result will
be written to the out file instead of shown in a window.
Returns:
np.ndarray or None: If neither `show` nor `out_file` is specified, the
visualized image is returned, otherwise None is returned.
"""
assert isinstance(class_names, (tuple, list))
img = mmcv.imread(img)
img = img.copy()
if isinstance(result, tuple):
bbox_result, segm_result = result
else:
bbox_result, segm_result = result, None
bboxes = np.vstack(bbox_result)
# draw segmentation masks
if segm_result is not None:
segms = mmcv.concat_list(segm_result)
inds = np.where(bboxes[:, -1] > score_thr)[0]
for i in inds:
color_mask = np.random.randint(0, 256, (1, 3), dtype=np.uint8)
mask = maskUtils.decode(segms[i]).astype(np.bool)
img[mask] = img[mask] * 0.5 + color_mask * 0.5
# draw bounding boxes
labels = [
np.full(bbox.shape[0], i, dtype=np.int32)
for i, bbox in enumerate(bbox_result)
]
labels = np.concatenate(labels)
mmcv.imshow_det_bboxes(
img,
bboxes,
labels,
class_names=class_names,
score_thr=score_thr,
show=show,
wait_time=wait_time,
out_file=out_file)
if not (show or out_file):
return img
Example 78
| Project: mmdetection Author: open-mmlab File: fcn_mask_head.py Apache License 2.0 | 4 votes |
|
def get_seg_masks(self, mask_pred, det_bboxes, det_labels, rcnn_test_cfg,
ori_shape, scale_factor, rescale):
"""Get segmentation masks from mask_pred and bboxes.
Args:
mask_pred (Tensor or ndarray): shape (n, #class+1, h, w).
For single-scale testing, mask_pred is the direct output of
model, whose type is Tensor, while for multi-scale testing,
it will be converted to numpy array outside of this method.
det_bboxes (Tensor): shape (n, 4/5)
det_labels (Tensor): shape (n, )
img_shape (Tensor): shape (3, )
rcnn_test_cfg (dict): rcnn testing config
ori_shape: original image size
Returns:
list[list]: encoded masks
"""
if isinstance(mask_pred, torch.Tensor):
mask_pred = mask_pred.sigmoid().cpu().numpy()
assert isinstance(mask_pred, np.ndarray)
# when enabling mixed precision training, mask_pred may be float16
# numpy array
mask_pred = mask_pred.astype(np.float32)
cls_segms = [[] for _ in range(self.num_classes - 1)]
bboxes = det_bboxes.cpu().numpy()[:, :4]
labels = det_labels.cpu().numpy() + 1
if rescale:
img_h, img_w = ori_shape[:2]
else:
img_h = np.round(ori_shape[0] * scale_factor).astype(np.int32)
img_w = np.round(ori_shape[1] * scale_factor).astype(np.int32)
scale_factor = 1.0
for i in range(bboxes.shape[0]):
if not isinstance(scale_factor, (float, np.ndarray)):
scale_factor = scale_factor.cpu().numpy()
bbox = (bboxes[i, :] / scale_factor).astype(np.int32)
label = labels[i]
w = max(bbox[2] - bbox[0] + 1, 1)
h = max(bbox[3] - bbox[1] + 1, 1)
if not self.class_agnostic:
mask_pred_ = mask_pred[i, label, :, :]
else:
mask_pred_ = mask_pred[i, 0, :, :]
im_mask = np.zeros((img_h, img_w), dtype=np.uint8)
bbox_mask = mmcv.imresize(mask_pred_, (w, h))
bbox_mask = (bbox_mask > rcnn_test_cfg.mask_thr_binary).astype(
np.uint8)
im_mask[bbox[1]:bbox[1] + h, bbox[0]:bbox[0] + w] = bbox_mask
rle = mask_util.encode(
np.array(im_mask[:, :, np.newaxis], order='F'))[0]
cls_segms[label - 1].append(rle)
return cls_segms
Example 79
| Project: mmdetection Author: open-mmlab File: base.py Apache License 2.0 | 4 votes |
|
def show_result(self, data, result, dataset=None, score_thr=0.3):
if isinstance(result, tuple):
bbox_result, segm_result = result
else:
bbox_result, segm_result = result, None
img_tensor = data['img'][0]
img_metas = data['img_meta'][0].data[0]
imgs = tensor2imgs(img_tensor, **img_metas[0]['img_norm_cfg'])
assert len(imgs) == len(img_metas)
if dataset is None:
class_names = self.CLASSES
elif isinstance(dataset, str):
class_names = get_classes(dataset)
elif isinstance(dataset, (list, tuple)):
class_names = dataset
else:
raise TypeError(
'dataset must be a valid dataset name or a sequence'
' of class names, not {}'.format(type(dataset)))
for img, img_meta in zip(imgs, img_metas):
h, w, _ = img_meta['img_shape']
img_show = img[:h, :w, :]
bboxes = np.vstack(bbox_result)
# draw segmentation masks
if segm_result is not None:
segms = mmcv.concat_list(segm_result)
inds = np.where(bboxes[:, -1] > score_thr)[0]
for i in inds:
color_mask = np.random.randint(
0, 256, (1, 3), dtype=np.uint8)
mask = maskUtils.decode(segms[i]).astype(np.bool)
img_show[mask] = img_show[mask] * 0.5 + color_mask * 0.5
# draw bounding boxes
labels = [
np.full(bbox.shape[0], i, dtype=np.int32)
for i, bbox in enumerate(bbox_result)
]
labels = np.concatenate(labels)
mmcv.imshow_det_bboxes(
img_show,
bboxes,
labels,
class_names=class_names,
score_thr=score_thr)
Example 80
| Project: speed_estimation Author: NeilNie File: helper.py MIT License | 3 votes |
|
def optical_flow(previous, current):
gray1 = cv2.cvtColor(previous, cv2.COLOR_RGB2GRAY)
gray2 = cv2.cvtColor(current, cv2.COLOR_RGB2GRAY)
flow = cv2.calcOpticalFlowFarneback(gray1, gray2, None, 0.5, 3, 15, 3, 5, 1.5, 0)
return flow
#
# def comma_validation_generator(data, batch_size):
#
# """
# Generate training images given image paths and associated steering angles
#
# :param data : (numpy.array) the loaded data (converted to list from pandas format)
# :param batch_size : (int) batch size for training
#
# :rtype: Iterator[images, angles] images for training
# the corresponding steering angles
#
# """
#
# images = np.empty([batch_size, configs.LENGTH, configs.IMG_HEIGHT, configs.IMG_WIDTH, configs.CHANNELS], dtype=np.int32)
# labels = np.empty([batch_size])
#
# while True:
#
# c = 0
#
# for index in np.random.permutation(data.shape[0]):
#
# imgs = np.empty([configs.LENGTH, configs.IMG_HEIGHT, configs.IMG_WIDTH, configs.CHANNELS], dtype=np.int32)
#
# if index < configs.LENGTH:
# start = 0
# end = configs.LENGTH
# elif index + configs.LENGTH >= len(data):
# start = len(data) - configs.LENGTH - 1
# end = len(data) - 1
# else:
# start = index
# end = index + configs.LENGTH
#
# for i in range(start, end):
# center_path = "/home/neil/dataset/speedchallenge/data/train/" + str(data[i][1])
# image = load_image(center_path)
# imgs[i - start] = image
#
# images[c] = imgs
# labels[c] = data[end][2]
#
# c += 1
#
# if c == batch_size:
# break
#
# yield images, labels
