Python torch.utils.data.size() Examples
The following are 30
code examples of torch.utils.data.size().
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example.
You may also want to check out all available functions/classes of the module
torch.utils.data
, or try the search function
.
Example #1
Source File: main.py From examples with BSD 3-Clause "New" or "Revised" License | 6 votes |
def test(epoch): model.eval() test_loss = 0 with torch.no_grad(): for i, (data, _) in enumerate(test_loader): data = data.to(device) recon_batch, mu, logvar = model(data) test_loss += loss_function(recon_batch, data, mu, logvar).item() if i == 0: n = min(data.size(0), 8) comparison = torch.cat([data[:n], recon_batch.view(args.batch_size, 1, 28, 28)[:n]]) save_image(comparison.cpu(), 'results/reconstruction_' + str(epoch) + '.png', nrow=n) test_loss /= len(test_loader.dataset) print('====> Test set loss: {:.4f}'.format(test_loss))
Example #2
Source File: test_vae_pytorch_example.py From UnsupervisedDeepLearning-Pytorch with MIT License | 6 votes |
def test(epoch): model.eval() test_loss = 0 for i, (data, _) in enumerate(test_loader): if args.cuda: data = data.cuda() data = Variable(data, volatile=True) recon_batch, mu, logvar = model(data) test_loss += loss_function(recon_batch, data, mu, logvar).data[0] if i == 0: n = min(data.size(0), 8) comparison = torch.cat([data[:n], recon_batch.view(args.batch_size, 1, 28, 28)[:n]]) save_image(comparison.data.cpu(), 'results/reconstruction_' + str(epoch) + '.png', nrow=n) test_loss /= len(test_loader.dataset) print('====> Test set loss: {:.4f}'.format(test_loss))
Example #3
Source File: main.py From pytorch-distributed-example with MIT License | 6 votes |
def evaluate(self): self.model.eval() test_loss = Average() test_acc = Accuracy() for data, target in self.test_loader: data = data.to(self.device) target = target.to(self.device) output = self.model(data) loss = F.cross_entropy(output, target) test_loss.update(loss.item(), data.size(0)) test_acc.update(output, target) return test_loss, test_acc
Example #4
Source File: main.py From pytorch-distributed-example with MIT License | 6 votes |
def train(self): self.model.train() train_loss = Average() train_acc = Accuracy() for data, target in self.train_loader: data = data.to(self.device) target = target.to(self.device) output = self.model(data) loss = F.cross_entropy(output, target) self.optimizer.zero_grad() loss.backward() self.optimizer.step() train_loss.update(loss.item(), data.size(0)) train_acc.update(output, target) return train_loss, train_acc
Example #5
Source File: main.py From PyTorch with MIT License | 6 votes |
def test(epoch): model.eval() test_loss = 0 with torch.no_grad(): for i, (data, _) in enumerate(test_loader): data = data.to(device) recon_batch, mu, logvar = model(data) test_loss += loss_function(recon_batch, data, mu, logvar).item() if i == 0: n = min(data.size(0), 8) comparison = torch.cat([data[:n], recon_batch.view(args.batch_size, 1, 28, 28)[:n]]) save_image(comparison.cpu(), 'results/reconstruction_' + str(epoch) + '.png', nrow=n) test_loss /= len(test_loader.dataset) print('====> Test set loss: {:.4f}'.format(test_loss))
Example #6
Source File: main.py From ArtificialIntelligenceEngines with MIT License | 6 votes |
def test(epoch): model.eval() test_loss = 0 with torch.no_grad(): # each data is of BATCH_SIZE (default 128) samples for i, (data, _) in enumerate(test_loader): data = data.to(device) recon_batch, mu, logvar = model(data) test_loss += loss_function(recon_batch, data, mu, logvar).item() if i == 0: n = min(data.size(0), 8) # for the first 128 batch of the epoch, show the first 8 input digits # with right below them the reconstructed output digits comparison = torch.cat([data[:n], recon_batch.view(args.batch_size, 1, 28, 28)[:n]]) save_image(comparison.cpu(), 'results/reconstruction_' + str(epoch) + '.png', nrow=n) test_loss /= len(test_loader.dataset) print('====> Test set loss: {:.4f}'.format(test_loss)) ########## create VAE ##########
Example #7
Source File: roibatchLoader.py From OICR-pytorch with MIT License | 6 votes |
def __getitem__(self, index): # get the anchor index for current sample index # here we set the anchor index to the last one # sample in this group minibatch_db = [self._roidb[index]] # [self._roidb[index_ratio]] blobs = get_minibatch(minibatch_db, self._num_classes) np.random.shuffle(blobs['rois']) rois = torch.from_numpy(blobs['rois'][:self.max_rois_size]) data = torch.from_numpy(blobs['data']) labels = torch.from_numpy(blobs['labels']) data_height, data_width = data.size(1), data.size(2) data = data.permute(0, 3, 1, 2).contiguous().view(3, data_height, data_width) info = torch.Tensor([rois.size(0), data_height, data_width]) return data, rois, labels, info
Example #8
Source File: ctgan.py From SDGym with MIT License | 6 votes |
def calc_gradient_penalty(netD, real_data, fake_data, device='cpu', pac=10, lambda_=10): alpha = torch.rand(real_data.size(0) // pac, 1, 1, device=device) alpha = alpha.repeat(1, pac, real_data.size(1)) alpha = alpha.view(-1, real_data.size(1)) interpolates = alpha * real_data + ((1 - alpha) * fake_data) # interpolates = torch.Variable(interpolates, requires_grad=True, device=device) disc_interpolates = netD(interpolates) gradients = torch.autograd.grad( outputs=disc_interpolates, inputs=interpolates, grad_outputs=torch.ones(disc_interpolates.size(), device=device), create_graph=True, retain_graph=True, only_inputs=True)[0] gradient_penalty = ( (gradients.view(-1, pac * real_data.size(1)).norm(2, dim=1) - 1) ** 2).mean() * lambda_ return gradient_penalty
Example #9
Source File: loader.py From nni with MIT License | 5 votes |
def test_train_loader(): train_loader, val_loader = get_train_loaders(1, batch_size=4, dev_mode=False, pad_mode='edge', meta_version=2, pseudo_label=True) print(train_loader.num, val_loader.num) for i, data in enumerate(train_loader): imgs, masks, salt_exists = data #pdb.set_trace() print(imgs.size(), masks.size(), salt_exists.size()) print(salt_exists) add_depth_channel(imgs, 'resize') print(masks) break #print(imgs) #print(masks)
Example #10
Source File: test_demo.py From PartNet with MIT License | 5 votes |
def writeply(savedir, data, label): path = os.path.dirname(savedir) if not os.path.exists(path): os.makedirs(path) if data.size(0) == 0: n_vertex = 0 else: n_vertex = data.size(1) with open(savedir, 'w') as f: f.write('ply\n') f.write('format ascii 1.0\n') f.write('comment 111231\n') f.write('element vertex %d\n' % n_vertex) f.write('property float x\n') f.write('property float y\n') f.write('property float z\n') f.write('property float nx\n') f.write('property float ny\n') f.write('property float nz\n') f.write('property uchar red\n') f.write('property uchar green\n') f.write('property uchar blue\n') f.write('property uchar label\n') f.write('end_header\n') for j in range(n_vertex): f.write('%g %g %g %g %g %g %d %d %d %d\n' % (*data[0, j], *color[label[j]], label[j]))
Example #11
Source File: loader.py From nni with MIT License | 5 votes |
def test_test_loader(): test_loader = get_test_loader(4, pad_mode='resize') print(test_loader.num) for i, data in enumerate(test_loader): print(data.size()) if i > 5: break
Example #12
Source File: batchLoader.py From Iterative-Visual-Reasoning.pytorch with MIT License | 5 votes |
def get_one_sample(self, index): num_images = 1 # Sample random scales to use for each image in this batch scales = (600,) max_scale = 600 random_scale_inds = npr.randint(0, high=len(scales), size=num_images) # Get the input image blob im_blob, im_scales = self.get_image_blob(index, random_scale_inds, scales, max_scale) blobs = {'data': im_blob} # gt boxes: (x1, y1, x2, y2, cls) gt_inds = np.where(self.roidb[index[0]]['gt_classes'] != 0)[0] gt_boxes = np.empty((len(gt_inds), 5), dtype=np.float32) gt_boxes[:, 0:4] = self.roidb[index[0]]['boxes'][gt_inds, :] * im_scales[0] gt_boxes[:, 4] = self.roidb[index[0]]['gt_classes'][gt_inds] blobs['gt_boxes'] = gt_boxes # height, width, scale blobs['im_info'] = np.array([im_blob.shape[1], im_blob.shape[2], im_scales[0]], dtype=np.float32) if self.args.with_global: blobs['memory_size'] = np.ceil(blobs['im_info'][:2] / self.args.BOTTLE_SCALE / 2.).astype(np.int32) # conv5 else: blobs['memory_size'] = np.ceil(blobs['im_info'][:2] / self.args.BOTTLE_SCALE).astype(np.int32) # conv4 blobs['num_gt'] = np.int32(gt_boxes.shape[0]) blobs['img'] = self.roidb[index[0]]['image'] blobs['index'] = index[0] return blobs
Example #13
Source File: batchLoader.py From Iterative-Visual-Reasoning.pytorch with MIT License | 5 votes |
def __getitem__(self, index): indexes = [index] blobs = self.get_one_sample(indexes) data = torch.from_numpy(blobs['data']) im_info = torch.from_numpy(blobs['im_info']) mem_size = torch.from_numpy(blobs['memory_size']) # we need to random shuffle the bounding box. data_height, data_width = data.size(1), data.size(2) if self.phase == 'train': # if the number of region is greater than 100 then random pick 100 regions # this opt can make the used memory of GPUs more stable. # only for train and val phase np.random.shuffle(blobs['gt_boxes']) if blobs['gt_boxes'].shape[0] > 100: print('sampling regions from %d to %d' % (blobs['gt_boxes'].shape[0], 100)) blobs['gt_boxes'] = blobs['gt_boxes'][:100] elif self.phase == 'eval': # np.random.shuffle(blobs['gt_boxes']) if blobs['gt_boxes'].shape[0] > 100: print('sampling regions from %d to %d' % (blobs['gt_boxes'].shape[0], 100)) blobs['gt_boxes'] = blobs['gt_boxes'][:100] else: pass # if self.args.with_global: # Arr_ = self._get_adjmat_Arr(blobs['gt_boxes']) # 5*r*r # Arr = torch.from_numpy(Arr_) # else: # Arr = 0. gt_boxes = torch.from_numpy(blobs['gt_boxes']) # permute trim_data to adapt to downstream processing data = data.permute(0, 3, 1, 2).contiguous().view(3, data_height, data_width) im_info = im_info.view(3) return data, im_info, gt_boxes, mem_size, blobs['data'], blobs['gt_boxes']
Example #14
Source File: ctgan.py From SDGym with MIT License | 5 votes |
def forward(self, input): assert input.size()[0] % self.pack == 0 return self.seq(input.view(-1, self.packdim))
Example #15
Source File: ctgan.py From SDGym with MIT License | 5 votes |
def cond_loss(data, output_info, c, m): loss = [] st = 0 st_c = 0 skip = False for item in output_info: if item[1] == 'tanh': st += item[0] skip = True elif item[1] == 'softmax': if skip: skip = False st += item[0] continue ed = st + item[0] ed_c = st_c + item[0] tmp = F.cross_entropy( data[:, st:ed], torch.argmax(c[:, st_c:ed_c], dim=1), reduction='none' ) loss.append(tmp) st = ed st_c = ed_c else: assert 0 loss = torch.stack(loss, dim=1) return (loss * m).sum() / data.size()[0]
Example #16
Source File: io.py From transfer with MIT License | 5 votes |
def make_batch(emblayer, sequences, oov='<oov>'): # sequences = pad(sequences, pad_left = pad_left) batch_size = len(sequences) length = len(sequences[0]) word2id, oovid = emblayer.word2id, emblayer.oovid data = torch.LongTensor([ word2id.get(w, word2id.get(w.lower(), oovid)) \ for seq in sequences for w in seq ]) assert data.size(0) == batch_size * length return data.view(batch_size, length).t().contiguous() # length * batch_size
Example #17
Source File: experiment_coding.py From integer_discrete_flows with MIT License | 5 votes |
def encode_images(img, model, decode): batchsize, img_c, img_h, img_w = img.size() c, h, w = model.args.input_size assert img_h == img_w and h == w if img_h != h: assert img_h % h == 0 steps = img_h // h states = [[] for i in range(batchsize)] state_sizes = [0 for i in range(batchsize)] bpd = [0 for i in range(batchsize)] error = 0 for j in range(steps): for i in range(steps): r = encode_patches( img[:, :, j*h:(j+1)*h, i*w:(i+1)*w], model, decode) for b in range(batchsize): if r[0][b] is None: states[b].append(None) else: states[b].extend(r[0][b]) state_sizes[b] += r[1][b] bpd[b] += r[2][b] / steps**2 error += r[3] return states, state_sizes, bpd, error else: return encode_patches(img, model, decode)
Example #18
Source File: experiment_coding.py From integer_discrete_flows with MIT License | 5 votes |
def encode_patches(imgs, model, decode): batchsize, img_c, img_h, img_w = imgs.size() c, h, w = model.args.input_size assert img_h == h and img_w == w states = model.encode(imgs) bpd = model.forward(imgs)[1].cpu().numpy() state_sizes = [] error = 0 for b in range(batchsize): if states[b] is None: # Using escape bit ;) state_sizes += [8 * img_c * img_h * img_w + 1] # Error remains unchanged. print('Escaping, not encoding.') else: if decode: x_recon = model.decode([states[b]]) error += torch.sum( torch.abs(x_recon.int() - imgs[b].int())).item() # Append state plus an escape bit state_sizes += [32 * len(states[b]) + 1] return states, state_sizes, bpd, error
Example #19
Source File: sequential_mnist.py From stable-nalu with MIT License | 5 votes |
def __getitem__(self, index): mnist_images = [] mnist_targets = [] for mnist_index in range(index * self._seq_length, (index + 1) * self._seq_length): image, target = self._dataset[self._index_mapping[mnist_index]] mnist_images.append(image) # image.size() = [1, 28, 28] mnist_targets.append(target) data = torch.stack(mnist_images) # data.size() = [seq_length, 1, 28, 28] target = self._operation(np.stack(mnist_targets)) return ( data, torch.tensor(target, dtype=torch.float32) )
Example #20
Source File: sequential_svhn.py From stable-nalu with MIT License | 5 votes |
def __getitem__(self, index): svhn_images = [] svhn_targets = [] for svhn_index in range(index * self._seq_length, (index + 1) * self._seq_length): image, target = self._dataset[self._index_mapping[svhn_index]] svhn_images.append(image) # image.size() = [3, 32, 32] svhn_targets.append(target) data = torch.stack(svhn_images) # data.size() = [seq_length, 3, 32, 32] target = self._operation(np.stack(svhn_targets)) return ( data, torch.tensor(target, dtype=torch.float32) )
Example #21
Source File: sr_dataset.py From pykaldi2 with MIT License | 5 votes |
def _utt2seg(data, seg_len, seg_shift): """ Cut an utterance (MxN matrix) to segments. """ if data.ndim == 1: data = np.reshape(data, (1, data.size)) dim, n_fr = data.shape n_seg = int(np.floor((n_fr - seg_len) / seg_shift)) + 1 seg = [] for i in range(n_seg): start = i * seg_shift stop = start + seg_len seg.append(data[:, start:stop]) return seg
Example #22
Source File: main.py From pytorch-distributed-example with MIT License | 5 votes |
def update(self, output, target): pred = output.argmax(dim=1) correct = pred.eq(target).sum().item() self.correct += correct self.count += output.size(0)
Example #23
Source File: main.py From pytorch-distributed-example with MIT License | 5 votes |
def forward(self, x): return self.fc(x.view(x.size(0), -1))
Example #24
Source File: main.py From pytorch-distributed-example with MIT License | 5 votes |
def main(): parser = argparse.ArgumentParser() parser.add_argument('--backend', type=str, default='gloo', help='Name of the backend to use.') parser.add_argument('-i', '--init-method', type=str, default='tcp://127.0.0.1:23456', help='URL specifying how to initialize the package.') parser.add_argument('-s', '--world-size', type=int, default=1, help='Number of processes participating in the job.') parser.add_argument('-r', '--rank', type=int, default=0, help='Rank of the current process.') parser.add_argument('--epochs', type=int, default=20) parser.add_argument('--no-cuda', action='store_true') parser.add_argument('-lr', '--learning-rate', type=float, default=1e-3) parser.add_argument('--root', type=str, default='data') parser.add_argument('--batch-size', type=int, default=128) args = parser.parse_args() print(args) if args.world_size > 1: distributed.init_process_group( backend=args.backend, init_method=args.init_method, world_size=args.world_size, rank=args.rank, ) run(args)
Example #25
Source File: wavenet_data.py From HandsOnDeepLearningWithPytorch with MIT License | 5 votes |
def one_hot_encode(data, channels=256): one_hot = np.zeros((data.size, channels), dtype=float) one_hot[np.arange(data.size), data.ravel()] = 1 return one_hot
Example #26
Source File: wavenet_data.py From HandsOnDeepLearningWithPytorch with MIT License | 5 votes |
def __init__(self, data_dir, receptive_fields, sample_size=0, sample_rate=16000, in_channels=256, batch_size=1, shuffle=True): """ DataLoader for WaveNet :param data_dir: :param receptive_fields: integer. size(length) of receptive fields :param sample_size: integer. number of timesteps to train at once. sample size has to be bigger than receptive fields. |-- receptive field --|---------------------| |------- samples -------------------| |---------------------|-- outputs --| :param sample_rate: sound sampling rates :param in_channels: number of input channels :param batch_size: :param shuffle: """ dataset = Dataset(data_dir, sample_rate, in_channels) super(DataLoader, self).__init__(dataset, batch_size, shuffle) if sample_size <= receptive_fields: raise Exception("sample_size has to be bigger than receptive_fields") self.sample_size = sample_size self.receptive_fields = receptive_fields self.collate_fn = self._collate_fn
Example #27
Source File: main.py From VQ-VAE with BSD 3-Clause "New" or "Revised" License | 5 votes |
def save_reconstructed_images(data, epoch, outputs, save_path, name): size = data.size() n = min(data.size(0), 8) batch_size = data.size(0) comparison = torch.cat([data[:n], outputs.view(batch_size, size[1], size[2], size[3])[:n]]) save_image(comparison.cpu(), os.path.join(save_path, name + '_' + str(epoch) + '.png'), nrow=n, normalize=True)
Example #28
Source File: train_OriNet_test_on_graffity.py From affnet with MIT License | 5 votes |
def extract_random_LAF(data, max_rot = math.pi, max_tilt = 1.0, crop_size = 32): st = int((data.size(2) - crop_size)/2) fin = st + crop_size if type(max_rot) is float: rot_LAFs, inv_rotmat = get_random_rotation_LAFs(data, max_rot) else: rot_LAFs = max_rot inv_rotmat = None aff_LAFs, inv_TA = get_random_norm_affine_LAFs(data, max_tilt); aff_LAFs[:,0:2,0:2] = torch.bmm(rot_LAFs[:,0:2,0:2],aff_LAFs[:,0:2,0:2]) data_aff = extract_patches(data, aff_LAFs, PS = data.size(2)) data_affcrop = data_aff[:,:, st:fin, st:fin].contiguous() return data_affcrop, data_aff, rot_LAFs,inv_rotmat,inv_TA
Example #29
Source File: main.py From pytorch-flows with MIT License | 5 votes |
def validate(epoch, model, loader, prefix='Validation'): global global_step, writer model.eval() val_loss = 0 pbar = tqdm(total=len(loader.dataset)) pbar.set_description('Eval') for batch_idx, data in enumerate(loader): if isinstance(data, list): if len(data) > 1: cond_data = data[1].float() cond_data = cond_data.to(device) else: cond_data = None data = data[0] data = data.to(device) with torch.no_grad(): val_loss += -model.log_probs(data, cond_data).sum().item() # sum up batch loss pbar.update(data.size(0)) pbar.set_description('Val, Log likelihood in nats: {:.6f}'.format( -val_loss / pbar.n)) writer.add_scalar('validation/LL', val_loss / len(loader.dataset), epoch) pbar.close() return val_loss / len(loader.dataset)
Example #30
Source File: amazon.py From Point-Then-Operate with Apache License 2.0 | 5 votes |
def __init__(self, mode, noisy_for_train, sentiment, direction): self.mode = mode self.root = os.path.join('../data', 'yelp') self.noisy = self.mode == 'train' and noisy_for_train # Load data from domain 0 and domain 1. path = os.path.join(self.root, 'sentiment.{}.{}'.format(mode, sentiment)) # Load vocabulary. print('----- Loading vocab -----') self.vocab = Vocabulary('../data/amazon/amazon.vocab') print('vocabulary size:', self.vocab.size) self.pad = self.vocab.word2id['<pad>'] self.go = self.vocab.word2id['<go>'] self.eos = self.vocab.word2id['<eos>'] self.unk = self.vocab.word2id['<unk>'] # Tokenize file content with open(path, 'r') as f: ids = [] for line in f: words = ['<go>'] + line.split() + ['<eos>'] if direction == 'forward': pass elif direction == 'backward': words.reverse() else: raise ValueError() for word in words: ids.append(self.vocab.word2id[word] if word in self.vocab.word2id else self.unk) self.ids = torch.LongTensor(ids) # (very_long, ) self.ids = batchify(self.ids, config.batch_size, config) # shape = (???, batch_size)