Python torch.utils.data.size() Examples

The following are 30 code examples of torch.utils.data.size(). You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may also want to check out all available functions/classes of the module torch.utils.data , or try the search function .
Example #1
Source File: main.py    From examples with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
def test(epoch):
    model.eval()
    test_loss = 0
    with torch.no_grad():
        for i, (data, _) in enumerate(test_loader):
            data = data.to(device)
            recon_batch, mu, logvar = model(data)
            test_loss += loss_function(recon_batch, data, mu, logvar).item()
            if i == 0:
                n = min(data.size(0), 8)
                comparison = torch.cat([data[:n],
                                      recon_batch.view(args.batch_size, 1, 28, 28)[:n]])
                save_image(comparison.cpu(),
                         'results/reconstruction_' + str(epoch) + '.png', nrow=n)

    test_loss /= len(test_loader.dataset)
    print('====> Test set loss: {:.4f}'.format(test_loss)) 
Example #2
Source File: test_vae_pytorch_example.py    From UnsupervisedDeepLearning-Pytorch with MIT License 6 votes vote down vote up
def test(epoch):
    model.eval()
    test_loss = 0
    for i, (data, _) in enumerate(test_loader):
        if args.cuda:
            data = data.cuda()
        data = Variable(data, volatile=True)
        recon_batch, mu, logvar = model(data)
        test_loss += loss_function(recon_batch, data, mu, logvar).data[0]
        if i == 0:
          n = min(data.size(0), 8)
          comparison = torch.cat([data[:n],
                                  recon_batch.view(args.batch_size, 1, 28, 28)[:n]])
          save_image(comparison.data.cpu(),
                     'results/reconstruction_' + str(epoch) + '.png', nrow=n)

    test_loss /= len(test_loader.dataset)
    print('====> Test set loss: {:.4f}'.format(test_loss)) 
Example #3
Source File: main.py    From pytorch-distributed-example with MIT License 6 votes vote down vote up
def evaluate(self):
        self.model.eval()

        test_loss = Average()
        test_acc = Accuracy()

        for data, target in self.test_loader:
            data = data.to(self.device)
            target = target.to(self.device)

            output = self.model(data)
            loss = F.cross_entropy(output, target)

            test_loss.update(loss.item(), data.size(0))
            test_acc.update(output, target)

        return test_loss, test_acc 
Example #4
Source File: main.py    From pytorch-distributed-example with MIT License 6 votes vote down vote up
def train(self):
        self.model.train()

        train_loss = Average()
        train_acc = Accuracy()

        for data, target in self.train_loader:
            data = data.to(self.device)
            target = target.to(self.device)

            output = self.model(data)
            loss = F.cross_entropy(output, target)

            self.optimizer.zero_grad()
            loss.backward()
            self.optimizer.step()

            train_loss.update(loss.item(), data.size(0))
            train_acc.update(output, target)

        return train_loss, train_acc 
Example #5
Source File: main.py    From PyTorch with MIT License 6 votes vote down vote up
def test(epoch):
    model.eval()
    test_loss = 0
    with torch.no_grad():
        for i, (data, _) in enumerate(test_loader):
            data = data.to(device)
            recon_batch, mu, logvar = model(data)
            test_loss += loss_function(recon_batch, data, mu, logvar).item()
            if i == 0:
                n = min(data.size(0), 8)
                comparison = torch.cat([data[:n],
                                      recon_batch.view(args.batch_size, 1, 28, 28)[:n]])
                save_image(comparison.cpu(),
                         'results/reconstruction_' + str(epoch) + '.png', nrow=n)

    test_loss /= len(test_loader.dataset)
    print('====> Test set loss: {:.4f}'.format(test_loss)) 
Example #6
Source File: main.py    From ArtificialIntelligenceEngines with MIT License 6 votes vote down vote up
def test(epoch):
    model.eval()
    test_loss = 0
    with torch.no_grad():
        # each data is of BATCH_SIZE (default 128) samples
        for i, (data, _) in enumerate(test_loader):
            data = data.to(device)
            recon_batch, mu, logvar = model(data)
            test_loss += loss_function(recon_batch, data, mu, logvar).item()
            if i == 0:
                n = min(data.size(0), 8)
                # for the first 128 batch of the epoch, show the first 8 input digits
                # with right below them the reconstructed output digits
                comparison = torch.cat([data[:n],
                                      recon_batch.view(args.batch_size, 1, 28, 28)[:n]])
                save_image(comparison.cpu(),
                         'results/reconstruction_' + str(epoch) + '.png', nrow=n)

    test_loss /= len(test_loader.dataset)
    print('====> Test set loss: {:.4f}'.format(test_loss))

########## create VAE ########## 
Example #7
Source File: roibatchLoader.py    From OICR-pytorch with MIT License 6 votes vote down vote up
def __getitem__(self, index):
    # get the anchor index for current sample index
    # here we set the anchor index to the last one
    # sample in this group
        minibatch_db =  [self._roidb[index]] # [self._roidb[index_ratio]]
        blobs = get_minibatch(minibatch_db, self._num_classes)
        np.random.shuffle(blobs['rois'])
        rois = torch.from_numpy(blobs['rois'][:self.max_rois_size])
        data = torch.from_numpy(blobs['data'])
        labels = torch.from_numpy(blobs['labels'])
        data_height, data_width = data.size(1), data.size(2)
        
        data = data.permute(0, 3, 1, 2).contiguous().view(3, data_height, data_width)

        info = torch.Tensor([rois.size(0), data_height, data_width])
    
        return data, rois, labels, info 
Example #8
Source File: ctgan.py    From SDGym with MIT License 6 votes vote down vote up
def calc_gradient_penalty(netD, real_data, fake_data, device='cpu', pac=10, lambda_=10):
    alpha = torch.rand(real_data.size(0) // pac, 1, 1, device=device)
    alpha = alpha.repeat(1, pac, real_data.size(1))
    alpha = alpha.view(-1, real_data.size(1))

    interpolates = alpha * real_data + ((1 - alpha) * fake_data)

    # interpolates = torch.Variable(interpolates, requires_grad=True, device=device)

    disc_interpolates = netD(interpolates)

    gradients = torch.autograd.grad(
        outputs=disc_interpolates, inputs=interpolates,
        grad_outputs=torch.ones(disc_interpolates.size(), device=device),
        create_graph=True, retain_graph=True, only_inputs=True)[0]

    gradient_penalty = (
        (gradients.view(-1, pac * real_data.size(1)).norm(2, dim=1) - 1) ** 2).mean() * lambda_
    return gradient_penalty 
Example #9
Source File: loader.py    From nni with MIT License 5 votes vote down vote up
def test_train_loader():
    train_loader, val_loader = get_train_loaders(1, batch_size=4, dev_mode=False, pad_mode='edge', meta_version=2, pseudo_label=True)
    print(train_loader.num, val_loader.num)
    for i, data in enumerate(train_loader):
        imgs, masks, salt_exists = data
        #pdb.set_trace()
        print(imgs.size(), masks.size(), salt_exists.size())
        print(salt_exists)
        add_depth_channel(imgs, 'resize')
        print(masks)
        break
        #print(imgs)
        #print(masks) 
Example #10
Source File: test_demo.py    From PartNet with MIT License 5 votes vote down vote up
def writeply(savedir, data, label):
	path = os.path.dirname(savedir)
	if not os.path.exists(path):
		os.makedirs(path)
	if data.size(0) == 0:
		n_vertex = 0
	else:
		n_vertex = data.size(1)
	with open(savedir, 'w') as f:
		f.write('ply\n')
		f.write('format ascii 1.0\n')
		f.write('comment 111231\n')
		f.write('element vertex %d\n' % n_vertex)
		f.write('property float x\n')
		f.write('property float y\n')
		f.write('property float z\n')
		f.write('property float nx\n')
		f.write('property float ny\n')
		f.write('property float nz\n')
		f.write('property uchar red\n')
		f.write('property uchar green\n')
		f.write('property uchar blue\n')
		f.write('property uchar label\n')
		f.write('end_header\n')
		for j in range(n_vertex):
			f.write('%g %g %g %g %g %g %d %d %d %d\n' % (*data[0, j], *color[label[j]], label[j])) 
Example #11
Source File: loader.py    From nni with MIT License 5 votes vote down vote up
def test_test_loader():
    test_loader = get_test_loader(4, pad_mode='resize')
    print(test_loader.num)
    for i, data in enumerate(test_loader):
        print(data.size())
        if i > 5:
            break 
Example #12
Source File: batchLoader.py    From Iterative-Visual-Reasoning.pytorch with MIT License 5 votes vote down vote up
def get_one_sample(self, index):
        num_images = 1
        # Sample random scales to use for each image in this batch
        scales = (600,)
        max_scale = 600
        random_scale_inds = npr.randint(0, high=len(scales), size=num_images)

        # Get the input image blob
        im_blob, im_scales = self.get_image_blob(index, random_scale_inds, scales, max_scale)

        blobs = {'data': im_blob}

        # gt boxes: (x1, y1, x2, y2, cls)

        gt_inds = np.where(self.roidb[index[0]]['gt_classes'] != 0)[0]
        gt_boxes = np.empty((len(gt_inds), 5), dtype=np.float32)
        gt_boxes[:, 0:4] = self.roidb[index[0]]['boxes'][gt_inds, :] * im_scales[0]
        gt_boxes[:, 4] = self.roidb[index[0]]['gt_classes'][gt_inds]
        blobs['gt_boxes'] = gt_boxes
        # height, width, scale
        blobs['im_info'] = np.array([im_blob.shape[1],
                                     im_blob.shape[2],
                                     im_scales[0]], dtype=np.float32)
        if self.args.with_global:
            blobs['memory_size'] = np.ceil(blobs['im_info'][:2] / self.args.BOTTLE_SCALE / 2.).astype(np.int32)  # conv5
        else:
            blobs['memory_size'] = np.ceil(blobs['im_info'][:2] / self.args.BOTTLE_SCALE).astype(np.int32)  # conv4
        blobs['num_gt'] = np.int32(gt_boxes.shape[0])
        blobs['img'] = self.roidb[index[0]]['image']
        blobs['index'] = index[0]

        return blobs 
Example #13
Source File: batchLoader.py    From Iterative-Visual-Reasoning.pytorch with MIT License 5 votes vote down vote up
def __getitem__(self, index):
        indexes = [index]
        blobs = self.get_one_sample(indexes)
        data = torch.from_numpy(blobs['data'])
        im_info = torch.from_numpy(blobs['im_info'])
        mem_size = torch.from_numpy(blobs['memory_size'])
        # we need to random shuffle the bounding box.
        data_height, data_width = data.size(1), data.size(2)

        if self.phase == 'train':
            # if the number of region is greater than 100 then random pick 100 regions
            # this opt can make the used memory of GPUs more stable.
            # only for train and val phase
            np.random.shuffle(blobs['gt_boxes'])
            if blobs['gt_boxes'].shape[0] > 100:
                print('sampling regions from %d to %d' % (blobs['gt_boxes'].shape[0], 100))
                blobs['gt_boxes'] = blobs['gt_boxes'][:100]
        elif self.phase == 'eval':
            # np.random.shuffle(blobs['gt_boxes'])
            if blobs['gt_boxes'].shape[0] > 100:
                print('sampling regions from %d to %d' % (blobs['gt_boxes'].shape[0], 100))
                blobs['gt_boxes'] = blobs['gt_boxes'][:100]
        else:
            pass

        # if self.args.with_global:
        #     Arr_ = self._get_adjmat_Arr(blobs['gt_boxes'])  # 5*r*r
        #     Arr = torch.from_numpy(Arr_)
        # else:
        #     Arr = 0.

        gt_boxes = torch.from_numpy(blobs['gt_boxes'])
        # permute trim_data to adapt to downstream processing
        data = data.permute(0, 3, 1, 2).contiguous().view(3, data_height, data_width)
        im_info = im_info.view(3)
        return data, im_info, gt_boxes, mem_size, blobs['data'], blobs['gt_boxes'] 
Example #14
Source File: ctgan.py    From SDGym with MIT License 5 votes vote down vote up
def forward(self, input):
        assert input.size()[0] % self.pack == 0
        return self.seq(input.view(-1, self.packdim)) 
Example #15
Source File: ctgan.py    From SDGym with MIT License 5 votes vote down vote up
def cond_loss(data, output_info, c, m):
    loss = []
    st = 0
    st_c = 0
    skip = False
    for item in output_info:
        if item[1] == 'tanh':
            st += item[0]
            skip = True

        elif item[1] == 'softmax':
            if skip:
                skip = False
                st += item[0]
                continue

            ed = st + item[0]
            ed_c = st_c + item[0]
            tmp = F.cross_entropy(
                data[:, st:ed],
                torch.argmax(c[:, st_c:ed_c], dim=1),
                reduction='none'
            )
            loss.append(tmp)
            st = ed
            st_c = ed_c

        else:
            assert 0
    loss = torch.stack(loss, dim=1)

    return (loss * m).sum() / data.size()[0] 
Example #16
Source File: io.py    From transfer with MIT License 5 votes vote down vote up
def make_batch(emblayer, sequences, oov='<oov>'):
    # sequences = pad(sequences, pad_left = pad_left)
    batch_size = len(sequences)
    length = len(sequences[0])
    word2id, oovid = emblayer.word2id, emblayer.oovid
    data = torch.LongTensor([ word2id.get(w, word2id.get(w.lower(), oovid)) \
                                for seq in sequences for w in seq ])
    assert data.size(0) == batch_size * length

    return data.view(batch_size, length).t().contiguous() # length * batch_size 
Example #17
Source File: experiment_coding.py    From integer_discrete_flows with MIT License 5 votes vote down vote up
def encode_images(img, model, decode):
    batchsize, img_c, img_h, img_w = img.size()
    c, h, w = model.args.input_size

    assert img_h == img_w and h == w

    if img_h != h:
        assert img_h % h == 0
        steps = img_h // h

        states = [[] for i in range(batchsize)]
        state_sizes = [0 for i in range(batchsize)]
        bpd = [0 for i in range(batchsize)]
        error = 0

        for j in range(steps):
            for i in range(steps):
                r = encode_patches(
                    img[:, :, j*h:(j+1)*h, i*w:(i+1)*w], model, decode)
                for b in range(batchsize):

                    if r[0][b] is None:
                        states[b].append(None)
                    else:
                        states[b].extend(r[0][b])
                    state_sizes[b] += r[1][b]
                    bpd[b] += r[2][b] / steps**2
                    error += r[3]
        return states, state_sizes, bpd, error
    else:
        return encode_patches(img, model, decode) 
Example #18
Source File: experiment_coding.py    From integer_discrete_flows with MIT License 5 votes vote down vote up
def encode_patches(imgs, model, decode):
    batchsize, img_c, img_h, img_w = imgs.size()
    c, h, w = model.args.input_size
    assert img_h == h and img_w == w

    states = model.encode(imgs)

    bpd = model.forward(imgs)[1].cpu().numpy()

    state_sizes = []
    error = 0

    for b in range(batchsize):
        if states[b] is None:
            # Using escape bit ;)
            state_sizes += [8 * img_c * img_h * img_w + 1]

            # Error remains unchanged.
            print('Escaping, not encoding.')

        else:
            if decode:
                x_recon = model.decode([states[b]])

                error += torch.sum(
                    torch.abs(x_recon.int() - imgs[b].int())).item()

            # Append state plus an escape bit
            state_sizes += [32 * len(states[b]) + 1]

    return states, state_sizes, bpd, error 
Example #19
Source File: sequential_mnist.py    From stable-nalu with MIT License 5 votes vote down vote up
def __getitem__(self, index):
        mnist_images = []
        mnist_targets = []
        for mnist_index in range(index * self._seq_length, (index + 1) * self._seq_length):
            image, target = self._dataset[self._index_mapping[mnist_index]]
            mnist_images.append(image)  # image.size() = [1, 28, 28]
            mnist_targets.append(target)

        data = torch.stack(mnist_images)  # data.size() = [seq_length, 1, 28, 28]
        target = self._operation(np.stack(mnist_targets))

        return (
            data,
            torch.tensor(target, dtype=torch.float32)
        ) 
Example #20
Source File: sequential_svhn.py    From stable-nalu with MIT License 5 votes vote down vote up
def __getitem__(self, index):
        svhn_images = []
        svhn_targets = []
        for svhn_index in range(index * self._seq_length, (index + 1) * self._seq_length):
            image, target = self._dataset[self._index_mapping[svhn_index]]
            svhn_images.append(image)  # image.size() = [3, 32, 32]
            svhn_targets.append(target)

        data = torch.stack(svhn_images)  # data.size() = [seq_length, 3, 32, 32]
        target = self._operation(np.stack(svhn_targets))

        return (
            data,
            torch.tensor(target, dtype=torch.float32)
        ) 
Example #21
Source File: sr_dataset.py    From pykaldi2 with MIT License 5 votes vote down vote up
def _utt2seg(data, seg_len, seg_shift):
    """ Cut an utterance (MxN matrix) to segments. """
    if data.ndim == 1:
        data = np.reshape(data, (1, data.size))
    dim, n_fr = data.shape
    n_seg = int(np.floor((n_fr - seg_len) / seg_shift)) + 1
    seg = []
    for i in range(n_seg):
        start = i * seg_shift
        stop = start + seg_len
        seg.append(data[:, start:stop])

    return seg 
Example #22
Source File: main.py    From pytorch-distributed-example with MIT License 5 votes vote down vote up
def update(self, output, target):
        pred = output.argmax(dim=1)
        correct = pred.eq(target).sum().item()

        self.correct += correct
        self.count += output.size(0) 
Example #23
Source File: main.py    From pytorch-distributed-example with MIT License 5 votes vote down vote up
def forward(self, x):
        return self.fc(x.view(x.size(0), -1)) 
Example #24
Source File: main.py    From pytorch-distributed-example with MIT License 5 votes vote down vote up
def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--backend', type=str, default='gloo', help='Name of the backend to use.')
    parser.add_argument('-i',
                        '--init-method',
                        type=str,
                        default='tcp://127.0.0.1:23456',
                        help='URL specifying how to initialize the package.')
    parser.add_argument('-s', '--world-size', type=int, default=1, help='Number of processes participating in the job.')
    parser.add_argument('-r', '--rank', type=int, default=0, help='Rank of the current process.')
    parser.add_argument('--epochs', type=int, default=20)
    parser.add_argument('--no-cuda', action='store_true')
    parser.add_argument('-lr', '--learning-rate', type=float, default=1e-3)
    parser.add_argument('--root', type=str, default='data')
    parser.add_argument('--batch-size', type=int, default=128)
    args = parser.parse_args()
    print(args)

    if args.world_size > 1:
        distributed.init_process_group(
            backend=args.backend,
            init_method=args.init_method,
            world_size=args.world_size,
            rank=args.rank,
        )

    run(args) 
Example #25
Source File: wavenet_data.py    From HandsOnDeepLearningWithPytorch with MIT License 5 votes vote down vote up
def one_hot_encode(data, channels=256):
    one_hot = np.zeros((data.size, channels), dtype=float)
    one_hot[np.arange(data.size), data.ravel()] = 1

    return one_hot 
Example #26
Source File: wavenet_data.py    From HandsOnDeepLearningWithPytorch with MIT License 5 votes vote down vote up
def __init__(self, data_dir, receptive_fields,
                 sample_size=0, sample_rate=16000, in_channels=256,
                 batch_size=1, shuffle=True):
        """
        DataLoader for WaveNet
        :param data_dir:
        :param receptive_fields: integer. size(length) of receptive fields
        :param sample_size: integer. number of timesteps to train at once.
                            sample size has to be bigger than receptive fields.
                            |-- receptive field --|---------------------|
                            |------- samples -------------------|
                            |---------------------|-- outputs --|
        :param sample_rate: sound sampling rates
        :param in_channels: number of input channels
        :param batch_size:
        :param shuffle:
        """
        dataset = Dataset(data_dir, sample_rate, in_channels)

        super(DataLoader, self).__init__(dataset, batch_size, shuffle)

        if sample_size <= receptive_fields:
            raise Exception("sample_size has to be bigger than receptive_fields")

        self.sample_size = sample_size
        self.receptive_fields = receptive_fields

        self.collate_fn = self._collate_fn 
Example #27
Source File: main.py    From VQ-VAE with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def save_reconstructed_images(data, epoch, outputs, save_path, name):
    size = data.size()
    n = min(data.size(0), 8)
    batch_size = data.size(0)
    comparison = torch.cat([data[:n],
                            outputs.view(batch_size, size[1], size[2], size[3])[:n]])
    save_image(comparison.cpu(),
               os.path.join(save_path, name + '_' + str(epoch) + '.png'), nrow=n, normalize=True) 
Example #28
Source File: train_OriNet_test_on_graffity.py    From affnet with MIT License 5 votes vote down vote up
def extract_random_LAF(data, max_rot = math.pi, max_tilt = 1.0, crop_size = 32):
    st = int((data.size(2) - crop_size)/2)
    fin = st + crop_size
    if type(max_rot) is float:
        rot_LAFs, inv_rotmat = get_random_rotation_LAFs(data, max_rot)
    else:
        rot_LAFs = max_rot
        inv_rotmat = None
    aff_LAFs, inv_TA = get_random_norm_affine_LAFs(data, max_tilt);
    aff_LAFs[:,0:2,0:2] = torch.bmm(rot_LAFs[:,0:2,0:2],aff_LAFs[:,0:2,0:2])
    data_aff = extract_patches(data,  aff_LAFs, PS = data.size(2))
    data_affcrop = data_aff[:,:, st:fin, st:fin].contiguous()
    return data_affcrop, data_aff, rot_LAFs,inv_rotmat,inv_TA 
Example #29
Source File: main.py    From pytorch-flows with MIT License 5 votes vote down vote up
def validate(epoch, model, loader, prefix='Validation'):
    global global_step, writer

    model.eval()
    val_loss = 0

    pbar = tqdm(total=len(loader.dataset))
    pbar.set_description('Eval')
    for batch_idx, data in enumerate(loader):
        if isinstance(data, list):
            if len(data) > 1:
                cond_data = data[1].float()
                cond_data = cond_data.to(device)
            else:
                cond_data = None

            data = data[0]
        data = data.to(device)
        with torch.no_grad():
            val_loss += -model.log_probs(data, cond_data).sum().item()  # sum up batch loss
        pbar.update(data.size(0))
        pbar.set_description('Val, Log likelihood in nats: {:.6f}'.format(
            -val_loss / pbar.n))

    writer.add_scalar('validation/LL', val_loss / len(loader.dataset), epoch)

    pbar.close()
    return val_loss / len(loader.dataset) 
Example #30
Source File: amazon.py    From Point-Then-Operate with Apache License 2.0 5 votes vote down vote up
def __init__(self, mode, noisy_for_train, sentiment, direction):
        self.mode = mode
        self.root = os.path.join('../data', 'yelp')
        self.noisy = self.mode == 'train' and noisy_for_train

        # Load data from domain 0 and domain 1.
        path = os.path.join(self.root, 'sentiment.{}.{}'.format(mode, sentiment))

        # Load vocabulary.
        print('----- Loading vocab -----')
        self.vocab = Vocabulary('../data/amazon/amazon.vocab')
        print('vocabulary size:', self.vocab.size)
        self.pad = self.vocab.word2id['<pad>']
        self.go = self.vocab.word2id['<go>']
        self.eos = self.vocab.word2id['<eos>']
        self.unk = self.vocab.word2id['<unk>']

        # Tokenize file content
        with open(path, 'r') as f:
            ids = []
            for line in f:
                words = ['<go>'] + line.split() + ['<eos>']
                if direction == 'forward':
                    pass
                elif direction == 'backward':
                    words.reverse()
                else:
                    raise ValueError()
                for word in words:
                    ids.append(self.vocab.word2id[word] if word in self.vocab.word2id else self.unk)
        self.ids = torch.LongTensor(ids)  # (very_long, )
        self.ids = batchify(self.ids, config.batch_size, config)  # shape = (???, batch_size)