Python torch.numel() Examples

def to_float(val):
    """ Check that val is one of the following:
    - pytorch autograd Variable with one element
    - pytorch tensor with one element
    - numpy array with one element
    - any type supporting float() operation
    And convert val to float
    """

    n_elements = 1
    if isinstance(val, np.ndarray):
        n_elements = val.size
    elif torch is not None and (isinstance(val, torch_autograd.Variable) or torch.is_tensor(val)):
        n_elements = torch.numel(val)

    assert n_elements == 1, \
        "val should have one element (got {})".format(n_elements)
    try:
        return float(val)
    except:
        raise TypeError("Unsupported type for val ({})".format(type(val))) 

def log_weights_sparsity(self, model, epoch):
        with open(self.fname, 'w') as csv_file:
            params_size = 0
            sparse_params_size = 0

            writer = csv.writer(csv_file)
            # write the header
            writer.writerow(['parameter', 'shape', 'volume', 'sparse volume', 'sparsity level'])

            for name, param in model.state_dict().items():
                if param.dim() in [2, 4]:
                    _density = density(param)
                    params_size += torch.numel(param)
                    sparse_params_size += param.numel() * _density
                    writer.writerow([name, size_to_str(param.size()),
                                       torch.numel(param),
                                       int(_density * param.numel()),
                                       (1-_density)*100]) 

def log_weights_sparsity(self, model, epoch):
        params_size = 0
        sparse_params_size = 0

        for name, param in model.state_dict().items():
            if param.dim() in [2, 4]:
                _density = density(param)
                params_size += torch.numel(param)
                sparse_params_size += param.numel() * _density
                self.tblogger.scalar_summary('sparsity/weights/' + name,
                                             sparsity(param)*100, epoch)
                self.tblogger.scalar_summary('sparsity-2D/weights/' + name,
                                             sparsity_2D(param)*100, epoch)

        self.tblogger.scalar_summary("sprasity/weights/total", 100*(1 - sparse_params_size/params_size), epoch)
        self.tblogger.sync_to_file() 

def density(tensor):
    """Computes the density of a tensor.

    Density is the fraction of non-zero elements in a tensor.
    If a tensor has a density of 1.0, then it has no zero elements.

    Args:
        tensor: the tensor for which we compute the density.

    Returns:
        density (float)
    """
    nonzero = torch.nonzero(tensor)
    if nonzero.dim() == 0:
        return 0.0
    return nonzero.size(0) / float(torch.numel(tensor)) 

def _calc_apoz(self, activations):
        """
        Calculate APoZ(average percentage of zeros) of activations.

        Parameters
        ----------
        activations : list
            Layer's output activations

        Returns
        -------
        torch.Tensor
            Filter's APoZ(average percentage of zeros) of the activations
        """
        activations = torch.cat(activations, 0)
        _eq_zero = torch.eq(activations, torch.zeros_like(activations))
        _apoz = torch.sum(_eq_zero, dim=(0, 2, 3)) / torch.numel(_eq_zero[:, 0, :, :])
        return _apoz 

def construct_feature_mask(
        self, inputs: Tuple[Tensor, ...]
    ) -> Tuple[Tuple[Tensor, ...], int]:
        feature_mask = []
        current_num_features = 0
        for i in range(len(inputs)):
            num_features = torch.numel(inputs[i][0])
            feature_mask.append(
                current_num_features
                + torch.reshape(
                    torch.arange(num_features, device=inputs[i].device),
                    inputs[i][0:1].shape,
                )
            )
            current_num_features += num_features
        total_features = current_num_features
        feature_mask = tuple(feature_mask)
        return feature_mask, total_features 

def grad_sparsity(self):
        global_state = self._global_state
        if self._iter == 0:
            global_state["sparsity_avg"] = 0.0
        non_zero_cnt = 0.0
        all_entry_cnt = 0.0
        for group in self._optimizer.param_groups:
            for p in group['params']:
                if p.grad is None:
                    continue
                grad = p.grad.data
                grad_non_zero = grad.nonzero()
                if grad_non_zero.dim() > 0:
                    non_zero_cnt += grad_non_zero.size()[0]
                all_entry_cnt += torch.numel(grad)
        beta = self._beta
        global_state["sparsity_avg"] = beta * global_state["sparsity_avg"] \
                                       + (1 - beta) * non_zero_cnt / float(all_entry_cnt)
        self._sparsity_avg = \
            global_state["sparsity_avg"] / self.zero_debias_factor()

        if self._verbose:
            logging.debug("sparsity %f, sparsity avg %f", non_zero_cnt / float(all_entry_cnt), self._sparsity_avg)

        return 

def grad_sparsity(self):
    global_state = self._global_state
    if self._iter == 0:
      global_state["sparsity_avg"] = 0.0
    non_zero_cnt = 0.0
    all_entry_cnt = 0.0
    for group in self._optimizer.param_groups:
      for p in group['params']:
        if p.grad is None:
          continue
        grad = p.grad.data
        grad_non_zero = grad.nonzero()
        if grad_non_zero.dim() > 0:
          non_zero_cnt += grad_non_zero.size()[0]
        all_entry_cnt += torch.numel(grad)
    beta = self._beta
    global_state["sparsity_avg"] = beta * global_state["sparsity_avg"] \
      + (1 - beta) * non_zero_cnt / float(all_entry_cnt)
    self._sparsity_avg = \
      global_state["sparsity_avg"] / self.zero_debias_factor()
    
    if self._verbose:
      logging.debug("sparsity %f, sparsity avg %f", non_zero_cnt / float(all_entry_cnt), self._sparsity_avg)

    return 

def grad_sparsity(self):
    global_state = self._global_state
    if self._iter == 0:
      global_state["sparsity_avg"] = 0.0
    non_zero_cnt = 0.0
    all_entry_cnt = 0.0
    for group in self._optimizer.param_groups:
      for p in group['params']:
        if p.grad is None:
          continue
        grad = p.grad.data
        grad_non_zero = grad.nonzero()
        if grad_non_zero.dim() > 0:
          non_zero_cnt += grad_non_zero.size()[0]
        all_entry_cnt += torch.numel(grad)
    beta = self._beta
    global_state["sparsity_avg"] = beta * global_state["sparsity_avg"] \
      + (1 - beta) * non_zero_cnt / float(all_entry_cnt)
    self._sparsity_avg = \
      global_state["sparsity_avg"] / self.zero_debias_factor()
    
    if DEBUG:
      logging.debug("sparsity %f, sparsity avg %f", non_zero_cnt / float(all_entry_cnt), self._sparsity_avg)

    return 

def l2_loss(pred_traj, pred_traj_gt, loss_mask, random=0, mode='average'):
    """
    Input:
    - pred_traj: Tensor of shape (seq_len, batch, 2). Predicted trajectory.
    - pred_traj_gt: Tensor of shape (seq_len, batch, 2). Groud truth
    predictions.
    - loss_mask: Tensor of shape (batch, seq_len)
    - mode: Can be one of sum, average, raw
    Output:
    - loss: l2 loss depending on mode
    """
    seq_len, batch, _ = pred_traj.size()
    loss = (loss_mask.unsqueeze(dim=2) *
            (pred_traj_gt.permute(1, 0, 2) - pred_traj.permute(1, 0, 2))**2)
    if mode == 'sum':
        return torch.sum(loss)
    elif mode == 'average':
        return torch.sum(loss) / torch.numel(loss_mask.data)
    elif mode == 'raw':
        return loss.sum(dim=2).sum(dim=1) 

def _perturb_func(inputs):
    def perturb_ratio(input):
        return (
            torch.arange(-torch.numel(input[0]) // 2, torch.numel(input[0]) // 2)
            .view(input[0].shape)
            .float()
            / 100
        )

    if isinstance(inputs, tuple):
        input1 = inputs[0]
        input2 = inputs[1]
    else:
        input1 = inputs
        input2 = None

    perturbed_input1 = input1 + perturb_ratio(input1)

    if input2 is None:
        return perturbed_input1

    return perturbed_input1, input2 + perturb_ratio(input2) 

def grad_sparsity(self):
    global_state = self._global_state
    if self._iter == 0:
      global_state["sparsity_avg"] = 0.0
    non_zero_cnt = 0.0
    all_entry_cnt = 0.0
    for group in self._optimizer.param_groups:
      for p in group['params']:
        if p.grad is None:
          continue
        grad = p.grad.data
        grad_non_zero = grad.nonzero()
        if grad_non_zero.dim() > 0:
          non_zero_cnt += grad_non_zero.size()[0]
        all_entry_cnt += torch.numel(grad)
    beta = self._beta
    global_state["sparsity_avg"] = beta * global_state["sparsity_avg"] \
      + (1 - beta) * non_zero_cnt / float(all_entry_cnt)
    self._sparsity_avg = \
      global_state["sparsity_avg"] / self.zero_debias_factor()
    return 

def var_loss_function_joint(output_samples_classification, target, output_samples_recon, inp, mu, std, device):
    recon_loss = nn.BCEWithLogitsLoss(reduction='sum')
    class_loss = nn.CrossEntropyLoss(reduction='sum')

    # Place-holders for the final loss values over all latent space samples
    recon_losses = torch.zeros(output_samples_recon.size(0)).to(device)
    cl_losses = torch.zeros(output_samples_classification.size(0)).to(device)

    # numerical value for stability of log computation
    eps = 1e-8

    # loop through each sample for each input and calculate the correspond loss. Normalize the losses.
    for i in range(output_samples_classification.size(0)):
        cl_losses[i] = class_loss(output_samples_classification[i], target) / torch.numel(target)
        recon_losses[i] = recon_loss(output_samples_recon[i], inp) / torch.numel(inp)

    # average the loss over all samples per input
    cl = torch.mean(cl_losses, dim=0)
    rl = torch.mean(recon_losses, dim=0)

    # Compute the KL divergence, normalized by latent dimensionality
    kld = -0.5 * torch.sum(1 + torch.log(eps + std ** 2) - (mu ** 2) - (std ** 2)) / torch.numel(mu)

    return cl, rl, kld 

def _get_feature_range_and_mask(self, input, input_mask, **kwargs):
        if input_mask is None:
            # Obtain feature mask for selected input tensor, matches size of
            # 1 input example, (1 x inputs[i].shape[1:])
            input_mask = torch.reshape(
                torch.arange(torch.numel(input[0]), device=input.device),
                input[0:1].shape,
            ).long()
        return (
            torch.min(input_mask).item(),
            torch.max(input_mask).item() + 1,
            input_mask,
        ) 

def _validate_target(num_samples: int, target: TargetType) -> None:
    if isinstance(target, list) or (
        isinstance(target, torch.Tensor) and torch.numel(target) > 1
    ):
        assert num_samples == len(target), (
            "The number of samples provied in the"
            "input {} does not match with the number of targets. {}".format(
                num_samples, len(target)
            )
        ) 

def _validate_target(num_samples: int, target: TargetType) -> None:
    if isinstance(target, list) or (
        isinstance(target, torch.Tensor) and torch.numel(target) > 1
    ):
        assert num_samples == len(target), (
            "The number of samples provied in the"
            "input {} does not match with the number of targets. {}".format(
                num_samples, len(target)
            )
        ) 

def forward(self, inputs, sparse_list):
        return (
            self.lin1(inputs) + (sparse_list[0] if torch.numel(sparse_list) > 0 else 0)
        ).sum() 

def _expand_target(
    target: TargetType,
    n_steps: int,
    expansion_type: ExpansionTypes = ExpansionTypes.repeat,
) -> TargetType:
    if isinstance(target, list):
        if expansion_type == ExpansionTypes.repeat:
            return target * n_steps
        elif expansion_type == ExpansionTypes.repeat_interleave:
            expanded_target = []
            for i in target:
                expanded_target.extend([i] * n_steps)
            return cast(Union[List[Tuple[int, ...]], List[int]], expanded_target)
        else:
            raise NotImplementedError(
                "Currently only `repeat` and `repeat_interleave`"
                " expansion_types are supported"
            )

    elif isinstance(target, torch.Tensor) and torch.numel(target) > 1:
        if expansion_type == ExpansionTypes.repeat:
            return torch.cat([target] * n_steps, dim=0)
        elif expansion_type == ExpansionTypes.repeat_interleave:
            return target.repeat_interleave(n_steps, dim=0)
        else:
            raise NotImplementedError(
                "Currently only `repeat` and `repeat_interleave`"
                " expansion_types are supported"
            )

    return target 

def forward(self, input_ids, sep_indices=None, sep_len=None, token_type_ids=None):
        seq_length = input_ids.size(1)
        position_ids = torch.arange(seq_length, dtype=torch.long, device=input_ids.device)
        position_ids = position_ids.unsqueeze(0).expand_as(input_ids)
        if token_type_ids is None:
            token_type_ids = torch.zeros_like(input_ids)
        
        words_embeddings = self.word_embeddings(input_ids)
        position_embeddings = self.position_embeddings(position_ids)

        token_type_ids_extension = token_type_ids - self.config.type_vocab_size
        token_type_ids_extension_mask = (token_type_ids_extension >= 0).float()
        token_type_ids_extension = (token_type_ids_extension.float() * token_type_ids_extension_mask).long()

        token_type_ids_mask = (token_type_ids < self.config.type_vocab_size).float()
        assert torch.sum(token_type_ids_extension_mask + token_type_ids_mask) ==  \
            torch.numel(token_type_ids) == torch.numel(token_type_ids_mask)
        token_type_ids = (token_type_ids.float() * token_type_ids_mask).long()

        token_type_embeddings = self.token_type_embeddings(token_type_ids)
        token_type_embeddings_extension = self.token_type_embeddings_extension(token_type_ids_extension)
        
        token_type_embeddings = (token_type_embeddings * token_type_ids_mask.unsqueeze(-1)) + \
                                (token_type_embeddings_extension * token_type_ids_extension_mask.unsqueeze(-1))
        
        embeddings = words_embeddings + position_embeddings + token_type_embeddings

        embeddings = self.LayerNorm(embeddings)
        embeddings = self.dropout(embeddings)
        return embeddings 

def _select_targets(output: Tensor, target: TargetType) -> Tensor:
    if target is None:
        return output

    num_examples = output.shape[0]
    dims = len(output.shape)
    device = output.device
    if isinstance(target, (int, tuple)):
        return _verify_select_column(output, target)
    elif isinstance(target, torch.Tensor):
        if torch.numel(target) == 1 and isinstance(target.item(), int):
            return _verify_select_column(output, cast(int, target.item()))
        elif len(target.shape) == 1 and torch.numel(target) == num_examples:
            assert dims == 2, "Output must be 2D to select tensor of targets."
            return torch.gather(output, 1, target.reshape(len(output), 1))
        else:
            raise AssertionError(
                "Tensor target dimension %r is not valid. %r"
                % (target.shape, output.shape)
            )
    elif isinstance(target, list):
        assert len(target) == num_examples, "Target list length does not match output!"
        if isinstance(target[0], int):
            assert dims == 2, "Output must be 2D to select tensor of targets."
            return torch.gather(
                output, 1, torch.tensor(target, device=device).reshape(len(output), 1)
            )
        elif isinstance(target[0], tuple):
            return torch.stack(
                [
                    output[(i,) + cast(Tuple, targ_elem)]
                    for i, targ_elem in enumerate(target)
                ]
            )
        else:
            raise AssertionError("Target element type in list is not valid.")
    else:
        raise AssertionError("Target type %r is not valid." % target) 

def __getitem__(self, index):
        item = self.dataset[index]
        if torch.is_tensor(item):
            return torch.numel(item)
        else:
            return np.size(item) 

def charbonnier_loss(difference, mask, alpha=1, beta=1., epsilon=0.001):
    '''
    : sum( (x*beta)^2 + epsilon^2)^alpha
    '''
    if mask is not None:
        assert difference.size(0) == mask.size(0)
        assert difference.size(2) == mask.size(2)
        assert difference.size(3) == mask.size(3)
    res = torch.pow(torch.pow(difference * beta, 2) + epsilon ** 2, alpha)
    if mask is not None:
        batch_pixels = torch.sum(mask)
        return torch.sum(res * mask) / batch_pixels
    else:
        batch_pixels = torch.numel(res)
        return torch.sum(res) / batch_pixels 

def SSIM_loss(img1, img2, kernel_size=8, stride=8, c1=0.00001, c2=0.00001):
    num = img1.size(0)
    channels = img1.size(1)

    kernel_h = kernel_w = kernel_size
    sigma = (kernel_w + kernel_h) / 12.
    gauss_kernel = torch.zeros((1, 1, kernel_h, kernel_w)).type(img1.type())
    for h in range(kernel_h):
        for w in range(kernel_w):
            gauss_kernel[0, 0, h, w] = math.exp(
                -(math.pow(h - kernel_h/2.0, 2) + math.pow(- kernel_w/2.0, 2))
                / (2.0 * sigma ** 2)) / (2 * 3.14159 * sigma ** 2)
    gauss_kernel = gauss_kernel / torch.sum(gauss_kernel)
    gauss_kernel = gauss_kernel.repeat(channels, 1, 1, 1)

    gauss_filter = nn.Conv2d(channels, channels, kernel_size,
                             stride=stride, padding=0,
                             groups=channels, bias=False)
    gauss_filter.weight.data = gauss_kernel
    gauss_filter.weight.requires_grad = False

    ux = gauss_filter(img1)
    uy = gauss_filter(img2)
    sx2 = gauss_filter(img1 ** 2)
    sy2 = gauss_filter(img2 ** 2)
    sxy = gauss_filter(img1 * img2)

    ux2 = ux ** 2
    uy2 = uy ** 2
    sx2 = sx2 - ux2
    sy2 = sy2 - uy2
    sxy = sxy - ux * uy

    lp = (2 * ux * uy + c1) / (ux2 + uy2 + c1)
    sc = (2 * sxy + c2) / (sx2 + sy2 + c2)

    ssim = lp * sc
    return (lp.numel() - torch.sum(ssim)) / num 

def get_num_parameters(model):
    """
    Returns the number of trainable parameters in a model of type nn.Module
    :param model: nn.Module containing trainable parameters
    :return: number of trainable parameters in model
    """
    num_parameters = 0
    for parameter in model.parameters():
        num_parameters += torch.numel(parameter)
    return num_parameters 

def forward(self, input_ids, sep_indices=None, sep_len=None, token_type_ids=None):
        seq_length = input_ids.size(1)
        position_ids = torch.arange(seq_length, dtype=torch.long, device=input_ids.device)
        position_ids = position_ids.unsqueeze(0).expand_as(input_ids)
        if token_type_ids is None:
            token_type_ids = torch.zeros_like(input_ids)
        
        words_embeddings = self.word_embeddings(input_ids)
        position_embeddings = self.position_embeddings(position_ids)

        token_type_ids_extension = token_type_ids - self.config.type_vocab_size
        token_type_ids_extension_mask = (token_type_ids_extension >= 0).float()
        token_type_ids_extension = (token_type_ids_extension.float() * token_type_ids_extension_mask).long()

        token_type_ids_mask = (token_type_ids < self.config.type_vocab_size).float()
        assert torch.sum(token_type_ids_extension_mask + token_type_ids_mask) ==  \
            torch.numel(token_type_ids) == torch.numel(token_type_ids_mask)
        token_type_ids = (token_type_ids.float() * token_type_ids_mask).long()

        token_type_embeddings = self.token_type_embeddings(token_type_ids)
        token_type_embeddings_extension = self.token_type_embeddings_extension(token_type_ids_extension)
        
        token_type_embeddings = (token_type_embeddings * token_type_ids_mask.unsqueeze(-1)) + \
                                (token_type_embeddings_extension * token_type_ids_extension_mask.unsqueeze(-1))
        
        embeddings = words_embeddings + position_embeddings + token_type_embeddings

        embeddings = self.LayerNorm(embeddings)
        embeddings = self.dropout(embeddings)
        return embeddings 

def forward(self, output, target):
        output = F.sigmoid(output) > 0.5
        target = target.byte()

        n_true = torch.eq(output, target)
        n_all = torch.numel(target)
        n_true = n_true.sum()
        if n_true == 0:
            return n_true

        return n_true.float() / n_all 

def get_n_parameters(model):
    total_n_parameters = 0
    for parameter in model.parameters():
        total_n_parameters += torch.numel(parameter)
    return total_n_parameters 

def fill(x):
    return float(x.nonzero().size(0))/torch.numel(x) 

def forward(self, images, labels, classes, reconstructions):
        left = F.relu(0.9 - classes, inplace=True) ** 2
        right = F.relu(classes - 0.1, inplace=True) ** 2
        margin_loss = labels * left + 0.5 * (1. - labels) * right
        margin_loss = margin_loss.sum()
        assert torch.numel(images) == torch.numel(reconstructions)
        images = images.view(reconstructions.size()[0], -1)
        reconstruction_loss = self.reconstruction_loss(reconstructions, images)
        return (margin_loss + 0.0005 * reconstruction_loss) / images.size(0) 

def loss_function(output, target):
    class_loss = nn.CrossEntropyLoss(reduction='sum')

    cl = class_loss(output, target) / torch.numel(target)
    return cl