Python torch.nn.init() Examples

def init_params(module, init_fn):
    '''Initialize module's weights using init_fn, and biases to 0.0'''
    bias_init = 0.0
    classname = util.get_class_name(module)
    if 'Net' in classname:  # skip if it's a net, not pytorch layer
        pass
    elif any(k in classname for k in ('BatchNorm', 'Conv', 'Linear')):
        init_fn(module.weight)
        nn.init.constant_(module.bias, bias_init)
    elif 'GRU' in classname:
        for name, param in module.named_parameters():
            if 'weight' in name:
                init_fn(param)
            elif 'bias' in name:
                nn.init.constant_(param, bias_init)
    else:
        pass


# params methods 

def __init__(self, input_size, hidden_size, output_size, args):
        super(SimpleSeparateSelectionModule, self).__init__()
        self.output_size = output_size
        self.hidden_size = hidden_size

        self.encoder = nn.Sequential(
            nn.Linear(input_size, hidden_size),
            nn.Tanh(),
            nn.Dropout(args.dropout))

        self.decoders = nn.ModuleList()
        for i in range(6):
            self.decoders.append(nn.Linear(hidden_size, output_size))

        # init
        init_cont(self.encoder, args.init_range)
        init_cont(self.decoders, args.init_range) 

def _initialize_weights(self):
        for m in self.modules():
            # 卷积的初始化方法
            if isinstance(m, nn.Conv2d):
                # TODO: 使用正态分布进行初始化（0, 0.01) 网络权重看看
                # n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
                # He kaiming 初始化, 方差为2/n. math.sqrt(2. / n) 或者直接使用现成的nn.init中的函数。在这里会梯度爆炸
                m.weight.data.normal_(0, 0.001)    # # math.sqrt(2. / n)
                # torch.nn.init.kaiming_normal_(m.weight)
                # bias都初始化为0
                if m.bias is not None:  # 当有BN层时，卷积层Con不加bias！
                    m.bias.data.zero_()
            # batchnorm使用全1初始化 bias全0
            elif isinstance(m, nn.BatchNorm2d):
                m.weight.data.fill_(1)
                m.bias.data.zero_()

            elif isinstance(m, nn.Linear):
                torch.nn.init.normal_(m.weight.data, 0, 0.01)  # todo: 0.001?
                # m.weight.data.normal_(0, 0.01)
                m.bias.data.zero_() 

def _get_graph_state(self, G, start=0, end_offset=0, init=False):
        # get the graph states, the R function
        Hg = []
        max_n_nodes = max(g.vcount() for g in G)
        for g in G:
            hg = [g.vs[i]['H_forward'] for i in range(start, g.vcount() - end_offset)]
            hg = torch.cat(hg, 0)
            hg = hg.unsqueeze(0)  # 1 * n * hs
            if g.vcount() < max_n_nodes:
                hg = torch.cat([hg, 
                    torch.zeros(1, max_n_nodes - g.vcount(), hg.shape[2]).to(self.get_device())],
                    1)  # 1 * max_n * hs
            Hg.append(hg)
        # gated sum node states as the graph state
        Hg = torch.cat(Hg, 0)  # batch * max_n * hs
        if not init:
            Hg = self._gated(Hg, self.gate, self.mapper).sum(1)  # batch * gs
        else:
            Hg = self._gated(Hg, self.gate_init, self.mapper_init).sum(1)  # batch * gs

        return Hg  # batch * gs 

def _initialize_v(self, G, v, H=None):
        G = [g for g in G if g.vcount() > v]
        if len(G) == 0:
            return
        if H is not None:
            idx = [i for i, g in enumerate(G) if g.vcount() > v]
            H = H[idx]
        v_types = [g.vs[v]['type'] for g in G]
        X = self._one_hot(v_types, self.nvt)
        if H is None:
            Hg = self._get_graph_state(G, 0, 1, init=True)  # exclude v itself
        else:  
            Hg = H
        Hv = self.finit(torch.cat([X, Hg], -1))
        for i, g in enumerate(G):
            g.vs[v]['H_forward'] = Hv[i:i+1]
        return 

def init_layers(net, init_fn_name):
    '''Primary method to initialize the weights of the layers of a network'''
    if init_fn_name is None:
        return

    # get nonlinearity
    nonlinearity = get_nn_name(net.hid_layers_activation).lower()
    if nonlinearity == 'leakyrelu':
        nonlinearity = 'leaky_relu'  # guard name

    # get init_fn and add arguments depending on nonlinearity
    init_fn = getattr(nn.init, init_fn_name)
    if 'kaiming' in init_fn_name:  # has 'nonlinearity' as arg
        assert nonlinearity in ['relu', 'leaky_relu'], f'Kaiming initialization not supported for {nonlinearity}'
        init_fn = partial(init_fn, nonlinearity=nonlinearity)
    elif 'orthogonal' in init_fn_name or 'xavier' in init_fn_name:  # has 'gain' as arg
        gain = nn.init.calculate_gain(nonlinearity)
        init_fn = partial(init_fn, gain=gain)
    else:
        pass

    # finally, apply init_params to each layer in its modules
    net.apply(partial(init_params, init_fn=init_fn)) 

def init(self):
        xavier_normal_(self.emb_e.weight.data)
        xavier_normal_(self.emb_rel.weight.data) 

def weights_init(self,m):
        if isinstance(m, nn.Conv2d):
            nn.init.orthogonal(m.weight.data, gain=0.8)
            try:
                if m.weight.data.shape[-1] == 8: #last layer:
                    nn.init.orthogonal(m.weight.data, gain=1.0)
                    print ('last layer init bias')
                    m.bias.data = torch.FloatTensor([0.8, 0, 0.8 ])
                else:
                    nn.init.constant(m.bias.data, 0.01)
            except:
                pass
        return 

def init(self):
        xavier_normal_(self.emb_e.weight.data)
        xavier_normal_(self.emb_rel.weight.data) 

def _weight_init(init_params, m):
  def get_inits(m, init_params, weight_key, bias_key):
    weight_init, bias_init = None, None
    if weight_key in init_params and m.weight is not None:
      weight_init = init_params[weight_key]
    if bias_key in init_params and m.bias is not None:
      bias_init = init_params[bias_key]

    return weight_init, bias_init

  weight_init, bias_init = None, None
  classname = m.__class__.__name__

  if m in init_params:
    # Allow specialized init of individual modules
    # This is kind of a hack, because we can't specify this through the
    # configuration. But our layers have no names, so we can't address
    # individual ones.
    weight_init, bias_init = get_inits(m, init_params[m],
                                       'weight', 'bias')
  elif classname.find('Conv2d') != -1:
    weight_init, bias_init = get_inits(m, init_params,
                                       'conv_weight', 'conv_bias')
  elif classname.find('ConvTranspose2d') != -1:
    weight_init, bias_init = get_inits(m, init_params,
                                       'conv_transposed_weight',
                                       'conv_transposed_bias')
  elif classname.find('Linear') != -1:
    weight_init, bias_init = get_inits(m, init_params,
                                       'linear_weight', 'linear_bias')
  elif classname.find('BatchNorm2d') != -1:
    weight_init, bias_init = get_inits(m, init_params,
                                       'batchnorm_weight', 'batchnorm_bias')

  if weight_init is not None:
    _get_init_fn(weight_init)(m.weight.data)
  if bias_init is not None:
    _get_init_fn(bias_init)(m.bias.data) 

def init_msra(module):
    for m in module.modules():
        if isinstance(m, nn.Conv2d) or isinstance(m, nn.Linear):
            torch.nn.init.kaiming_uniform(m.weight)
            print(m.weight.size(), 'xavier init')
            try:
                m.bias.data.zero_()
            except:
                print('has no bias') 

def init_xavier(module):
    for m in module.modules():
        if isinstance(m, nn.Conv2d) or isinstance(m, nn.Linear):
            torch.nn.init.xavier_uniform_(m.weight)
            print(m.weight.size(), 'xavier init')
            try:
                m.bias.data.zero_()
            except:
                print('has no bias') 

def init_conv_weight(module, std=0.001):
    d = dict(module.named_parameters())
    if 'conv.weight' in d.keys():
        nn.init.normal(d['conv.weight'], std=std)
    if 'conv.bias' in d.keys():
        nn.init.constant(d['conv.bias'], 0) 

def init(self):
        xavier_normal_(self.emb_e_real.weight.data)
        xavier_normal_(self.emb_e_img.weight.data)
        xavier_normal_(self.emb_rel_real.weight.data)
        xavier_normal_(self.emb_rel_img.weight.data) 

def __init__(self, input_size, hidden_size, args):
        super(RecurrentUnit, self).__init__()

        self.x2h = nn.Sequential(
            nn.Linear(input_size, hidden_size),
            nn.Tanh())

        self.cell = nn.GRUCell(
            input_size=hidden_size,
            hidden_size=hidden_size,
            bias=True)

        # init
        init_cont(self.x2h, args.init_range)
        init_rnn_cell(self.cell, args.init_range) 

def __init__(self, word_dict, item_dict, context_dict, count_dict, args):
        super(SelectionModel, self).__init__()

        self.nhid_pos = 32
        self.nhid_speaker = 32
        self.len_cutoff = 10

        domain = get_domain(args.domain)

        self.word_dict = word_dict
        self.item_dict = item_dict
        self.context_dict = context_dict
        self.count_dict = count_dict
        self.args = args

        self.word_encoder = nn.Embedding(len(self.word_dict), args.nembed_word)
        self.pos_encoder = nn.Embedding(self.len_cutoff, self.nhid_pos)
        self.speaker_encoder = nn.Embedding(len(self.word_dict), self.nhid_speaker)
        self.ctx_encoder = MlpContextEncoder(len(self.context_dict), domain.input_length(),
            args.nembed_ctx, args.nhid_ctx, args.dropout, args.init_range, args.skip_values)

        self.sel_head = SelectionModule(
            query_size=args.nhid_ctx,
            value_size=args.nembed_word + self.nhid_pos + self.nhid_speaker,
            hidden_size=args.nhid_attn,
            selection_size=args.nhid_sel,
            num_heads=6,
            output_size=len(item_dict),
            args=args)

        self.dropout = nn.Dropout(args.dropout)

        # init embeddings
        self.word_encoder.weight.data.uniform_(-self.args.init_range, self.args.init_range)
        self.pos_encoder.weight.data.uniform_(-self.args.init_range, self.args.init_range)
        self.speaker_encoder.weight.data.uniform_(-self.args.init_range, self.args.init_range) 

def __init__(self, num_classes=1000):
        """ Constructor
        Args:
            num_classes: number of classes
        """
        super(Xception, self).__init__()
        
        self.num_classes = num_classes

        self.conv1 = nn.Conv2d(3, 24, kernel_size=3, stride=2, padding=1, bias=False)      # 224 x 224 -> 112 x 112
        self.bn1 = nn.BatchNorm2d(24)
        self.relu = nn.ReLU(inplace=True)
        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=0, ceil_mode=True)     # -> 56 x 56

        # Stage 2
        self.block1 = _Block(24, 144, 1+3, 2, start_with_relu=False, grow_first=True)     # -> 28 x 28

        # Stage 3
        self.block2 = _Block(144, 288, 1+7, 2, start_with_relu=True, grow_first=True)     # -> 14 x 14

        # Stage 4
        self.block3 = _Block(288, 576, 1+3, 2, start_with_relu=True, grow_first=True)     # -> 7 x 7

        self.avgpool = nn.AvgPool2d(7)
        self.fc = nn.Linear(576, num_classes)



        #------- init weights --------
        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
                m.weight.data.normal_(0, math.sqrt(2. / n))
            elif isinstance(m, nn.BatchNorm2d):
                m.weight.data.fill_(1)
                m.bias.data.zero_()
        #----------------------------- 

def init_weights(model):
    global conf
    for name, m in model.named_modules():
        if isinstance(m, nn.Conv2d):
            init_fn = getattr(nn.init, conf["network"]["weight_init"] + '_')
            if conf["network"]["weight_init"].startswith("xavier") or conf["network"]["weight_init"] == "orthogonal":
                gain = conf["network"]["weight_gain_multiplier"]
                if conf["network"]["activation"] == "relu" or conf["network"]["activation"] == "elu":
                    gain *= nn.init.calculate_gain("relu")
                elif conf["network"]["activation"] == "leaky_relu":
                    gain *= nn.init.calculate_gain("leaky_relu", conf["network"]["activation_param"])
                init_fn(m.weight, gain)
            elif conf["network"]["weight_init"].startswith("kaiming"):
                if conf["network"]["activation"] == "relu" or conf["network"]["activation"] == "elu":
                    init_fn(m.weight, 0)
                else:
                    init_fn(m.weight, conf["network"]["activation_param"])

            if hasattr(m, "bias") and m.bias is not None:
                nn.init.constant_(m.bias, 0.)
        elif isinstance(m, nn.BatchNorm2d) or isinstance(m, ABN):
            nn.init.constant_(m.weight, 1.)
            nn.init.constant_(m.bias, 0.)
        elif isinstance(m, nn.Linear):
            nn.init.xavier_uniform_(m.weight, .1)
            nn.init.constant_(m.bias, 0.) 

def weights_init(self,m):
        if isinstance(m, nn.Conv2d):
            nn.init.orthogonal(m.weight.data, gain=0.8)
            try:
                if m.weight.data.shape[-1] == 8: #last layer:
                    nn.init.orthogonal(m.weight.data, gain=1.0)
                    print ('last layer init bias')
                    m.bias.data = torch.FloatTensor([0.8, 0, 0.8 ])
                else:
                    nn.init.constant(m.bias.data, 0.01)
            except:
                pass
        return 

def weights_init(self,m):
        if isinstance(m, nn.Conv2d):
            nn.init.orthogonal(m.weight.data, gain=0.8)
            try:
                if m.weight.data.shape[-1] == 8: #last layer:
                    nn.init.orthogonal(m.weight.data, gain=1.0)
                    print ('last layer init bias')
                    m.bias.data = torch.FloatTensor([0.8,0, 0.8, 0, 1])
                else:
                    nn.init.constant(m.bias.data, 0.01)
            except:
                pass
        return 

def weights_init(self,m):
        if isinstance(m, nn.Conv2d):
            nn.init.orthogonal(m.weight.data, gain=0.8)
            try:
                nn.init.constant(m.bias.data, 0.01)
            except:
                pass
        return 

def weights_init(self,m):
        if isinstance(m, nn.Conv2d):
            nn.init.orthogonal(m.weight.data, gain=0.8)
            try:
                nn.init.constant(m.bias.data, 0.01)
            except:
                pass
        return 

def weights_init(self,m):
        if isinstance(m, nn.Conv2d):
            nn.init.orthogonal(m.weight.data, gain=0.8)
            try:
                if m.weight.data.shape[-1] == 8: #last layer:
                    nn.init.orthogonal(m.weight.data, gain=1.0)
                    print ('last layer init bias')
                    m.bias.data = torch.FloatTensor([1,0,0,1])
                else:
                    nn.init.constant(m.bias.data, 0.01)
            except:
                pass
        return 

def weights_init(self,m):
        if isinstance(m, nn.Conv2d):
            nn.init.orthogonal(m.weight.data, gain=0.8)
            try:
                if m.weight.data.shape[-1] == 8: #last layer:
                    nn.init.orthogonal(m.weight.data, gain=1.0)
                    print ('last layer init bias')
                    m.bias.data = torch.FloatTensor([0.8,0,0,0.8])
                else:
                    nn.init.constant(m.bias.data, 0.01)
            except:
                pass
        return 

def weights_init(self,m):
        if isinstance(m, nn.Conv2d):
            nn.init.orthogonal(m.weight.data, gain=0.8)
            try:
                if m.weight.data.shape[-1] == 8: #last layer:
                    nn.init.orthogonal(m.weight.data, gain=1.0)
                    print ('last layer init bias')
                    m.bias.data = torch.FloatTensor([1,0, 1, 0, 1])
                else:
                    nn.init.constant(m.bias.data, 0.01)
            except:
                pass
        return 

def weights_init(self,m):
        if isinstance(m, nn.Conv2d):
            nn.init.orthogonal(m.weight.data, gain=0.8)
            try:
                if m.weight.data.shape[-1] == 8: #last layer:
                    nn.init.orthogonal(m.weight.data, gain=1.0)
                    print ('last layer init bias')
                    m.bias.data = torch.FloatTensor([0.8,0, 0.8, 0, 1])
                else:
                    nn.init.constant(m.bias.data, 0.01)
            except:
                pass
        return 

def weights_init(self,m):
        if isinstance(m, nn.Conv2d):
            nn.init.orthogonal(m.weight.data, gain=0.8)
            try:
                nn.init.constant(m.bias.data, 0.01)
            except:
                pass
        return 

def weights_init(self,m):
        if isinstance(m, nn.Conv2d):
            nn.init.orthogonal(m.weight.data, gain=0.8)
            try:
                if m.weight.data.shape[-1] == 8: #last layer:
                    nn.init.orthogonal(m.weight.data, gain=1.0)
                    print ('last layer init bias')
                    m.bias.data = torch.FloatTensor([1,0,0,1])
                else:
                    nn.init.constant(m.bias.data, 0.01)
            except:
                pass
        return 

def weights_init(self,m):
        if isinstance(m, nn.Conv2d):
            nn.init.orthogonal(m.weight.data, gain=0.9)
            try:
                nn.init.constant(m.bias.data, 0.01)
            except:
                pass
        return 

def init(self):
        xavier_normal_(self.emb_e.weight.data)
        xavier_normal_(self.emb_rel.weight.data)
        xavier_normal_(self.gc1.weight.data)
        xavier_normal_(self.gc2.weight.data)