Python torch.nn.MultiLabelSoftMarginLoss() Examples

def train(model_name='model.pkl'):
    cnn = CNN()
    cnn.train()
    print('init net')
    criterion = nn.MultiLabelSoftMarginLoss()
    optimizer = torch.optim.Adam(
        cnn.parameters(), lr=setting.TRAIN_LEARNING_RATE)

    # Train the Model
    train_dataloader = dataset.get_train_data_loader()
    for epoch in range(setting.TRAIN_NUM_EPOCHS):
        for i, (images, labels) in enumerate(train_dataloader):
            images = Variable(images)
            labels = Variable(labels.float())
            predict_labels = cnn(images)
            loss = criterion(predict_labels, labels)
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
        print('epoch: % -3s loss: %s' % (epoch, loss.item()))
    torch.save(cnn.state_dict(), setting.MODEL_PATH /
               model_name)  # current is model.pkl
    print('save last model') 

def train(data, model, optimizer, epoch, args):
    model.train()
    loss_funct = nn.MultiLabelSoftMarginLoss()

    avg_loss = 0.0
    n_batches = 0
    progress_bar = tqdm(data)
    for batch_idx, sample_batched in enumerate(progress_bar):
        img, target = load_tensor_data(sample_batched, args.cuda, volatile=False)

        # forward and backward pass
        optimizer.zero_grad()
        output = model(img)
        loss = loss_funct(output, target)
        loss.backward()

        optimizer.step()

        # Show progress
        progress_bar.set_postfix(dict(loss=loss.data[0]))
        avg_loss += loss.data[0]
        n_batches += 1

        if batch_idx % args.log_interval == 0:
            avg_loss /= n_batches
            processed = batch_idx * args.batch_size
            n_samples = len(data) * args.batch_size
            progress = float(processed) / n_samples
            print('Train Epoch: {} [{}/{} ({:.0%})] Train loss: {}'.format(
                epoch, processed, n_samples, progress, avg_loss))
            avg_loss = 0.0
            n_batches = 0 

def __init__(self):
    logger.info('Set Data Loader')
    self.dataset = AnimeFaceDataset(avatar_tag_dat_path=avatar_tag_dat_path,
                                    transform=transforms.Compose([ToTensor()]))
    self.data_loader = torch.utils.data.DataLoader(self.dataset,
                                                   batch_size=batch_size,
                                                   shuffle=True,
                                                   num_workers=num_workers, drop_last=True)
    checkpoint, checkpoint_name = self.load_checkpoint(model_dump_path)
    if checkpoint == None:
      logger.info('Don\'t have pre-trained model. Ignore loading model process.')
      logger.info('Set Generator and Discriminator')
      self.G = Generator().to(device)
      self.D = Discriminator().to(device)
      logger.info('Initialize Weights')
      self.G.apply(initital_network_weights).to(device)
      self.D.apply(initital_network_weights).to(device)
      logger.info('Set Optimizers')
      self.optimizer_G = torch.optim.Adam(self.G.parameters(), lr=learning_rate, betas=(beta_1, 0.999))
      self.optimizer_D = torch.optim.Adam(self.D.parameters(), lr=learning_rate, betas=(beta_1, 0.999))
      self.epoch = 0
    else:
      logger.info('Load Generator and Discriminator')
      self.G = Generator().to(device)
      self.D = Discriminator().to(device)
      logger.info('Load Pre-Trained Weights From Checkpoint'.format(checkpoint_name))
      self.G.load_state_dict(checkpoint['G'])
      self.D.load_state_dict(checkpoint['D'])
      logger.info('Load Optimizers')
      self.optimizer_G = torch.optim.Adam(self.G.parameters(), lr=learning_rate, betas=(beta_1, 0.999))
      self.optimizer_D = torch.optim.Adam(self.D.parameters(), lr=learning_rate, betas=(beta_1, 0.999))
      self.optimizer_G.load_state_dict(checkpoint['optimizer_G'])
      self.optimizer_D.load_state_dict(checkpoint['optimizer_D'])
      self.epoch = checkpoint['epoch']
    logger.info('Set Criterion')
    self.label_criterion = nn.BCEWithLogitsLoss().to(device)
    self.tag_criterion = nn.MultiLabelSoftMarginLoss().to(device) 

def __init__(self, vocab_size, emb_mat):
        kernel_size = 3
        filters = 128
        super(Cnn, self).__init__()
        self.loss_fn = nn.MultiLabelSoftMarginLoss()
        self.encoder = nn.Embedding(vocab_size, conf.emb_size)
        self.title_conv = nn.Sequential(
            nn.Conv1d(conf.emb_size, filters, kernel_size=kernel_size),
            nn.BatchNorm1d(filters),
            nn.ReLU(inplace=True),
            nn.MaxPool1d(kernel_size=(conf.title_seq_len - 2 * kernel_size + 2))
        )
        self.content_conv = nn.Sequential(
            nn.Conv1d(conf.emb_size, filters, kernel_size=3),
            nn.BatchNorm1d(filters),
            nn.ReLU(inplace=True),
            nn.MaxPool1d(kernel_size=(conf.content_seq_len - 2 * kernel_size + 2))
        )
        self.fc = nn.Sequential(
            nn.Dropout(0.5),
            nn.Linear((filters + filters), 1024),
            nn.ReLU(inplace=True),
            nn.Dropout(0.25),
            nn.Linear(1024, conf.n_classes),
        )

        self.encoder.weight.data.copy_(emb_mat) 

def __init__(self, vocab_size, emb_mat):
        kernel_sizes = [1, 2, 3, 4]
        super(TextCNN, self).__init__()
        self.encoder = nn.Embedding(vocab_size, conf.emb_size)
        self.loss_fn = nn.MultiLabelSoftMarginLoss()

        title_convs = [nn.Sequential(
            nn.Conv1d(in_channels=conf.emb_size, out_channels=conf.emb_hidden_size, kernel_size=kernel_size),
            nn.BatchNorm1d(conf.emb_hidden_size),
            nn.ReLU(inplace=True),

            nn.Conv1d(in_channels=conf.emb_hidden_size, out_channels=conf.emb_hidden_size, kernel_size=kernel_size),
            nn.BatchNorm1d(conf.emb_hidden_size),
            nn.ReLU(inplace=True),
            nn.MaxPool1d(kernel_size=(conf.title_seq_len - kernel_size * 2 + 2))
        )
            for kernel_size in kernel_sizes]

        content_convs = [nn.Sequential(
            nn.Conv1d(in_channels=conf.emb_size, out_channels=conf.emb_hidden_size, kernel_size=kernel_size),
            nn.BatchNorm1d(conf.emb_hidden_size),
            nn.ReLU(inplace=True),

            nn.Conv1d(in_channels=conf.emb_hidden_size, out_channels=conf.emb_hidden_size, kernel_size=kernel_size),
            nn.BatchNorm1d(conf.emb_hidden_size),
            nn.ReLU(inplace=True),
            nn.MaxPool1d(kernel_size=(conf.content_seq_len - kernel_size * 2 + 2))
        )
            for kernel_size in kernel_sizes]

        self.title_convs = nn.ModuleList(title_convs)
        self.content_convs = nn.ModuleList(content_convs)

        self.fc = nn.Sequential(
            nn.Linear(len(kernel_sizes) * (conf.emb_hidden_size + conf.emb_hidden_size), conf.liner_size),
            nn.BatchNorm1d(conf.liner_size),
            nn.ReLU(inplace=True),
            nn.Linear(conf.liner_size, conf.n_classes)
        )

        self.encoder.weight.data.copy_(emb_mat) 

def __init__(self, vocab_size, emb_mat):
        self.k_max = 3
        super(LSTM, self).__init__()
        self.loss_fn = nn.MultiLabelSoftMarginLoss()
        self.encoder = nn.Embedding(vocab_size, conf.emb_size)

        self.title_lstm = nn.LSTM(conf.emb_size, conf.hidden_size, num_layers=3, bidirectional=True)
        self.content_lstm = nn.LSTM(conf.emb_size, conf.hidden_size, num_layers=3, bidirectional=True)
        self.fc = nn.Sequential(
            nn.Linear(conf.hidden_size * 4 * self.k_max, conf.liner_size),
            nn.BatchNorm1d(conf.liner_size),
            nn.ReLU(inplace=True),
            nn.Linear(conf.liner_size, conf.n_classes)
        )
        self.encoder.weight.data.copy_(emb_mat) 

def __init__(self, vocab_size, emb_mat):
        super(RCNN, self).__init__()
        self.k_max = 3
        self.loss_fn = nn.MultiLabelSoftMarginLoss()
        self.encoder = nn.Embedding(vocab_size, conf.emb_size)
        self.encoder.weight.data.copy_(emb_mat)

        self.title_lstm = nn.LSTM(conf.emb_size, conf.hidden_size,
                                  bidirectional=True, num_layers=conf.n_layers)
        self.title_conv = nn.Sequential(
            nn.Conv1d(conf.hidden_size * 2 + conf.emb_size, conf.title_dim, kernel_size=3),
            nn.BatchNorm1d(conf.title_dim),
            nn.ReLU(inplace=True),
            nn.Conv1d(conf.title_dim, conf.title_dim, kernel_size=3),
            nn.BatchNorm1d(conf.title_dim),
            nn.ReLU(inplace=True)
        )

        self.content_lstm = nn.LSTM(conf.emb_size, conf.hidden_size,
                                    bidirectional=True, num_layers=conf.n_layers)
        self.content_conv = nn.Sequential(
            nn.Conv1d(conf.hidden_size * 2 + conf.emb_size, conf.content_dim, kernel_size=3),
            nn.BatchNorm1d(conf.content_dim),
            nn.ReLU(inplace=True),
            nn.Conv1d(conf.content_dim, conf.content_dim, kernel_size=3),
            nn.BatchNorm1d(conf.content_dim),
            nn.ReLU(inplace=True)
        )

        self.fc = nn.Sequential(
            nn.Linear(self.k_max * (conf.title_dim + conf.content_dim), conf.liner_size),
            nn.BatchNorm1d(conf.liner_size),
            nn.ReLU(inplace=True),
            nn.Linear(conf.liner_size, conf.n_classes)
        ) 

def _init_data(self, manager, cfg):
        self.data_train = manager.create_dataset('train', cfg['batchsz'])
        self.data_valid = manager.create_dataset('val', cfg['batchsz'])
        self.data_test = manager.create_dataset('test', cfg['batchsz'])
        self.save_dir = cfg['save_dir']
        self.print_per_batch = cfg['print_per_batch']
        self.save_per_epoch = cfg['save_per_epoch']
        self.multi_entropy_loss = nn.MultiLabelSoftMarginLoss() 

def __init__(self, manager, cfg):
        self.data_train = manager.create_dataset('train', cfg['batchsz'])
        self.data_valid = manager.create_dataset('val', cfg['batchsz'])
        self.data_test = manager.create_dataset('test', cfg['batchsz'])
        self.save_dir = cfg['save_dir']
        self.print_per_batch = cfg['print_per_batch']
        self.save_per_epoch = cfg['save_per_epoch']

        voc_file = os.path.join(root_dir, 'data/crosswoz/sys_da_voc.json')
        voc_opp_file = os.path.join(root_dir, 'data/crosswoz/usr_da_voc.json')
        vector = CrossWozVector(voc_file, voc_opp_file)
        self.policy = MultiDiscretePolicy(vector.state_dim, cfg['h_dim'], vector.sys_da_dim).to(device=DEVICE)
        self.policy.eval()
        self.policy_optim = torch.optim.Adam(self.policy.parameters(), lr=cfg['lr'])
        self.multi_entropy_loss = nn.MultiLabelSoftMarginLoss() 

def main_voc2007():
    global args, best_prec1, use_gpu
    args = parser.parse_args()

    use_gpu = torch.cuda.is_available()

    # define dataset
    train_dataset = Voc2007Classification(args.data, 'trainval')
    val_dataset = Voc2007Classification(args.data, 'test')
    num_classes = 20

    # load model
    model = resnet101_wildcat(num_classes, pretrained=True, kmax=args.k, alpha=args.alpha, num_maps=args.maps)
    print('classifier', model.classifier)
    print('spatial pooling', model.spatial_pooling)

    # define loss function (criterion)
    criterion = nn.MultiLabelSoftMarginLoss()

    # define optimizer
    optimizer = torch.optim.SGD(model.get_config_optim(args.lr, args.lrp),
                                lr=args.lr,
                                momentum=args.momentum,
                                weight_decay=args.weight_decay)

    state = {'batch_size': args.batch_size, 'image_size': args.image_size, 'max_epochs': args.epochs,
             'evaluate': args.evaluate, 'resume': args.resume}
    state['difficult_examples'] = True
    state['save_model_path'] = '../expes/models/voc2007/'

    engine = MultiLabelMAPEngine(state)
    engine.learning(model, criterion, train_dataset, val_dataset, optimizer) 

def main():
    cnn = CNN()
    cnn.train()
    print('init net')
    criterion = nn.MultiLabelSoftMarginLoss()
    optimizer = torch.optim.Adam(cnn.parameters(), lr=learning_rate)

    # Train the Model
    train_dataloader = my_dataset.get_train_data_loader()
    for epoch in range(num_epochs):
        for i, (images, labels) in enumerate(train_dataloader):
            images = Variable(images)
            labels = Variable(labels.float())
            predict_labels = cnn(images)
            # print(predict_labels.type)
            # print(labels.type)
            loss = criterion(predict_labels, labels)
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
            if (i+1) % 10 == 0:
                print("epoch:", epoch, "step:", i, "loss:", loss.item())
            if (i+1) % 100 == 0:
                torch.save(cnn.state_dict(), "./model.pkl")   #current is model.pkl
                print("save model")
        print("epoch:", epoch, "step:", i, "loss:", loss.item())
    torch.save(cnn.state_dict(), "./model.pkl")   #current is model.pkl
    print("save last model") 

def main(args):
    global C, H, W
    coco_labels = json.load(open(args.coco_labels))
    num_classes = coco_labels['num_classes']
    if args.model == 'inception_v3':
        C, H, W = 3, 299, 299
        model = pretrainedmodels.inceptionv3(pretrained='imagenet')

    elif args.model == 'resnet152':
        C, H, W = 3, 224, 224
        model = pretrainedmodels.resnet152(pretrained='imagenet')

    elif args.model == 'inception_v4':
        C, H, W = 3, 299, 299
        model = pretrainedmodels.inceptionv4(
            num_classes=1000, pretrained='imagenet')

    else:
        print("doesn't support %s" % (args['model']))

    load_image_fn = utils.LoadTransformImage(model)
    dim_feats = model.last_linear.in_features
    model = MILModel(model, dim_feats, num_classes)
    model = model.cuda()
    dataset = CocoDataset(coco_labels)
    dataloader = DataLoader(
        dataset, batch_size=args.batch_size, shuffle=True)
    optimizer = optim.Adam(
        model.parameters(), lr=args.learning_rate, weight_decay=args.weight_decay)
    exp_lr_scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=args.learning_rate_decay_every,
                                                 gamma=args.learning_rate_decay_rate)

    crit = nn.MultiLabelSoftMarginLoss()
    if not os.path.isdir(args.checkpoint_path):
        os.mkdir(args.checkpoint_path)
    train(dataloader, model, crit, optimizer,
          exp_lr_scheduler, load_image_fn, args) 

def main(args):
    cnn = CNN().to(device)
    
    cnn.train()
    criterion = nn.MultiLabelSoftMarginLoss()
    optimizer = torch.optim.Adam(cnn.parameters(), lr=learning_rate)

    if args.resume:
        cnn.load_state_dict(torch.load(args.model_path, map_location=device))

    max_acc = 0
    # Train the Model
    train_dataloader = datasets.get_train_data_loader()
    for epoch in range(num_epochs):
        for i, (images, labels) in enumerate(train_dataloader):
            images = Variable(images)
            labels = Variable(labels.float())
            predict_labels = cnn(images)
            loss = criterion(predict_labels, labels)
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
            if (i+1) % 2 == 0:
                print("epoch: %03g \t step: %03g \t loss: %.5f \t\r" % (epoch, i+1, loss.item()))
                torch.save(cnn.state_dict(), "./weights/cnn_%03g.pt" % epoch)
        print("epoch: %03g \t step: %03g \t loss: %.5f \t" % (epoch, i, loss.item()))
        torch.save(cnn.state_dict(), "./weights/cnn_%03g.pt" % epoch)
        acc = test.test_data("./weights/cnn_%03g.pt" % epoch)
        if max_acc < acc:
            print("update accuracy %.5f." % acc)
            max_acc = acc
            shutil.copy("./weights/cnn_%03g.pt" % epoch, "./weights/cnn_best.pt")
        else:
            print("do not update %.5f." % acc)
        
    torch.save(cnn.state_dict(), "./weights/cnn_last.pt")
    print("save last model") 

def main_voc2007():
    global args, best_prec1, use_gpu 
    args = parser.parse_args()

    use_gpu = torch.cuda.is_available()

    # define dataset
    train_dataset = Voc2007Classification(args.data, 'trainval')
    val_dataset = Voc2007Classification(args.data, 'test')
    num_classes = 20

    # load model
    model = vgg16_sp(num_classes, pretrained=True)
    
    print(model)

    criterion = nn.MultiLabelSoftMarginLoss()

    state = {'batch_size': args.batch_size, 'max_epochs': args.epochs, 
            'image_size': args.image_size, 'evaluate': args.evaluate, 'resume': args.resume,
             'lr':args.lr, 'momentum':args.momentum, 'weight_decay':args.weight_decay}
    state['difficult_examples'] = True
    state['save_model_path'] = 'logs/voc2007/'

    engine = MultiLabelMAPEngine(state)
    engine.multi_learning(model, criterion, train_dataset, val_dataset) 

def __init__(self, vocab_size, emb_mat):
        kernel_size = 3
        filters = 128
        super(CnnRes, self).__init__()
        self.loss_fn = nn.MultiLabelSoftMarginLoss()
        self.encoder = nn.Embedding(vocab_size, conf.emb_size)

        self.title_conv1 = nn.Sequential(
            nn.Conv1d(conf.emb_size, filters, kernel_size=kernel_size, ),
            nn.BatchNorm1d(filters),
            nn.ReLU(inplace=True),
            nn.Conv1d(filters, conf.emb_size, kernel_size=kernel_size),
            nn.BatchNorm1d(conf.emb_size),
        )

        self.title_conv2 = nn.Sequential(
            nn.Conv1d(conf.emb_size, filters * 2, kernel_size=kernel_size),
            nn.BatchNorm1d(filters * 2),
        )
        self.title_conv3 = nn.Sequential(
            nn.Conv1d(filters * 2, filters * 4, kernel_size=kernel_size),
            nn.BatchNorm1d(filters * 4),
            nn.ReLU(inplace=True),
            nn.MaxPool1d(kernel_size=(conf.title_seq_len - 2 * kernel_size + 2))
        )

        self.content_conv1 = nn.Sequential(
            nn.Conv1d(conf.emb_size, filters, kernel_size=kernel_size),
            nn.BatchNorm1d(filters),
            nn.ReLU(inplace=True),
            nn.Conv1d(filters, conf.emb_size, kernel_size=kernel_size),
            nn.BatchNorm1d(conf.emb_size),
        )

        self.content_conv2 = nn.Sequential(
            nn.Conv1d(conf.emb_size, filters * 2, kernel_size=kernel_size),
            nn.BatchNorm1d(filters * 2),
        )
        self.content_conv3 = nn.Sequential(
            nn.Conv1d(filters * 2, filters * 4, kernel_size=kernel_size),
            nn.BatchNorm1d(filters * 4),
            nn.ReLU(inplace=True),
            nn.MaxPool1d(kernel_size=(conf.content_seq_len - 2 * kernel_size + 2))
        )

        self.fc = nn.Sequential(
            nn.Dropout(0.5),
            nn.Linear(4096, 1024),
            nn.ReLU(inplace=True),
            nn.Dropout(0.25),
            nn.Linear(1024, conf.n_classes),
        )

        self.encoder.weight.data.copy_(emb_mat)