Python torch.optim.ASGD Examples
The following are 14
code examples of torch.optim.ASGD().
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example.
You may also want to check out all available functions/classes of the module
torch.optim
, or try the search function
.
Example #1
Source File: optimization.py From SeaRNN-open with MIT License | 6 votes |
def create_optimizer(parameters, opt): lr = opt.learning_rate # default learning rates: # sgd - 0.5, adagrad - 0.01, adadelta - 1, adam - 0.001, adamax - 0.002, asgd - 0.01, rmsprop - 0.01, rprop - 0.01 optim_method = opt.optim_method.casefold() if optim_method == 'sgd': optimizer = optim.SGD(parameters, lr=lr if lr else 0.5, weight_decay=opt.weight_decay) elif optim_method == 'adagrad': optimizer = optim.Adagrad(parameters, lr=lr if lr else 0.01, weight_decay=opt.weight_decay) elif optim_method == 'adadelta': optimizer = optim.Adadelta(parameters, lr=lr if lr else 1, weight_decay=opt.weight_decay) elif optim_method == 'adam': optimizer = optim.Adam(parameters, lr=lr if lr else 0.001, weight_decay=opt.weight_decay) elif optim_method == 'adamax': optimizer = optim.Adamax(parameters, lr=lr if lr else 0.002, weight_decay=opt.weight_decay) elif optim_method == 'asgd': optimizer = optim.ASGD(parameters, lr=lr if lr else 0.01, t0=5000, weight_decay=opt.weight_decay) elif optim_method == 'rmsprop': optimizer = optim.RMSprop(parameters, lr=lr if lr else 0.01, weight_decay=opt.weight_decay) elif optim_method == 'rprop': optimizer = optim.Rprop(parameters, lr=lr if lr else 0.01) else: raise RuntimeError("Invalid optim method: " + opt.optim_method) return optimizer
Example #2
Source File: make_optimizer.py From MAMS-for-ABSA with Apache License 2.0 | 6 votes |
def make_optimizer(config, model): mode = config['mode'] config = config['aspect_' + mode + '_model'][config['aspect_' + mode + '_model']['type']] lr = config['learning_rate'] weight_decay = config['weight_decay'] opt = { 'sgd': optim.SGD, 'adadelta': optim.Adadelta, 'adam': optim.Adam, 'adamax': optim.Adamax, 'adagrad': optim.Adagrad, 'asgd': optim.ASGD, 'rmsprop': optim.RMSprop, 'adabound': adabound.AdaBound } if 'momentum' in config: optimizer = opt[config['optimizer']](model.parameters(), lr=lr, weight_decay=weight_decay, momentum=config['momentum']) else: optimizer = opt[config['optimizer']](model.parameters(), lr=lr, weight_decay=weight_decay) return optimizer
Example #3
Source File: optimizer.py From XenonPy with BSD 3-Clause "New" or "Revised" License | 6 votes |
def __init__(self, *, lr=0.002, lambd=0.0001, alpha=0.75, t0=1000000.0, weight_decay=0): """Implements Averaged Stochastic Gradient Descent. It has been proposed in `Acceleration of stochastic approximation by averaging`_. Arguments: lr (float, optional): learning rate (default: 1e-2) lambd (float, optional): decay term (default: 1e-4) alpha (float, optional): power for eta update (default: 0.75) t0 (float, optional): point at which to start averaging (default: 1e6) weight_decay (float, optional): weight decay (L2 penalty) (default: 0) .. _Acceleration of stochastic approximation by averaging: http://dl.acm.org/citation.cfm?id=131098 """ super().__init__(optim.ASGD, lr=lr, lambd=lambd, alpha=alpha, t0=t0, weight_decay=weight_decay)
Example #4
Source File: get_optimizer.py From PyMIC with Apache License 2.0 | 5 votes |
def get_optimiser(name, net_params, optim_params): lr = optim_params['learning_rate'] momentum = optim_params['momentum'] weight_decay = optim_params['weight_decay'] if(name == "SGD"): return optim.SGD(net_params, lr, momentum = momentum, weight_decay = weight_decay) elif(name == "Adam"): return optim.Adam(net_params, lr, weight_decay = 1e-5) elif(name == "SparseAdam"): return optim.SparseAdam(net_params, lr) elif(name == "Adadelta"): return optim.Adadelta(net_params, lr, weight_decay = weight_decay) elif(name == "Adagrad"): return optim.Adagrad(net_params, lr, weight_decay = weight_decay) elif(name == "Adamax"): return optim.Adamax(net_params, lr, weight_decay = weight_decay) elif(name == "ASGD"): return optim.ASGD(net_params, lr, weight_decay = weight_decay) elif(name == "LBFGS"): return optim.LBFGS(net_params, lr) elif(name == "RMSprop"): return optim.RMSprop(net_params, lr, momentum = momentum, weight_decay = weight_decay) elif(name == "Rprop"): return optim.Rprop(net_params, lr) else: raise ValueError("unsupported optimizer {0:}".format(name))
Example #5
Source File: utils.py From DeMa-BWE with BSD 3-Clause "New" or "Revised" License | 5 votes |
def get_optimizer(s): """ Parse optimizer parameters. Input should be of the form: - "sgd,lr=0.01" - "adagrad,lr=0.1,lr_decay=0.05" """ if "," in s: method = s[:s.find(',')] optim_params = {} for x in s[s.find(',') + 1:].split(','): split = x.split('=') assert len(split) == 2 assert re.match("^[+-]?(\d+(\.\d*)?|\.\d+)$", split[1]) is not None optim_params[split[0]] = float(split[1]) else: method = s optim_params = {} if method == 'adadelta': optim_fn = optim.Adadelta elif method == 'adagrad': optim_fn = optim.Adagrad elif method == 'adam': optim_fn = optim.Adam elif method == 'adamax': optim_fn = optim.Adamax elif method == 'asgd': optim_fn = optim.ASGD elif method == 'rmsprop': optim_fn = optim.RMSprop elif method == 'rprop': optim_fn = optim.Rprop elif method == 'sgd': optim_fn = optim.SGD assert 'lr' in optim_params else: raise Exception('Unknown optimization method: "%s"' % method) return optim_fn, optim_params
Example #6
Source File: model.py From DiPS with Apache License 2.0 | 5 votes |
def _optim(self): self.params = list(self.encoder.parameters()) + list(self.decoder.parameters()) if self.config.opt == 'adam': self.optimizer = optim.Adam(self.params, lr=self.config.lr) elif self.config.opt == 'adadelta': self.optimizer = optim.Adadelta(self.params, lr=self.config.lr) elif self.config.opt == 'asgd': self.optimizer = optim.ASGD(self.params, lr=self.config.lr) else: self.optimizer = optim.SGD(self.params, lr=self.config.lr)
Example #7
Source File: learner.py From emmental with MIT License | 4 votes |
def _set_optimizer(self, model: EmmentalModel) -> None: """Set optimizer for learning process. Args: model: The model to set up the optimizer. """ optimizer_config = Meta.config["learner_config"]["optimizer_config"] opt = optimizer_config["optimizer"] # If Meta.config["learner_config"]["optimizer_config"]["parameters"] is None, # create a parameter group with all parameters in the model, else load user # specified parameter groups. if optimizer_config["parameters"] is None: parameters = filter(lambda p: p.requires_grad, model.parameters()) else: parameters = optimizer_config["parameters"](model) optim_dict = { # PyTorch optimizer "asgd": optim.ASGD, # type: ignore "adadelta": optim.Adadelta, # type: ignore "adagrad": optim.Adagrad, # type: ignore "adam": optim.Adam, # type: ignore "adamw": optim.AdamW, # type: ignore "adamax": optim.Adamax, # type: ignore "lbfgs": optim.LBFGS, # type: ignore "rms_prop": optim.RMSprop, # type: ignore "r_prop": optim.Rprop, # type: ignore "sgd": optim.SGD, # type: ignore "sparse_adam": optim.SparseAdam, # type: ignore # Customize optimizer "bert_adam": BertAdam, } if opt in ["lbfgs", "r_prop", "sparse_adam"]: optimizer = optim_dict[opt]( parameters, lr=optimizer_config["lr"], **optimizer_config[f"{opt}_config"], ) elif opt in optim_dict.keys(): optimizer = optim_dict[opt]( parameters, lr=optimizer_config["lr"], weight_decay=optimizer_config["l2"], **optimizer_config[f"{opt}_config"], ) elif isinstance(opt, optim.Optimizer): # type: ignore optimizer = opt(parameters) else: raise ValueError(f"Unrecognized optimizer option '{opt}'") self.optimizer = optimizer if Meta.config["meta_config"]["verbose"]: logger.info(f"Using optimizer {self.optimizer}")
Example #8
Source File: mutils.py From e-SNLI with MIT License | 4 votes |
def get_optimizer(s): """ Parse optimizer parameters. Input should be of the form: - "sgd,lr=0.01" - "adagrad,lr=0.1,lr_decay=0.05" """ if "," in s: method = s[:s.find(',')] optim_params = {} for x in s[s.find(',') + 1:].split(','): split = x.split('=') assert len(split) == 2 assert re.match("^[+-]?(\d+(\.\d*)?|\.\d+)$", split[1]) is not None optim_params[split[0]] = float(split[1]) else: method = s optim_params = {} if method == 'adadelta': optim_fn = optim.Adadelta elif method == 'adagrad': optim_fn = optim.Adagrad elif method == 'adam': optim_fn = optim.Adam elif method == 'adamax': optim_fn = optim.Adamax elif method == 'asgd': optim_fn = optim.ASGD elif method == 'rmsprop': optim_fn = optim.RMSprop elif method == 'rprop': optim_fn = optim.Rprop elif method == 'sgd': optim_fn = optim.SGD assert 'lr' in optim_params else: raise Exception('Unknown optimization method: "%s"' % method) # check that we give good parameters to the optimizer expected_args = inspect.getargspec(optim_fn.__init__)[0] assert expected_args[:2] == ['self', 'params'] if not all(k in expected_args[2:] for k in optim_params.keys()): raise Exception('Unexpected parameters: expected "%s", got "%s"' % ( str(expected_args[2:]), str(optim_params.keys()))) return optim_fn, optim_params
Example #9
Source File: utils.py From e-SNLI with MIT License | 4 votes |
def get_optimizer(s): """ Parse optimizer parameters. Input should be of the form: - "sgd,lr=0.01" - "adagrad,lr=0.1,lr_decay=0.05" """ if "," in s: method = s[:s.find(',')] optim_params = {} for x in s[s.find(',') + 1:].split(','): split = x.split('=') assert len(split) == 2 assert re.match("^[+-]?(\d+(\.\d*)?|\.\d+)$", split[1]) is not None optim_params[split[0]] = float(split[1]) else: method = s optim_params = {} if method == 'adadelta': optim_fn = optim.Adadelta elif method == 'adagrad': optim_fn = optim.Adagrad elif method == 'adam': optim_fn = optim.Adam elif method == 'adamax': optim_fn = optim.Adamax elif method == 'asgd': optim_fn = optim.ASGD elif method == 'rmsprop': optim_fn = optim.RMSprop elif method == 'rprop': optim_fn = optim.Rprop elif method == 'sgd': optim_fn = optim.SGD assert 'lr' in optim_params else: raise Exception('Unknown optimization method: "%s"' % method) # check that we give good parameters to the optimizer expected_args = inspect.getargspec(optim_fn.__init__)[0] assert expected_args[:2] == ['self', 'params'] if not all(k in expected_args[2:] for k in optim_params.keys()): raise Exception('Unexpected parameters: expected "%s", got "%s"' % ( str(expected_args[2:]), str(optim_params.keys()))) return optim_fn, optim_params
Example #10
Source File: mutils.py From neural_chat with MIT License | 4 votes |
def get_optimizer(s): """ Parse optimizer parameters. Input should be of the form: - "sgd,lr=0.01" - "adagrad,lr=0.1,lr_decay=0.05" """ if "," in s: method = s[:s.find(',')] optim_params = {} for x in s[s.find(',') + 1:].split(','): split = x.split('=') assert len(split) == 2 assert re.match("^[+-]?(\d+(\.\d*)?|\.\d+)$", split[1]) is not None optim_params[split[0]] = float(split[1]) else: method = s optim_params = {} if method == 'adadelta': optim_fn = optim.Adadelta elif method == 'adagrad': optim_fn = optim.Adagrad elif method == 'adam': optim_fn = optim.Adam elif method == 'adamax': optim_fn = optim.Adamax elif method == 'asgd': optim_fn = optim.ASGD elif method == 'rmsprop': optim_fn = optim.RMSprop elif method == 'rprop': optim_fn = optim.Rprop elif method == 'sgd': optim_fn = optim.SGD assert 'lr' in optim_params else: raise Exception('Unknown optimization method: "%s"' % method) # check that we give good parameters to the optimizer expected_args = inspect.getargspec(optim_fn.__init__)[0] assert expected_args[:2] == ['self', 'params'] if not all(k in expected_args[2:] for k in optim_params.keys()): raise Exception('Unexpected parameters: expected "%s", got "%s"' % ( str(expected_args[2:]), str(optim_params.keys()))) return optim_fn, optim_params
Example #11
Source File: utils.py From dl4mt-seqgen with BSD 3-Clause "New" or "Revised" License | 4 votes |
def get_optimizer(parameters, s): """ Parse optimizer parameters. Input should be of the form: - "sgd,lr=0.01" - "adagrad,lr=0.1,lr_decay=0.05" """ if "," in s: method = s[:s.find(',')] optim_params = {} for x in s[s.find(',') + 1:].split(','): split = x.split('=') assert len(split) == 2 assert re.match("^[+-]?(\d+(\.\d*)?|\.\d+)$", split[1]) is not None optim_params[split[0]] = float(split[1]) else: method = s optim_params = {} if method == 'adadelta': optim_fn = optim.Adadelta elif method == 'adagrad': optim_fn = optim.Adagrad elif method == 'adam': optim_fn = optim.Adam optim_params['betas'] = (optim_params.get('beta1', 0.9), optim_params.get('beta2', 0.999)) optim_params.pop('beta1', None) optim_params.pop('beta2', None) elif method == 'adam_inverse_sqrt': optim_fn = AdamInverseSqrtWithWarmup optim_params['betas'] = (optim_params.get('beta1', 0.9), optim_params.get('beta2', 0.999)) optim_params.pop('beta1', None) optim_params.pop('beta2', None) elif method == 'adamax': optim_fn = optim.Adamax elif method == 'asgd': optim_fn = optim.ASGD elif method == 'rmsprop': optim_fn = optim.RMSprop elif method == 'rprop': optim_fn = optim.Rprop elif method == 'sgd': optim_fn = optim.SGD assert 'lr' in optim_params else: raise Exception('Unknown optimization method: "%s"' % method) # check that we give good parameters to the optimizer expected_args = inspect.getargspec(optim_fn.__init__)[0] assert expected_args[:2] == ['self', 'params'] if not all(k in expected_args[2:] for k in optim_params.keys()): raise Exception('Unexpected parameters: expected "%s", got "%s"' % ( str(expected_args[2:]), str(optim_params.keys()))) return optim_fn(parameters, **optim_params)
Example #12
Source File: utils.py From transferable_sent2vec with MIT License | 4 votes |
def get_optimizer(s): """ Parse optimizer parameters. Input should be of the form: - "sgd,lr=0.01" - "adagrad,lr=0.1,lr_decay=0.05" """ if "," in s: method = s[:s.find(',')] optim_params = {} for x in s[s.find(',') + 1:].split(','): split = x.split('=') assert len(split) == 2 assert re.match("^[+-]?(\d+(\.\d*)?|\.\d+)$", split[1]) is not None optim_params[split[0]] = float(split[1]) else: method = s optim_params = {} if method == 'adadelta': optim_fn = optim.Adadelta elif method == 'adagrad': optim_fn = optim.Adagrad elif method == 'adam': optim_fn = optim.Adam elif method == 'adamax': optim_fn = optim.Adamax elif method == 'asgd': optim_fn = optim.ASGD elif method == 'rmsprop': optim_fn = optim.RMSprop elif method == 'rprop': optim_fn = optim.Rprop elif method == 'sgd': optim_fn = optim.SGD assert 'lr' in optim_params else: raise Exception('Unknown optimization method: "%s"' % method) # check that we give good parameters to the optimizer expected_args = inspect.getargspec(optim_fn.__init__)[0] assert expected_args[:2] == ['self', 'params'] if not all(k in expected_args[2:] for k in optim_params.keys()): raise Exception('Unexpected parameters: expected "%s", got "%s"' % ( str(expected_args[2:]), str(optim_params.keys()))) return optim_fn, optim_params
Example #13
Source File: mutils.py From T2F with MIT License | 4 votes |
def get_optimizer(s): """ Parse optimizer parameters. Input should be of the form: - "sgd,lr=0.01" - "adagrad,lr=0.1,lr_decay=0.05" """ if "," in s: method = s[:s.find(',')] optim_params = {} for x in s[s.find(',') + 1:].split(','): split = x.split('=') assert len(split) == 2 assert re.match("^[+-]?(\d+(\.\d*)?|\.\d+)$", split[1]) is not None optim_params[split[0]] = float(split[1]) else: method = s optim_params = {} if method == 'adadelta': optim_fn = optim.Adadelta elif method == 'adagrad': optim_fn = optim.Adagrad elif method == 'adam': optim_fn = optim.Adam elif method == 'adamax': optim_fn = optim.Adamax elif method == 'asgd': optim_fn = optim.ASGD elif method == 'rmsprop': optim_fn = optim.RMSprop elif method == 'rprop': optim_fn = optim.Rprop elif method == 'sgd': optim_fn = optim.SGD assert 'lr' in optim_params else: raise Exception('Unknown optimization method: "%s"' % method) # check that we give good parameters to the optimizer expected_args = inspect.getargspec(optim_fn.__init__)[0] assert expected_args[:2] == ['self', 'params'] if not all(k in expected_args[2:] for k in optim_params.keys()): raise Exception('Unexpected parameters: expected "%s", got "%s"' % ( str(expected_args[2:]), str(optim_params.keys()))) return optim_fn, optim_params
Example #14
Source File: utils.py From nlp-recipes with MIT License | 4 votes |
def get_optimizer(s): """ Parse optimizer parameters. Input should be of the form: - "sgd,lr=0.01" - "adagrad,lr=0.1,lr_decay=0.05" """ if "," in s: method = s[:s.find(',')] optim_params = {} for x in s[s.find(',') + 1:].split(','): split = x.split('=') assert len(split) == 2 assert re.match("^[+-]?(\d+(\.\d*)?|\.\d+)$", split[1]) is not None optim_params[split[0]] = float(split[1]) else: method = s optim_params = {} if method == 'adadelta': optim_fn = optim.Adadelta elif method == 'adagrad': optim_fn = optim.Adagrad elif method == 'adam': optim_fn = optim.Adam elif method == 'adamax': optim_fn = optim.Adamax elif method == 'asgd': optim_fn = optim.ASGD elif method == 'rmsprop': optim_fn = optim.RMSprop elif method == 'rprop': optim_fn = optim.Rprop elif method == 'sgd': optim_fn = optim.SGD assert 'lr' in optim_params else: raise Exception('Unknown optimization method: "%s"' % method) # check that we give good parameters to the optimizer expected_args = inspect.getfullargspec(optim_fn.__init__)[0] assert expected_args[:2] == ['self', 'params'] if not all(k in expected_args[2:] for k in optim_params.keys()): raise Exception('Unexpected parameters: expected "%s", got "%s"' % ( str(expected_args[2:]), str(optim_params.keys()))) return optim_fn, optim_params