import os import time import copy import argparse import math import numpy as np import torch import torch.nn as nn from torch.autograd import Variable import torch.nn.functional as F import utils import matplotlib matplotlib.use('Agg') import matplotlib.pyplot as plt ee = 1e-8 MAX_DIST = 127 def harmonic_mean(a, b): return 2.*a*b/(a+b) if a > 0. and b > 0. else 0. class Identity(nn.Module): def forward(self, x): return x def test(work_name, data_loader, model, T, opts, save_path, start_time=time.time()): model.eval() if opts.test_data_norm: novel_biases = np.concatenate([-(2.**np.arange(4.,-3.-ee,-1.)), np.arange(-0.1,0.2+ee,0.002), 2.**np.arange(-2.,4.+ee,1.)]) else: novel_biases = np.concatenate([-(2.**np.arange(4.,2.-ee,-1.)), np.arange(-2.5,5.+ee,0.05), 2.**np.arange(3.,4.+ee,1.)]) num_leaves = len(T['wnids_leaf']) num_classes = len(T['wnids']) num_labels = len(T['label_hnd']) # len(T['wnids_leaf']) + len(T['wnids_novel']) num_pts = novel_biases.shape[0] classes = {'known': np.arange(num_leaves), 'novel': np.arange(num_leaves, num_labels), 'super': np.arange(num_leaves, num_classes)} dtypes = ['val'] if work_name == 'val' else ['known', 'novel'] hierarchical_measure = T.get('dist_mat') is not None counters = init_counters(num_pts, num_classes if work_name == 'val' else num_labels, hierarchical_measure) results = init_results(num_pts, None, hierarchical_measure) labels = {'known': [], 'novel': []} preds = {'known': [], 'novel': []} novel_classes = torch.LongTensor(np.arange(num_leaves, num_classes)) if opts.gpu: novel_classes = novel_classes.cuda() for dtype in dtypes: num_data = len(data_loader[dtype].dataset) labels[dtype] = -np.ones(num_data, dtype=int) preds[dtype] = -np.ones([num_pts, num_data], dtype=int) for i, (inputs, targets) in enumerate(data_loader[dtype]): inputs = Variable(inputs, volatile=True) if opts.gpu: inputs, targets = inputs.cuda(), targets.cuda() logits = model(inputs) pos = i*opts.batch_size labels[dtype][pos:pos+targets.size(0)] = targets.cpu().numpy() for b, novel_bias in enumerate(novel_biases): # flatten prediction logits_biased = logits.clone() logits_biased[:, novel_classes] += novel_bias preds_ = logits_biased.max(dim=1)[1].data.cpu().numpy() preds[dtype][b, pos:pos+targets.size(0)] = preds_ if work_name == 'val': # known performance evaluation with full taxonomy count_test(b, counters, preds_, targets, T, hierarchical_measure) # novel performance evaluation with deficient taxonomy count_val(b, counters, logits_biased, targets, T, opts.gpu, hierarchical_measure) else: # performance evaluation with full taxonomy count_test(b, counters, preds_, targets, T, hierarchical_measure) # print('{dtype} eval {time:8.3f} s'.format(dtype=dtype, time=time.time()-start_time)) # acc counters_to_results(counters, results, classes) results_to_guarantee(results, novel_biases, opts.known_guarantee) print_results(work_name, results, save_path, hierarchical_measure, start_time) if opts.save_results: counters['novel_biases'] = novel_biases results['novel_biases'] = novel_biases preds['novel_biases'] = novel_biases np.save('{save_path}_{work_name}_counters.npy'.format(save_path=save_path, work_name=work_name), counters) np.save('{save_path}_{work_name}_results.npy'.format(save_path=save_path, work_name=work_name), results) np.save('{save_path}_{work_name}_preds_all.npy'.format(save_path=save_path, work_name=work_name), preds) if work_name == 'test': np.save('{save_path}_{work_name}_labels.npy'.format(save_path=save_path, work_name=work_name), labels) preds_opt = {'known': preds['known'][results['acc']['loc']], 'novel': preds['novel'][results['acc']['loc']]} np.save('{save_path}_{work_name}_preds.npy'.format(save_path=save_path, work_name=work_name), preds_opt) def val_td(data_loader, model, T, opts, save_path, start_time=time.time()): model.eval() work_name = 'val' ths = np.concatenate([[0.], 2.**np.arange(-10,2.+ee,.5)]) num_leaves = len(T['wnids_leaf']) num_classes = len(T['wnids']) num_labels = len(T['label_hnd']) # len(T['wnids_leaf']) + len(T['wnids_novel']) num_pts = ths.shape[0] classes = {'known': np.arange(num_leaves), 'novel': np.arange(num_leaves, num_labels)} dtypes = ['val'] num_models = len(model) num_children = T['num_children'] multi_inds = T['multi_inds'] classifiable = T['classifiable'] hierarchical_measure = T.get('dist_mat') is not None counters = init_counters(num_pts, num_children, hierarchical_measure) results = init_results(num_pts, num_models, hierarchical_measure) for dtype in dtypes: for i, (inputs, targets) in enumerate(data_loader[dtype]): inputs = Variable(inputs, volatile=True) if opts.gpu: inputs, targets = inputs.cuda(), targets.cuda() logits = [[]]*num_models kld_u = [[]]*num_models for m, sub in enumerate(model): # filter classifiable labels b_relevant = torch.zeros(targets.size(), out=torch.ByteTensor()) if opts.gpu: b_relevant = b_relevant.cuda() for k in classifiable[m]: b_relevant |= (targets == k) if not b_relevant.any(): continue relevant = b_relevant.nonzero().view(-1) relevant_labels = targets[relevant].cpu().numpy() # feedforward logits[m] = sub(inputs[relevant]) if opts.test_relu: logits[m] = F.relu(logits[m]) if opts.test_data_norm: logits[m] = F.normalize(logits[m], p=2, dim=1) kld_u[m] = - math.log(num_children[m]) \ - F.log_softmax(logits[m], dim=1).data.sum(dim=1) / num_children[m] logits[m] = logits[m].data.cpu().numpy() kld_u[m] = kld_u[m].cpu().numpy() for t, th in enumerate(ths): # prediction in super class b_conf = kld_u[m] > th # assert (kld_u[m] > -1e-4).all(), 'some kld negative' preds_ = -1*np.ones_like(relevant_labels) if b_conf.any(): preds_[b_conf] = logits[m][b_conf].argmax(axis=1) # performance evaluation count_super(t, m, counters, preds_, relevant_labels, multi_inds[m]) # print('{dtype} eval {time:8.3f} s'.format(dtype=dtype, time=time.time()-start_time)) counters_to_results_super(counters, results, ths, num_models) print_results_super(work_name, results, save_path, start_time) if opts.save_results: counters['ths'] = ths results['ths'] = ths np.save('{save_path}_{work_name}_counters.npy'.format(save_path=save_path, work_name=work_name), counters) np.save('{save_path}_{work_name}_results.npy'.format(save_path=save_path, work_name=work_name), results) return results['ths_opt'] def test_td(ths_opt, data_loader, model, T, opts, save_path, start_time=time.time()): model.eval() work_name = 'test' num_leaves = len(T['wnids_leaf']) num_classes = len(T['wnids']) num_labels = len(T['label_hnd']) # len(T['wnids_leaf']) + len(T['wnids_novel']) classes = {'known': np.arange(num_leaves), 'novel': np.arange(num_leaves, num_labels)} dtypes = ['known', 'novel'] num_models = len(model) num_children = T['num_children'] children = [np.array(ch) for ch in T['children']] root = T['root'] if not isinstance(ths_opt, (np.ndarray, list)): ths_opt = ths_opt*np.ones(num_models) hierarchical_measure = T.get('dist_mat') is not None counters = init_counters(1, num_labels, hierarchical_measure) results = init_results(1, None, hierarchical_measure) labels = {'known': [], 'novel': []} preds = {'known': [], 'novel': []} cls_order = [root] m = 0 while m < len(cls_order): for ch in children[cls_order[m]]: if ch >= num_leaves and ch not in cls_order: cls_order.append(ch) m += 1 assert len(cls_order) == num_models, 'some classes are unvisitable' for dtype in dtypes: num_data = len(data_loader[dtype].dataset) labels[dtype] = -np.ones(num_data, dtype=int) preds[dtype] = -np.ones(num_data, dtype=int) for i, (inputs, targets) in enumerate(data_loader[dtype]): inputs = Variable(inputs, volatile=True) if opts.gpu: inputs, targets = inputs.cuda(), targets.cuda() pos = i*opts.batch_size labels[dtype][pos:pos+targets.size(0)] = targets.cpu().numpy() logits = [[]]*num_models kld_u = [[]]*num_models # feedforward for m, sub in enumerate(model): logits[m] = sub(inputs) if opts.test_relu: logits[m] = F.relu(logits[m]) if opts.test_data_norm: logits[m] = F.normalize(logits[m], p=2, dim=1) kld_u[m] = - math.log(num_children[m]) \ - F.log_softmax(logits[m], dim=1).data.sum(dim=1) / num_children[m] logits[m] = logits[m].data.cpu().numpy() kld_u[m] = kld_u[m].cpu().numpy() # top-down prediction preds_ = root*np.ones_like(targets.cpu().numpy()) for k in cls_order: b_relevant = (preds_ == k) if not b_relevant.any(): continue m = k - num_leaves b_conf = kld_u[m][b_relevant] > ths_opt[m] # assert (kld_u[m][b_relevant] > -1e-4).all(), 'some kld negative' if not b_conf.any(): continue relevant = b_relevant.nonzero()[0][b_conf] preds_[relevant] = children[k][logits[m][relevant].argmax(axis=1)] preds[dtype][pos:pos+targets.size(0)] = preds_ # performance evaluation with full taxonomy count_test(0, counters, preds_, targets, T, hierarchical_measure) # print('{dtype} eval {time:8.3f} s'.format(dtype=dtype, time=time.time()-start_time)) # acc counters_to_results(counters, results, classes) results_to_guarantee(results, [ths_opt[0]], opts.known_guarantee) print_results(work_name, results, save_path, hierarchical_measure, start_time) if opts.save_results: counters['ths'] = ths_opt results['ths'] = ths_opt preds['ths'] = ths_opt np.save('{save_path}_{work_name}_counters.npy'.format(save_path=save_path, work_name=work_name), counters) np.save('{save_path}_{work_name}_results.npy'.format(save_path=save_path, work_name=work_name), results) np.save('{save_path}_{work_name}_preds_all.npy'.format(save_path=save_path, work_name=work_name), preds) if work_name == 'test': np.save('{save_path}_{work_name}_labels.npy'.format(save_path=save_path, work_name=work_name), labels) preds_opt = {'known': preds['known'][results['acc']['loc']], 'novel': preds['novel'][results['acc']['loc']]} np.save('{save_path}_{work_name}_preds.npy'.format(save_path=save_path, work_name=work_name), preds_opt) def init_results(num_pts, num_classes=None, hierarchical_measure=False): if num_classes is None: acc_float = np.zeros(num_pts, dtype=float) auc_float = 0. else: acc_float = np.zeros([num_pts, num_classes], dtype=float) auc_float = np.zeros(num_classes, dtype=float) acc = {'known': acc_float, 'novel' : copy.deepcopy(acc_float), 'harmonic': copy.deepcopy(acc_float), 'auc' : auc_float, 'g_bias': copy.deepcopy(auc_float), 'g_known': copy.deepcopy(auc_float), 'g_novel': copy.deepcopy(auc_float), 'g_harmonic': copy.deepcopy(auc_float), } results = {'acc': acc} if hierarchical_measure: results.update({'HP': copy.deepcopy(acc), 'HR': copy.deepcopy(acc), 'HF': copy.deepcopy(acc), 'HE': copy.deepcopy(acc), }) return results def init_counters(num_pts, num_classes, hierarchical_measure=False): # top-down val if isinstance(num_classes, list): counter_data = np.array( [np.zeros(num_ch+1, dtype=int) for num_ch in num_classes]) counter_int = np.array([[np.zeros(num_ch+1, dtype=int) for num_ch in num_classes] for _ in range(num_pts)]) counter_float = np.array([[np.zeros(num_ch+1, dtype=float) for num_ch in num_classes] for _ in range(num_pts)]) # top-down test or flatten else: counter_data = np.zeros(num_classes, dtype=int) counter_int = np.zeros([num_pts, num_classes], dtype=int) counter_float = np.zeros([num_pts, num_classes], dtype=float) counters = {'data': counter_data, 'acc' : counter_int} if hierarchical_measure: counters.update({'HP': counter_float, 'HR': copy.deepcopy(counter_float), 'HF': copy.deepcopy(counter_float), 'HE': copy.deepcopy(counter_int), }) return counters def count_val(p, counters, logits, labels, T, gpu, hierarchical_measure=False): num_leaves = len(T['wnids_leaf']) num_classes = len(T['wnids']) num_supers = num_classes - num_leaves descendants = T['descendants'] ch_slice = T['ch_slice'] relevant = T['relevant'] if hierarchical_measure: HP_mat = T['HP_mat'] HF_mat = T['HF_mat'] dist_mat = T['dist_mat'] for m in range(num_supers): c = m + num_leaves for i in range(ch_slice[m], ch_slice[m+1]): # filter classifiable labels classify_me = torch.zeros(labels.size(), out=torch.ByteTensor()) if gpu: classify_me = classify_me.cuda() for k in descendants[c]: classify_me |= (labels == k) if not classify_me.any(): continue loc = torch.LongTensor(relevant[i]) if gpu: loc = loc.cuda() preds_c = logits[classify_me.nonzero().view(-1)][:, loc].max(dim=1)[1].data.cpu().numpy() acc = (preds_c == 0) # 0-th label is GT if hierarchical_measure: HP = HP_mat[relevant[i][preds_c], relevant[i][0]] HR = HP_mat[relevant[i][0], relevant[i][preds_c]] HF = HF_mat[relevant[i][preds_c], relevant[i][0]] HE = dist_mat[relevant[i][preds_c], relevant[i][0]] if p == 0: counters['data'][c] += preds_c.shape[0] counters['acc'][p,c] += acc.sum() if hierarchical_measure: counters['HP'][p,c] += HP.sum() counters['HR'][p,c] += HR.sum() counters['HF'][p,c] += HF.sum() counters['HE'][p,c] += HE.sum() def count_test(p, counters, preds, labels, T, hierarchical_measure=False): label_hnd = T['label_hnd'] if hierarchical_measure: HP_mat = T['HP_mat'] HF_mat = T['HF_mat'] dist_mat = T['dist_mat'] for l in np.unique(labels.cpu().numpy()): preds_l = preds[(labels == int(l)).cpu().numpy().astype(bool)] acc = np.zeros_like(preds_l, dtype=bool) if hierarchical_measure: HE = MAX_DIST*np.ones_like(preds_l, dtype=int) HP, HR, HF = np.zeros_like(preds_l), np.zeros_like(preds_l), np.zeros_like(preds_l) for c in label_hnd[l]: acc |= (preds_l == c) if hierarchical_measure: HE = np.minimum(HE, dist_mat[preds_l, c]) HP = np.maximum(HP, HP_mat[preds_l, c]) HR = np.maximum(HR, HP_mat[c, preds_l]) HF = np.maximum(HF, HF_mat[preds_l, c]) if p == 0: counters['data'][l] += preds_l.shape[0] counters['acc'][p,l] += acc.sum() if hierarchical_measure: counters['HE'][p,l] += HE.sum() counters['HP'][p,l] += HP.sum() counters['HR'][p,l] += HR.sum() counters['HF'][p,l] += HF.sum() def count_super(p, m, counters, preds, labels, label_to_ch): for l in np.unique(labels): preds_l = preds[labels == l] # in -> known if label_to_ch[l]: acc = np.zeros_like(preds_l, dtype=bool) for c in label_to_ch[l]: if p == 0: counters['data'][m][c] += preds_l.shape[0] acc |= (preds_l == c) acc_sum = acc.sum() for c in label_to_ch[l]: counters['acc'][p,m][c] += acc_sum # out -> novel else: if p == 0: counters['data'][m][-1] += preds_l.shape[0] acc_sum = (preds_l < 0).sum() counters['acc'][p,m][-1] += acc_sum def counters_to_results(counters, results, classes, label_hnd=None): nonempty = (counters['data'] > 0).nonzero()[0] testable = dict() for dtype in ['known', 'novel']: testable[dtype] = classes[dtype][np.isin(classes[dtype], nonempty)] if label_hnd is None: num_data = dict() for dtype in ['known', 'novel']: num_data[dtype] = counters['data'][testable[dtype]] else: label_hnd_inv = dict() for dtype in ['known', 'novel']: for l in testable[dtype]: for c in label_hnd[l]: if label_hnd_inv.get(c) is None: label_hnd_inv[c] = [l] else: label_hnd_inv[c].append(l) for mtype in results: # acc for dtype in ['known', 'novel']: for p in range(results[mtype][dtype].shape[0]): if label_hnd is None: # leaf-wise results[mtype][dtype][p] = (counters[mtype][p][testable[dtype]] / num_data[dtype]).mean() else: # super-wise; del dtype_ = 'super' if dtype == 'novel' else dtype num_classes_testable = 0 for c in classes[dtype_]: if label_hnd_inv.get(c) is not None: num_classes_testable += 1 l = label_hnd_inv[c] results[mtype][dtype][p] += (counters[mtype][p][l] / counters['data'][l]).mean() results[mtype][dtype][p] /= num_classes_testable # harmonic acc for p in range(results[mtype]['harmonic'].shape[0]): results[mtype]['harmonic'][p] = harmonic_mean(results[mtype]['known'][p], results[mtype]['novel'][p]) # AUC y = np.concatenate([[0.], results[mtype]['novel'], [results[mtype]['novel'][-1]]]) x = np.concatenate([[results[mtype]['known'][0]], results[mtype]['known'], [0.]]) results[mtype]['auc'] = -np.trapz(y,x); def counters_to_results_super(counters, results, pts, num_models): num_pts = pts.shape[0] for m in range(num_models): b_known_testable = counters['data'][m] > 0 b_novel_testable = b_known_testable[-1] b_known_testable[-1] = False b_known_testable_any = b_known_testable.any() for p in range(num_pts): if b_known_testable_any: # results['acc']['known'][p,m] = \ # (counters['acc'][p,m][b_known_testable] / counters['data'][m][b_known_testable]).mean() results['acc']['known'][p,m] = counters['acc'][p,m][:-1].sum() / counters['data'][m][:-1].sum() if b_novel_testable: results['acc']['novel'][p,m] = counters['acc'][p,m][-1] / counters['data'][m][-1] if b_known_testable_any and b_novel_testable: results['acc']['harmonic'][p,m] = harmonic_mean(results['acc']['known'][p,m], results['acc']['novel'][p,m]) elif b_known_testable_any: results['acc']['harmonic'][p,m] = results['acc']['known'][p,m] elif b_novel_testable: results['acc']['harmonic'][p,m] = results['acc']['novel'][p,m] # find optimal points i_opt = {'global': 0, 'local': []} results['acc_opt'] = {'global': {'known': [], 'novel': [], 'harmonic': []}, 'local' : {'known': [], 'novel': [], 'harmonic': []}} results['ths_opt'] = {'global': 0., 'local': []} i_opt['global'] = results['acc']['harmonic'].mean(axis=1).argmax(axis=0) i_opt['local'] = results['acc']['harmonic'].argmax(axis=0) for mtype in ['known', 'novel', 'harmonic']: results['acc_opt']['global'][mtype] = results['acc'][mtype][i_opt['global']] results['acc_opt']['local'][mtype] = results['acc'][mtype][i_opt['local'], range(num_models)] results['ths_opt']['global'] = pts[i_opt['global']] results['ths_opt']['local'] = pts[i_opt['local']] def results_to_guarantee(results, pts, kg): # novel bias increasing -> known acc decreasing loc = np.abs(results['acc']['known'] - kg).argmin() closest = results['acc']['known'][loc] if closest < kg: locs = np.array([max(loc-1, 0), loc]) elif closest == kg: locs = np.array([loc, loc]) else: locs = np.array([loc, min(loc+1, results['acc']['known'].shape[0]-1)]) # linear interpolation x = results['acc']['known'][locs] if x[0] == x[1]: for mtype in results: results[mtype]['g_known'] = results[mtype]['known'][loc] results[mtype]['g_novel'] = results[mtype]['novel'][loc] results[mtype]['g_harmonic'] = results[mtype]['harmonic'][loc] results['acc']['g_bias'] = pts[loc] else: for mtype in results: y = results[mtype]['known'][locs] results[mtype]['g_known'] = (kg-x[0])*(y[1]-y[0])/(x[1]-x[0])+y[0] y = results[mtype]['novel'][locs] results[mtype]['g_novel'] = (kg-x[0])*(y[1]-y[0])/(x[1]-x[0])+y[0] y = results[mtype]['harmonic'][locs] results[mtype]['g_harmonic'] = (kg-x[0])*(y[1]-y[0])/(x[1]-x[0])+y[0] y = pts[locs] results['acc']['g_bias'] = (kg-x[0])*(y[1]-y[0])/(x[1]-x[0])+y[0] results['acc']['loc'] = loc def print_results(work_name, results, save_path, hierarchical_measure=False, start_time=time.time()): if hierarchical_measure: mtypes = ['acc', 'HF'] else: mtypes = ['acc'] print(save_path) print('{work_name}; '.format(work_name=work_name), end='') print('{time:8.3f} s'.format(time=time.time()-start_time)) for m, mtype in enumerate(mtypes): print('bias: {res:7.4f}; '.format(res=results['acc']['g_bias']), end='') print('{mtype:4s}'.format(mtype=mtype), end='') print('known: {res:5.2f}; '.format(res=results[mtype]['g_known']*100.), end='') print('novel: {res:5.2f}; '.format(res=results[mtype]['g_novel']*100.), end='') if mtype == 'acc': print('auc : {res:5.2f}; '.format(res=results[mtype]['auc']*100.)) else: print('hmean: {res:5.2f}; '.format(res=results[mtype]['g_harmonic']*100.)) # plot known vs. novel plt.figure(m) plt.plot(results[mtype]['known'], results[mtype]['novel'], 'k.-') if mtype == 'HE': plt.xticks(np.arange(0., 11., 1.)) plt.yticks(np.arange(0., 11., 1.)) plt.axis([0., 10., 0., 10.]) else: plt.xticks(np.arange(0., 1.1, .1)) plt.yticks(np.arange(0., 1.1, .1)) plt.axis([0., 1., 0., 1.]) plt.grid() plt.xlabel('known class accuracy') plt.ylabel('novel class accuracy') plt.title('known: {res:5.2f}; '.format(res=results[mtype]['g_known']*100.) + \ 'novel: {res:5.2f}; '.format(res=results[mtype]['g_novel']*100.) + \ 'hmean: {res:5.2f}; '.format(res=results[mtype]['g_harmonic']*100.) + \ 'auc : {res:5.2f}; '.format(res=results[mtype]['auc']*100.) ) plt.savefig(save_path + '_' + work_name + '_' + mtype + '.png') plt.clf() plt.close() def print_results_super(work_name, results, save_path, start_time=time.time()): print(save_path) print('{work_name}; '.format(work_name=work_name), end='') print('global th: {th:6.4f}; '.format(th=results['ths_opt']['global']), end='') print('{time:8.3f} s'.format(time=time.time()-start_time)) for ran in ['global', 'local']: ran_short = 'glb' if ran == 'global' else 'loc' print('{ran} acc '.format(ran=ran_short), end='') print('known: {known:5.2f}; '.format(known=results['acc_opt'][ran]['known'].mean()*100.), end='') print('novel: {novel:5.2f}; '.format(novel=results['acc_opt'][ran]['novel'].mean()*100.), end='') print('hmean: {harmonic:5.2f}'.format(harmonic=results['acc_opt'][ran]['harmonic'].mean()*100.)) if __name__ == '__main__': opts = argparse.Namespace() opts.gpu = True opts.dataset = 'ImageNet' #'ImageNet' #'AWA2' #'CUB' opts.cnn = 'resnet101' opts.test_relu = False opts.test_data_norm = False opts.workers = 0 opts.batch_size = 5000 if opts.dataset == 'ImageNet' else 0 opts.num_epochs = 50 if opts.dataset == 'ImageNet' else 5000 opts.known_guarantee = 0.5 opts.darts_path = 'train_darts/{dataset}/{cnn}'.format(dataset=opts.dataset, cnn=opts.cnn) opts.save_results = False if opts.dataset == 'AWA2': acc_guarantees = [99] else: acc_guarantees = [90] # acc_guarantees = [80, 85, 90, 95, 99] start_time = time.time() T = np.load('taxonomy/{dataset}/taxonomy.npy'.format(dataset=opts.dataset)).item() utils.update_taxonomy('LOO', T, -1, start_time) identity = Identity() for ag in acc_guarantees: # data loader dtypes = ['val', 'known', 'novel'] opts.ag = ag data_loader = utils.get_feature_loader(dtypes, opts, start_time) save_path = '{darts_path}/{ag:.2f}'.format(darts_path=opts.darts_path, ag=ag/100.) print(save_path) test('val', data_loader, identity, T, opts, save_path, start_time) test('test', data_loader, identity, T, opts, save_path, start_time)