Python torch.utils.data.dataset() Examples
The following are 12
code examples of torch.utils.data.dataset().
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example.
You may also want to check out all available functions/classes of the module
torch.utils.data
, or try the search function
.
.
Example #1
| Source File: data_loaders.py From DAVANet with MIT License | 6 votes |
|
def get_dataset(self, dataset_type, transforms=None):
files = []
# Load data for each sequence
for file in self.files_list:
if dataset_type == DatasetType.TRAIN and file['phase'] == 'Train':
name = file['name']
pair_num = file['pair_num']
samples = file['sample']
files_num_old = len(files)
files.extend(self.get_files_of_taxonomy(name, samples))
print('[INFO] %s Collecting files of Taxonomy [Name = %s, Pair Numbur = %s, Loaded = %r]' % (
dt.now(), name, pair_num, pair_num == (len(files)-files_num_old)))
elif dataset_type == DatasetType.TEST and file['phase'] == 'Test':
name = file['name']
pair_num = file['pair_num']
samples = file['sample']
files_num_old = len(files)
files.extend(self.get_files_of_taxonomy(name, samples))
print('[INFO] %s Collecting files of Taxonomy [Name = %s, Pair Numbur = %s, Loaded = %r]' % (
dt.now(), name, pair_num, pair_num == (len(files)-files_num_old)))
print('[INFO] %s Complete collecting files of the dataset for %s. Total Pair Numbur: %d.\n' % (dt.now(), dataset_type.name, len(files)))
return StereoDeblurDataset(files, transforms)
Example #2
| Source File: main-all.py From PyTorchText with MIT License | 5 votes |
|
def val(model,dataset):
dataset.train(False)
model.eval()
dataloader = data.DataLoader(dataset,
batch_size = opt.batch_size,
shuffle = False,
num_workers = opt.num_workers,
pin_memory = True
)
predict_label_and_marked_label_list=[]
for ii,((title,content),label) in tqdm.tqdm(enumerate(dataloader)):
title,content,label = (Variable(title[0].cuda()),Variable(title[1].cuda())),(Variable(content[0].cuda()),Variable(content[1].cuda())),Variable(label.cuda())
score = model(title,content)
# !TODO: 优化此处代码
# 1. append
# 2. for循环
# 3. topk 代替sort
predict = score.data.topk(5,dim=1)[1].cpu().tolist()
true_target = label.data.float().topk(5,dim=1)#[1].cpu().tolist()#sort(dim=1,descending=True)
true_index=true_target[1][:,:5]
true_label=true_target[0][:,:5]
tmp= []
for jj in range(label.size(0)):
true_index_=true_index[jj]
true_label_=true_label[jj]
true=true_index_[true_label_>0]
tmp.append((predict[jj],true.tolist()))
predict_label_and_marked_label_list.extend(tmp)
del score
dataset.train(True)
model.train()
scores,prec_,recall_,_ss=get_score(predict_label_and_marked_label_list)
return (scores,prec_,recall_,_ss)
Example #3
| Source File: main4model.py From PyTorchText with MIT License | 5 votes |
|
def val(model,dataset):
dataset.train(False)
model.eval()
dataloader = data.DataLoader(dataset,
batch_size = opt.batch_size,
shuffle = False,
num_workers = opt.num_workers,
pin_memory = True
)
predict_label_and_marked_label_list=[]
for ii,((title,content),label) in tqdm.tqdm(enumerate(dataloader)):
title,content,label = (Variable(title[0].cuda()),Variable(title[1].cuda())),(Variable(content[0].cuda()),Variable(content[1].cuda())),Variable(label.cuda())
score = model(title,content)
# !TODO: 优化此处代码
# 1. append
# 2. for循环
# 3. topk 代替sort
predict = score.data.topk(5,dim=1)[1].cpu().tolist()
true_target = label.data.float().topk(5,dim=1)#[1].cpu().tolist()#sort(dim=1,descending=True)
true_index=true_target[1][:,:5]
true_label=true_target[0][:,:5]
tmp= []
for jj in range(label.size(0)):
true_index_=true_index[jj]
true_label_=true_label[jj]
true=true_index_[true_label_>0]
tmp.append((predict[jj],true.tolist()))
predict_label_and_marked_label_list.extend(tmp)
del score
dataset.train(True)
model.train()
scores,prec_,recall_,_ss=get_score(predict_label_and_marked_label_list)
return (scores,prec_,recall_,_ss)
Example #4
| Source File: main-all.back.py From PyTorchText with MIT License | 5 votes |
|
def val(model,dataset):
dataset.train(False)
model.eval()
dataloader = data.DataLoader(dataset,
batch_size = opt.batch_size,
shuffle = False,
num_workers = opt.num_workers,
pin_memory = True
)
predict_label_and_marked_label_list=[]
for ii,((title,content),label) in tqdm.tqdm(enumerate(dataloader)):
title,content,label = (Variable(title[0].cuda()),Variable(title[1].cuda())),(Variable(content[0].cuda()),Variable(content[1].cuda())),Variable(label.cuda())
score = model(title,content)
# !TODO: 优化此处代码
# 1. append
# 2. for循环
# 3. topk 代替sort
predict = score.data.topk(5,dim=1)[1].cpu().tolist()
true_target = label.data.float().topk(5,dim=1)#[1].cpu().tolist()#sort(dim=1,descending=True)
true_index=true_target[1][:,:5]
true_label=true_target[0][:,:5]
tmp= []
for jj in range(label.size(0)):
true_index_=true_index[jj]
true_label_=true_label[jj]
true=true_index_[true_label_>0]
tmp.append((predict[jj],true.tolist()))
predict_label_and_marked_label_list.extend(tmp)
del score
dataset.train(True)
model.train()
scores,prec_,recall_,_ss=get_score(predict_label_and_marked_label_list)
return (scores,prec_,recall_,_ss)
Example #5
| Source File: main_boost.py From PyTorchText with MIT License | 5 votes |
|
def val(model,dataset):
dataset.train(False)
model.eval()
dataloader = data.DataLoader(dataset,
batch_size = opt.batch_size,
shuffle = False,
num_workers = opt.num_workers,
pin_memory = True
)
predict_label_and_marked_label_list=[]
for ii,((title,content),label) in tqdm.tqdm(enumerate(dataloader)):
title,content,label = (Variable(title[0].cuda()),Variable(title[1].cuda())),(Variable(content[0].cuda()),Variable(content[1].cuda())),Variable(label.cuda())
score = model(title,content)
# !TODO: 优化此处代码
# 1. append
# 2. for循环
# 3. topk 代替sort
predict = score.data.topk(5,dim=1)[1].cpu().tolist()
true_target = label.data.float().topk(5,dim=1)#[1].cpu().tolist()#sort(dim=1,descending=True)
true_index=true_target[1][:,:5]
true_label=true_target[0][:,:5]
tmp= []
for jj in range(label.size(0)):
true_index_=true_index[jj]
true_label_=true_label[jj]
true=true_index_[true_label_>0]
tmp.append((predict[jj],true.tolist()))
predict_label_and_marked_label_list.extend(tmp)
del score
dataset.train(True)
model.train()
scores,prec_,recall_,_ss=get_score(predict_label_and_marked_label_list)
return (scores,prec_,recall_,_ss)
Example #6
| Source File: nn_model.py From FATE with Apache License 2.0 | 5 votes |
|
def predict(self, data: data.dataset, **kwargs):
predict_data = DataLoader(data, batch_size=data.batch_size, shuffle=False)
for batch_id, (feature, label) in enumerate(predict_data):
feature = torch.tensor(feature, dtype=torch.float32)
# label = torch.tensor(label, dtype=torch.float32)
y = self._model(feature)
if batch_id == 0:
result = y.detach().numpy()
else:
result = np.vstack((result, y.detach().numpy()))
return result
Example #7
| Source File: data_loaders.py From STFAN with MIT License | 5 votes |
|
def __init__(self):
self.img_blur_path_template = cfg.DIR.IMAGE_BLUR_PATH
self.img_clear_path_template = cfg.DIR.IMAGE_CLEAR_PATH
# Load all files of the dataset
with io.open(cfg.DIR.DATASET_JSON_FILE_PATH, encoding='utf-8') as file:
self.files_list = json.loads(file.read())
Example #8
| Source File: data_loaders.py From DAVANet with MIT License | 5 votes |
|
def __init__(self):
self.img_left_path_template = cfg.DIR.IMAGE_LEFT_PATH
self.img_right_path_template = cfg.DIR.IMAGE_RIGHT_PATH
self.disp_left_path_template = cfg.DIR.DISPARITY_LEFT_PATH
self.disp_right_path_template = cfg.DIR.DISPARITY_RIGHT_PATH
# Load all files of the dataset
with io.open(cfg.DIR.DATASET_JSON_FILE_PATH, encoding='utf-8') as file:
self.files_list = json.loads(file.read())
Example #9
| Source File: data_loaders.py From DAVANet with MIT License | 5 votes |
|
def get_dataset(self, dataset_type, transforms=None):
files = []
# Load data for each category
for file in self.files_list:
if dataset_type == DatasetType.TRAIN and (file['phase'] == 'TRAIN' or file['phase'] == 'TEST'):
categories = file['categories']
phase = file['phase']
classes = file['classes']
names = file['names']
samples = file['sample']
print('[INFO] %s Collecting files of Taxonomy [categories = %s, phase = %s, classes = %s, names = %s]' % (
dt.now(), categories, phase, classes, names))
files.extend(
self.get_files_of_taxonomy(categories, phase, classes, names, samples))
elif dataset_type == DatasetType.TEST and file['phase'] == 'TEST':
categories = file['categories']
phase = file['phase']
classes = file['classes']
names = file['names']
samples = file['sample']
print('[INFO] %s Collecting files of Taxonomy [categories = %s, phase = %s, classes = %s, names = %s]' % (
dt.now(), categories, phase, classes, names))
files.extend(
self.get_files_of_taxonomy(categories, phase, classes, names, samples))
print('[INFO] %s Complete collecting files of the dataset for %s. Total files: %d.' % (dt.now(), dataset_type.name, len(files)))
return FlyingThings3DDataset(files, transforms)
Example #10
| Source File: data_loaders.py From DAVANet with MIT License | 5 votes |
|
def __init__(self):
self.img_left_blur_path_template = cfg.DIR.IMAGE_LEFT_BLUR_PATH
self.img_left_clear_path_template = cfg.DIR.IMAGE_LEFT_CLEAR_PATH
self.img_right_blur_path_template = cfg.DIR.IMAGE_RIGHT_BLUR_PATH
self.img_right_clear_path_template = cfg.DIR.IMAGE_RIGHT_CLEAR_PATH
self.disp_left_path_template = cfg.DIR.DISPARITY_LEFT_PATH
self.disp_right_path_template = cfg.DIR.DISPARITY_RIGHT_PATH
# Load all files of the dataset
with io.open(cfg.DIR.DATASET_JSON_FILE_PATH, encoding='utf-8') as file:
self.files_list = json.loads(file.read())
Example #11
| Source File: main.py From PyTorchText with MIT License | 4 votes |
|
def val(model,dataset):
'''
计算模型在验证集上的分数
'''
dataset.train(False)
model.eval()
dataloader = data.DataLoader(dataset,
batch_size = opt.batch_size,
shuffle = False,
num_workers = opt.num_workers,
pin_memory = True
)
predict_label_and_marked_label_list=[]
for ii,((title,content),label) in tqdm.tqdm(enumerate(dataloader)):
title,content,label = Variable(title.cuda(),volatile=True),\
Variable(content.cuda(),volatile=True),\
Variable(label.cuda(),volatile=True)
score = model(title,content)
# !TODO: 优化此处代码
# 1. append
# 2. for循环
# 3. topk 代替sort
predict = score.data.topk(5,dim=1)[1].cpu().tolist()
true_target = label.data.float().topk(5,dim=1)
true_index=true_target[1][:,:5]
true_label=true_target[0][:,:5]
tmp= []
for jj in range(label.size(0)):
true_index_=true_index[jj]
true_label_=true_label[jj]
true=true_index_[true_label_>0]
tmp.append((predict[jj],true.tolist()))
predict_label_and_marked_label_list.extend(tmp)
del score
dataset.train(True)
model.train()
scores,prec_,recall_,_ss=get_score(predict_label_and_marked_label_list)
return (scores,prec_,recall_,_ss)
Example #12
| Source File: data_loaders.py From STFAN with MIT License | 4 votes |
|
def get_dataset(self, dataset_type, transforms=None):
sequences = []
# Load data for each sequence
for file in self.files_list:
if dataset_type == DatasetType.TRAIN and file['phase'] == 'train':
name = file['name']
phase = file['phase']
samples = file['sample']
sam_len = len(samples)
seq_len = cfg.DATA.SEQ_LENGTH
seq_num = int(sam_len/seq_len)
for n in range(seq_num):
sequence = self.get_files_of_taxonomy(phase, name, samples[seq_len*n: seq_len*(n+1)])
sequences.extend(sequence)
if not seq_len%seq_len == 0:
sequence = self.get_files_of_taxonomy(phase, name, samples[-seq_len:])
sequences.extend(sequence)
seq_num += 1
print('[INFO] %s Collecting files of Taxonomy [Name = %s]' % (dt.now(), name + ': ' + str(seq_num)))
elif dataset_type == DatasetType.TEST and file['phase'] == 'test':
name = file['name']
phase = file['phase']
samples = file['sample']
sam_len = len(samples)
seq_len = cfg.DATA.SEQ_LENGTH
seq_num = int(sam_len / seq_len)
for n in range(seq_num):
sequence = self.get_files_of_taxonomy(phase, name, samples[seq_len*n: seq_len*(n+1)])
sequences.extend(sequence)
if not seq_len % seq_len == 0:
sequence = self.get_files_of_taxonomy(phase, name, samples[-seq_len:])
sequences.extend(sequence)
seq_num += 1
print('[INFO] %s Collecting files of Taxonomy [Name = %s]' % (dt.now(), name + ': ' + str(seq_num)))
print('[INFO] %s Complete collecting files of the dataset for %s. Seq Number: %d.\n' % (dt.now(), dataset_type.name, len(sequences)))
return VideoDeblurDataset(sequences, transforms)
