Python numpy.load() Examples

def get_named_set(lang_codes, feature_set):
    if feature_set == 'id':
        return get_id_set(lang_codes)
    
    if feature_set not in FEATURE_SETS:
        print("ERROR: Invalid feature set " + feature_set, file=sys.stderr)
        sys.exit()
        
    filename, source, prefix = FEATURE_SETS[feature_set]
    feature_database = np.load(filename)
    lang_codes = [ get_language_code(l, feature_database) for l in lang_codes ]
    lang_indices = [ get_language_index(l, feature_database) for l in lang_codes ]
    feature_names = get_feature_names(prefix, feature_database)
    feature_indices = [ get_feature_index(f, feature_database) for f in feature_names ]
    source_index = get_source_index(source, feature_database)
    feature_values = feature_database["data"][lang_indices,:,:][:,feature_indices,:][:,:,source_index]
    feature_values = feature_values.squeeze(axis=2)
    return feature_names, feature_values 

def gen_pruned_features(name):
    print name
    feature_dir = 'data/feature_' + args.domain + \
        '_' + str(args.n_boxes) + 'boxes/' + name + '/'
    n_clips = len(glob.glob(feature_dir + BOX_FEATURE + '*.npy'))
    for clip in xrange(1, n_clips+1):
        pruned_boxes = np.load(feature_dir + BOX_FEATURE + '{:04d}.npy'.format(clip)) # (50, args.n_boxes, 4)
        roisavg = np.load(feature_dir + 'roisavg{:04d}.npy'.format(clip)) # (50, args.n_boxes, 512)

        pruned_roisavg = np.zeros((50, args.n_boxes, 512))
        for frame in xrange(50):
            for box_id in xrange(args.n_boxes):
                if not np.array_equal(pruned_boxes[frame][box_id], np.zeros((4))):
                    pruned_roisavg[frame][box_id] = roisavg[frame][box_id]

        np.save('{}pruned_roisavg{:04d}'.format(feature_dir, clip), pruned_roisavg) 

def __init__(self):
        if not self.code_table:
            with open(CATEGORY_CODES) as codes:
                self.code_table = {int(k): v for k, v in json.loads(codes.read()).items()}

        caffe_models = os.path.expanduser(CAFFE_MODELS)
        model = 'squeezenet', 'init_net.pb', 'predict_net.pb', 'ilsvrc_2012_mean.npy', 227
        self.model = model

        mean_file = os.path.join(caffe_models, model[0], model[3])
        if not os.path.exists(mean_file):
            self.mean = 128
        else:
            mean = np.load(mean_file).mean(1).mean(1)
            self.mean = mean[:, np.newaxis, np.newaxis]

        init_net = os.path.join(caffe_models, model[0], model[1])
        predict_net = os.path.join(caffe_models, model[0], model[2])

        with open(init_net) as f:
            self.init_net = f.read()
        with open(predict_net) as f:
            self.predict_net = f.read() 

def test_xyz2lab(self):
        assert_array_almost_equal(xyz2lab(self.xyz_array),
                                  self.lab_array, decimal=3)

        # Test the conversion with the rest of the illuminants.
        for I in ["d50", "d55", "d65", "d75"]:
            for obs in ["2", "10"]:
                fname = "lab_array_{0}_{1}.npy".format(I, obs)
                lab_array_I_obs = np.load(
                    os.path.join(os.path.dirname(__file__), 'data', fname))
                assert_array_almost_equal(lab_array_I_obs,
                                          xyz2lab(self.xyz_array, I, obs),
                                          decimal=2)
        for I in ["a", "e"]:
            fname = "lab_array_{0}_2.npy".format(I)
            lab_array_I_obs = np.load(
                os.path.join(os.path.dirname(__file__), 'data', fname))
            assert_array_almost_equal(lab_array_I_obs,
                                      xyz2lab(self.xyz_array, I, "2"),
                                      decimal=2) 

def test_xyz2luv(self):
        assert_array_almost_equal(xyz2luv(self.xyz_array),
                                  self.luv_array, decimal=3)

        # Test the conversion with the rest of the illuminants.
        for I in ["d50", "d55", "d65", "d75"]:
            for obs in ["2", "10"]:
                fname = "luv_array_{0}_{1}.npy".format(I, obs)
                luv_array_I_obs = np.load(
                    os.path.join(os.path.dirname(__file__), 'data', fname))
                assert_array_almost_equal(luv_array_I_obs,
                                          xyz2luv(self.xyz_array, I, obs),
                                          decimal=2)
        for I in ["a", "e"]:
            fname = "luv_array_{0}_2.npy".format(I)
            luv_array_I_obs = np.load(
                os.path.join(os.path.dirname(__file__), 'data', fname))
            assert_array_almost_equal(luv_array_I_obs,
                                      xyz2luv(self.xyz_array, I, "2"),
                                      decimal=2) 

def test_luv2xyz(self):
        assert_array_almost_equal(luv2xyz(self.luv_array),
                                  self.xyz_array, decimal=3)

        # Test the conversion with the rest of the illuminants.
        for I in ["d50", "d55", "d65", "d75"]:
            for obs in ["2", "10"]:
                fname = "luv_array_{0}_{1}.npy".format(I, obs)
                luv_array_I_obs = np.load(
                    os.path.join(os.path.dirname(__file__), 'data', fname))
                assert_array_almost_equal(luv2xyz(luv_array_I_obs, I, obs),
                                          self.xyz_array, decimal=3)
        for I in ["a", "e"]:
            fname = "luv_array_{0}_2.npy".format(I, obs)
            luv_array_I_obs = np.load(
                os.path.join(os.path.dirname(__file__), 'data', fname))
            assert_array_almost_equal(luv2xyz(luv_array_I_obs, I, "2"),
                                      self.xyz_array, decimal=3) 

def dataset_from_file(filename):
    """Load a dataset from file.

    Args:
        filename (string): the name of the file from which extract the dataset

    Returns:
        tuple: the dataset (np.ndarray) and the ngrams (list of strings)
    """
    loader = np.load(filename)
    num_entries = loader['num_entries'][0]
    sp_dataset = sparse.csr_matrix((loader['data'], loader['indices'], loader['indptr']),
                         shape = loader['shape'])
    dataset = sp_dataset.toarray()
    samp_entries, num_features = dataset.shape
    return dataset.reshape(int(samp_entries / num_entries), num_entries, num_features), loader['ngrams'] 

def _load_accumulators(self, main_loop):
        """Nasty method, use carefully"""
        for cg_name, model in main_loop.models.iteritems():
            source = numpy.load(self.path_to_accumulators.format(cg_name))
            accums_dict = {name.replace("-", "/"): value
                           for name, value in source.items()}
            source.close()
            algo = main_loop.algorithm.algorithms[cg_name]
            model_params = model.get_params()
            steps = algo.steps.items()

            for pidx in xrange(len(steps)):
                # Get parameter name and its accumulators
                p = steps[pidx][0]
                name = [k for k, v in model_params.iteritems() if v == p][0]
                accums = accums_dict[name]

                # This is num_accums_per_param
                col = len(accums)
                for aidx in xrange(col):
                    algo.step_rule_updates[pidx*col+aidx][0].set_value(
                        accums[aidx]) 

def _load_accumulators(self, main_loop):
        """Load accumulators with some checks."""
        for cg_name, model in main_loop.models.iteritems():

            # Load accumulators
            accum_filename = self.path_to_accumulators.format(cg_name)
            if not os.path.isfile(accum_filename):
                logger.error(" Accumulators file does not exist [{}]"
                             .format(accum_filename))
                continue

            source = numpy.load(accum_filename)
            accums_to_load = {k: v for k, v in source.items()}
            source.close()

            algo = main_loop.algorithm.algorithms[cg_name]
            accums = algo.step_rule_updates

            # Set accumulators
            for acc in accums:
                try:
                    acc.set_value(accums_to_load[acc.name])
                except:
                    logger.error(" Could not load {}".format(acc.name)) 

def load_params(self, saveto):
        try:
            logger.info(" ...loading model parameters")
            params_all = numpy.load(saveto)
            params_this = self.get_params()
            missing = set(params_this) - set(params_all)
            for pname in params_this.keys():
                if pname in params_all:
                    val = params_all[pname]
                    self._set_param_value(params_this[pname], val, pname)
                elif self.num_decs > 1 and self.decoder.share_att and \
                        pname in self.decoder.shared_params_map:
                    val = params_all[self.decoder.shared_params_map[pname]]
                    self._set_param_value(params_this[pname], val, pname)
                else:
                    logger.warning(
                        " Parameter does not exist: {}".format(pname))

            logger.info(
                " Number of params loaded: {}"
                .format(len(params_this) - len(missing)))
        except Exception as e:
            logger.error(" Error {0}".format(str(e))) 

def load_data():
    """Draw the Mott lobes."""

    res = np.load(r'data_%d.npy' % GRID_SIZE)
    x = res[:, 0]
    y = res[:, 1]
    z = []
    for i, entry in enumerate(res):
        z.append(kinetic_energy(entry[2:], -1.))
    plt.pcolor(
        np.reshape(x, (GRID_SIZE, GRID_SIZE)),
        np.reshape(y, (GRID_SIZE, GRID_SIZE)),
        np.reshape(z, (GRID_SIZE, GRID_SIZE))
    )
    plt.xlabel('$dt/U$')
    plt.ylabel('$\mu/U$')
    plt.show() 

def __init__(
            self,
            name,
            grid_size,
            x_label,
            y_label,
            x_lim,
            y_lim,
            file_name,
            order_parameter
    ):
        self.name = name
        self.grid_size = grid_size
        self.x_lim = x_lim
        self.y_lim = y_lim
        self.x_label = x_label
        self.y_label = y_label
        self.file_name = file_name
        self.order_parameter = order_parameter
        # load data file
        data = np.load(file_name)
        shape = (grid_size, grid_size)
        # for background plotting
        xx = np.reshape(data[:, 0], shape)
        yy = np.reshape(data[:, 1], shape)
        # color plot
        zz = [
            order_parameter(entry)
            for entry in data[:, 2:]
        ]
        zz = np.reshape(zz, shape)
        self.plot_data = xx, yy, zz
        # interpolation
        # rescale parameter space to (0,1)
        data[:, 0] -= np.min(data[:, 0])
        data[:, 0] /= np.max(data[:, 0])
        data[:, 1] -= np.min(data[:, 1])
        data[:, 1] /= np.max(data[:, 1])
        self._interp = LinearNDInterpolator(data[:, 0:2], data[:, 2:], fill_value=np.nan) 

def _get_batch_normalization_weights(self,layer_name):
        beta = '%s/batch_normalization/beta:0'%(layer_name)
        gamma = '%s/batch_normalization/gamma:0'%(layer_name)
        mean = '%s/batch_normalization/moving_mean:0'%(layer_name)
        variance = '%s/batch_normalization/moving_variance:0'%(layer_name)
        if self.weights is None or beta not in self.weights:
            print('{:>23} {:>23}'.format(beta, 'using default initializer'))
            return None, None, None, None
        else:
            betax = self.weights[beta]
            gammax = self.weights[gamma]
            meanx = self.weights[mean]
            variancex = self.weights[variance]

            self.loaded_weights[beta]=1
            self.loaded_weights[gamma]=1
            self.loaded_weights[mean]=1
            self.loaded_weights[variance]=1
            #print('{:>23} {:>23}'.format(beta, 'load from %s'%self.flags.load_path))
            return betax,gammax,meanx,variancex 

def post_sub_one(inx):
    w,h = 1918,1280
    path,out,threshold = inx
    data = np.load(path).item()
    imgs,pred = data['name'], data['pred']
    #print(pred.shape)
    fo = open(out,'w')
    #masks = pred>threshold
    for name,mask in zip(imgs,np.squeeze(pred)):
        mask = imresize(mask,[h,w])
        mask = mask>threshold
        code = rle_encode(mask)
        code = [str(i) for i in code]
        code = " ".join(code)
        fo.write("%s,%s\n"%(name,code))
    fo.close()
    return 0 

def show_one_img_mask(data):
    w,h = 1918,1280
    a = randint(0,31)
    path = "../input/test"
    data = np.load(data).item()
    name,masks = data['name'][a],data['pred']
    img = Image.open("%s/%s"%(path,name))
    #img.show()
    plt.imshow(img)
    plt.show()
    mask = np.squeeze(masks[a])
    mask = imresize(mask,[h,w]).astype(np.float32)
    print(mask.shape,mask[0])
    img = Image.fromarray(mask*256)#.resize([w,h])
    plt.imshow(img)
    plt.show() 

def split(flags):
    if os.path.exists(flags.split_path):
        return np.load(flags.split_path).item()
    folds = flags.folds
    path = flags.input_path
    random.seed(6)
    img_list = ["%s/%s"%(path,img) for img in os.listdir(path)]
    random.shuffle(img_list)
    dic = {}
    n = len(img_list)
    num = (n+folds-1)//folds
    for i in range(folds):
        s,e = i*num,min(i*num+num,n)
        dic[i] = img_list[s:e]
    np.save(flags.split_path,dic)
    return dic 

def make_benchmark_figure():

    fig = plt.figure(figsize=(6,6))
    ax = fig.add_subplot(1, 1, 1, xscale='linear', yscale='log')


    d1 = np.load('./data/random_data_benchmark.npy')
    d2 = np.load('./data/real_data_benchmark.npy')
    d3 = np.load('./data/real_data_orange3_benchmark.npy')

    ax.scatter(d1[:24, 0], d1[:24, 2], c='r', edgecolor='none', label='Random Data (Polo)')
    ax.scatter(d2[:24, 0], d2[:24, 2], c='green', edgecolor='none', label='Gene expression data (Polo)')
    ax.scatter(d3[:24, 0], d3[:24, 2], c='blue', edgecolor='none', label='Gene expression data (Orange3)')

    ax.legend(loc=2)
    ax.grid('on')
    ax.set_xlabel('log2(Number of leaves)')
    ax.set_ylabel('Run time, seconds')
    fig.tight_layout()
    fig.savefig('data/bench.png', dpi=75) 

def read_data():
    with open(PICKLE_FILENAME, 'rb') as f:
        save = pickle.load(f)
        train_dataset = save['train_dataset']
        train_labels = save['train_labels']
        valid_dataset = save['valid_dataset']
        valid_labels = save['valid_labels']
        test_dataset = save['test_dataset']
        test_labels = save['test_labels']
        del save
        print('Training set', train_dataset.shape, train_labels.shape)
        print('Valid set', valid_dataset.shape, valid_labels.shape)
        print('Test set', test_dataset.shape, test_labels.shape)

    return [train_dataset, valid_dataset,
            test_dataset], [train_labels, valid_labels, test_labels] 

def lrelu(x, leak=0.2, name="lrelu"):
    """Leaky rectifier.
    """
    with tf.variable_scope(name):
        f1 = 0.5 * (1 + leak)
        f2 = 0.5 * (1 - leak)
        return f1 * x + f2 * abs(x)


# load CIFAR-10
# trainx, trainy = cifar10_data.load(args.data_dir, subset='train')
# trainx = trainx.transpose(0, 2, 3, 1)

# trainx_unl = trainx.copy()
# trainx_unl2 = trainx.copy()

# testx, testy = cifar10_data.load(args.data_dir, subset='test')
# testx = testx.transpose(0, 2, 3, 1)

# nr_batches_train = int(trainx.shape[0]/args.batch_size)
# nr_batches_test = int(testx.shape[0]/args.batch_size)


# load MNIST data 

def open(filename, frame='unspecified'):
        """Create a Point from data saved in a file.

        Parameters
        ----------
        filename : :obj:`str`
            The file to load data from.

        frame : :obj:`str`
            The frame to apply to the created point.

        Returns
        -------
        :obj:`Point`
            A point created from the data in the file.
        """
        data = BagOfPoints.load_data(filename)
        return Point(data, frame) 

def open(filename, frame='unspecified'):
        """Create a Direction from data saved in a file.

        Parameters
        ----------
        filename : :obj:`str`
            The file to load data from.

        frame : :obj:`str`
            The frame to apply to the created Direction.

        Returns
        -------
        :obj:`Direction`
            A Direction created from the data in the file.
        """
        data = BagOfPoints.load_data(filename)
        return Direction(data, frame) 

def open(filename, frame='unspecified'):
        """Create a PointCloud from data saved in a file.

        Parameters
        ----------
        filename : :obj:`str`
            The file to load data from.

        frame : :obj:`str`
            The frame to apply to the created PointCloud.

        Returns
        -------
        :obj:`PointCloud`
            A PointCloud created from the data in the file.
        """
        data = BagOfPoints.load_data(filename)
        return PointCloud(data, frame) 

def open(filename, frame='unspecified'):
        """Create a NormalCloud from data saved in a file.

        Parameters
        ----------
        filename : :obj:`str`
            The file to load data from.

        frame : :obj:`str`
            The frame to apply to the created NormalCloud.

        Returns
        -------
        :obj:`NormalCloud`
            A NormalCloud created from the data in the file.
        """
        data = BagOfPoints.load_data(filename)
        return NormalCloud(data, frame) 

def open(filename, frame='unspecified'):
        """Create a RgbCloud from data saved in a file.

        Parameters
        ----------
        filename : :obj:`str`
            The file to load data from.

        frame : :obj:`str`
            The frame to apply to the created RgbCloud.

        Returns
        -------
        :obj:`RgbCloud`
            A RgdCloud created from the data in the file.
        """
        data = BagOfPoints.load_data(filename)
        return RgbCloud(data, frame) 

def __init__(self,
                 audio_file: Path,
                 id: Optional[str] = None,
                 sample_rate_to_convert_to: int = 16000,
                 label: Optional[str] = "nolabel",
                 fourier_window_length: int = 512,
                 hop_length: int = 128,
                 mel_frequency_count: int = 128,
                 label_with_tags: str = None,
                 positional_label: Optional[PositionalLabel] = None):
        # The default values for hop_length and fourier_window_length are powers of 2 near the values specified in the wave2letter paper.

        if id is None:
            id = name_without_extension(audio_file)

        self.audio_file = audio_file

        super().__init__(
            id=id, get_raw_audio=lambda: librosa.load(str(self.audio_file), sr=self.sample_rate)[0],
            label=label, sample_rate=sample_rate_to_convert_to,
            fourier_window_length=fourier_window_length, hop_length=hop_length, mel_frequency_count=mel_frequency_count,
            label_with_tags=label_with_tags, positional_label=positional_label) 

def load_word2vec_matrix(vec_file, word_index, config):
    if os.path.isfile(DirConfig.W2V_CACHE):
        print('---- Load word vectors from cache.')
        embedding_matrix = np.load(open(DirConfig.W2V_CACHE, 'rb'))
        return embedding_matrix

    print('---- loading word2vec ...')
    word2vec = KeyedVectors.load_word2vec_format(
        vec_file, binary=True)
    print('Found %s word vectors of word2vec' % len(word2vec.vocab))

    nb_words = min(config.MAX_NB_WORDS, len(word_index)) + 1
    embedding_matrix = np.zeros((nb_words, config.WORD_EMBEDDING_DIM))
    for word, i in word_index.items():
        if word in word2vec.vocab:
            embedding_matrix[i] = word2vec.word_vec(word)
    print('Null word embeddings: %d' % \
          np.sum(np.sum(embedding_matrix, axis=1) == 0))

    # check the words which not in embedding vectors
    not_found_words = []
    for word, i in word_index.items():
        if word not in word2vec.vocab:
            not_found_words.append(word)

    np.save(open(DirConfig.W2V_CACHE, 'wb'), embedding_matrix)
    return embedding_matrix 

def get_sample_item_file(wav_file_names_sample, item_file, output):
    """
    From a sampled dataset, get an item file for running an ABX task
    Parameters
    ----------
    item file : text file containing at least as columns : #filename, onset, offset, 
    #phoneme and context and side information such as image ID
    item_file : string,
         path to the item file of the whole dataset
    output: string, 
        path where the sample item file will be stored
    """
    wav_names=[]
    temp=np.load(wav_file_names_sample)
    for s in temp:
        wav_names.append(s.split(".")[0])
    
    df=pd.read_csv(item_file, sep="\t", index_col="#filename")
    df_sample=df.loc[wav_names]
    
    df_sample.to_csv(output, sep="\t", header=True, index=False)
    
    return(df_sample) 

def __init__(self, batchsize=64, max_length=15, mode='train'):
        self.batchsize = batchsize
        self.d_vocabulary = None
        self.batch_index = None
        self.batch_len = None
        self.rev_adict = None
        self.max_length = max_length
        self.mode = mode
        self.qdic, self.adic = VQADataProvider.load_data(mode)

        with open('./result/vdict.json','r') as f:
            self.vdict = json.load(f)
        with open('./result/adict.json','r') as f:
            self.adict = json.load(f)

        self.n_ans_vocabulary = len(self.adict)
        self.nlp = spacy.load('en', vectors='en_glove_cc_300_1m_vectors')
        self.glove_dict = {} # word -> glove vector 

def load_vqa_json(data_split):
        """
        Parses the question and answer json files for the given data split. 
        Returns the question dictionary and the answer dictionary.
        """
        qdic, adic = {}, {}

        with open(config.DATA_PATHS[data_split]['ques_file'], 'r') as f:
            qdata = json.load(f)['questions']
            for q in qdata:
                qdic[data_split + QID_KEY_SEPARATOR + str(q['question_id'])] = \
                    {'qstr': q['question'], 'iid': q['image_id']}

        if 'test' not in data_split:
            with open(config.DATA_PATHS[data_split]['ans_file'], 'r') as f:
                adata = json.load(f)['annotations']
                for a in adata:
                    adic[data_split + QID_KEY_SEPARATOR + str(a['question_id'])] = \
                        a['answers']

        print 'parsed', len(qdic), 'questions for', data_split
        return qdic, adic 

def load_genome_json():
        """
        Parses the genome json file. Returns the question dictionary and the
        answer dictionary.
        """
        qdic, adic = {}, {}

        with open(config.DATA_PATHS['genome']['genome_file'], 'r') as f:
            qdata = json.load(f)
            for q in qdata:
                key = 'genome' + QID_KEY_SEPARATOR + str(q['id'])
                qdic[key] = {'qstr': q['question'], 'iid': q['image']}
                adic[key] = [{'answer': q['answer']}]

        print 'parsed', len(qdic), 'questions for genome'
        return qdic, adic 

def __init__(self, batchsize=64, max_length=15, mode='train'):
        self.batchsize = batchsize
        self.d_vocabulary = None
        self.batch_index = None
        self.batch_len = None
        self.rev_adict = None
        self.max_length = max_length
        self.mode = mode
        self.qdic, self.adic = VQADataProvider.load_data(mode)

        with open('./result/vdict.json','r') as f:
            self.vdict = json.load(f)
        with open('./result/adict.json','r') as f:
            self.adict = json.load(f)

        self.n_ans_vocabulary = len(self.adict) 

def load_genome_json():
        """
        Parses the genome json file. Returns the question dictionary and the
        answer dictionary.
        """
        qdic, adic = {}, {}

        with open(config.DATA_PATHS['genome']['genome_file'], 'r') as f:
            qdata = json.load(f)
            for q in qdata:
                key = 'genome' + QID_KEY_SEPARATOR + str(q['id'])
                qdic[key] = {'qstr': q['question'], 'iid': q['image']}
                adic[key] = [{'answer': q['answer']}]

        print 'parsed', len(qdic), 'questions for genome'
        return qdic, adic 

def __init__(self, batchsize=64, max_length=15, mode='train'):
        self.batchsize = batchsize
        self.d_vocabulary = None
        self.batch_index = None
        self.batch_len = None
        self.rev_adict = None
        self.max_length = max_length
        self.mode = mode
        self.qdic, self.adic = VQADataProvider.load_data(mode)

        with open('./result/vdict.json','r') as f:
            self.vdict = json.load(f)
        with open('./result/adict.json','r') as f:
            self.adict = json.load(f)

        self.n_ans_vocabulary = len(self.adict)
        self.nlp = spacy.load('en', vectors='en_glove_cc_300_1m_vectors')
        self.glove_dict = {} # word -> glove vector 

def load_vqa_json(data_split):
        """
        Parses the question and answer json files for the given data split. 
        Returns the question dictionary and the answer dictionary.
        """
        qdic, adic = {}, {}

        with open(config.DATA_PATHS[data_split]['ques_file'], 'r') as f:
            qdata = json.load(f)['questions']
            for q in qdata:
                qdic[data_split + QID_KEY_SEPARATOR + str(q['question_id'])] = \
                    {'qstr': q['question'], 'iid': q['image_id']}

        if 'test' not in data_split:
            with open(config.DATA_PATHS[data_split]['ans_file'], 'r') as f:
                adata = json.load(f)['annotations']
                for a in adata:
                    adic[data_split + QID_KEY_SEPARATOR + str(a['question_id'])] = \
                        a['answers']

        print 'parsed', len(qdic), 'questions for', data_split
        return qdic, adic 

def load_genome_json():
        """
        Parses the genome json file. Returns the question dictionary and the
        answer dictionary.
        """
        qdic, adic = {}, {}

        with open(config.DATA_PATHS['genome']['genome_file'], 'r') as f:
            qdata = json.load(f)
            for q in qdata:
                key = 'genome' + QID_KEY_SEPARATOR + str(q['id'])
                qdic[key] = {'qstr': q['question'], 'iid': q['image']}
                adic[key] = [{'answer': q['answer']}]

        print 'parsed', len(qdic), 'questions for genome'
        return qdic, adic 

def test_individual_stability_matrix():
    """
    Tests individual_stability_matrix method on three gaussian blobs.
    """
    import utils
    import numpy as np
    import scipy as sp
    desired = np.load(home + '/git_repo/PyBASC/tests/ism_test.npy')
    blobs = generate_blobs()
    ism = utils.individual_stability_matrix(blobs, 20, 3)
    #how to use test here?
#    np.corrcoef(ism.flatten(),desired.flatten())
#    np.testing.assert_equal(ism,desired)
#    
#    corr=np.array(sp.spatial.distance.cdist(ism, desired, metric = 'correlation'))
#    
    assert False 

def test_ndarray_to_vol():
    import basc
    import nibabel as nb
    
    subject_file = home + '/git_repo/PyBASC/sample_data/sub1/Func_Quarter_Res.nii.gz'
    subject_file = home + '/git_repo/PyBASC/sample_data/test.nii.gz'
    data = nb.load(subject_file).get_data().astype('float32')
    roi_mask_file= home + '/git_repo/PyBASC/masks/LC_Quarter_Res.nii.gz'
    print( 'Data Loaded')

    
    roi_mask_file_nb = nb.load(roi_mask_file)

    roi_mask_nparray = nb.load(roi_mask_file).get_data().astype('float32').astype('bool')
    roi1data = data[roi_mask_nparray]
    
    data_array=roi1data
    sample_file=subject_file
    filename=home + '/git_repo/PyBASC/sample_data/ndarray_to_vol_test.nii.gz'
    
    basc.ndarray_to_vol(data_array, roi_mask_file, roi_mask_file, filename) 

def get_dataset(dataset_path='Data/Train_Data'):
    # Getting all data from data path:
    try:
        X = np.load('Data/npy_train_data/X.npy')
        Y = np.load('Data/npy_train_data/Y.npy')
    except:
        inputs_path = dataset_path+'/input'
        images = listdir(inputs_path) # Geting images
        X = []
        Y = []
        for img in images:
            img_path = inputs_path+'/'+img

            x_img = get_img(img_path).astype('float32').reshape(64, 64, 3)
            x_img /= 255.

            y_img = get_img(img_path.replace('input/', 'mask/mask_')).astype('float32').reshape(64, 64, 1)
            y_img /= 255.

            X.append(x_img)
            Y.append(y_img)
        X = np.array(X)
        Y = np.array(Y)
        # Create dateset:
        if not os.path.exists('Data/npy_train_data/'):
            os.makedirs('Data/npy_train_data/')
        np.save('Data/npy_train_data/X.npy', X)
        np.save('Data/npy_train_data/Y.npy', Y)
    X, X_test, Y, Y_test = train_test_split(X, Y, test_size=0.1, random_state=42)
    return X, X_test, Y, Y_test 

def __init__(self,
                 saved_model=None,
                 train_folder=None,
                 feature=_feature.__func__):
        """
        :param saved_model: optional saved train set and labels as .npz
        :param train_folder: optional custom train data to process
        :param feature: feature function - compatible with saved_model
        """
        self.feature = feature
        if train_folder is not None:
            self.train_set, self.train_labels, self.model = \
                self.create_model(train_folder)
        else:
            if cv2.__version__[0] == '2':
                self.model = cv2.KNearest()
            else:
                self.model = cv2.ml.KNearest_create()
            if saved_model is None:
                saved_model = TRAIN_DATA+'raw_pixel_data.npz'
            with np.load(saved_model) as data:
                self.train_set = data['train_set']
                self.train_labels = data['train_labels']
                if cv2.__version__[0] == '2':
                    self.model.train(self.train_set, self.train_labels)
                else:
                    self.model.train(self.train_set, cv2.ml.ROW_SAMPLE,
                                     self.train_labels) 

def load(self, model_filename):
        self.__model = load_model("%s.model" % model_filename)
        self.__chars = np.load("%s.cvocab.npy" % model_filename).tolist()
        self.__trigrams = np.load("%s.tvocab.npy" % model_filename).tolist()
        self.__classes = np.load("%s.classes.npy" % model_filename).tolist()

        self.__char_indexes = dict((c, i) for i, c in enumerate(self.__chars))
        self.__indexes_char = dict((i, c) for i, c in enumerate(self.__chars))

        self.__trigrams_indexes = dict((t, i) for i, t in enumerate(self.__trigrams))
        self.__indices_trigrams = dict((i, t) for i, t in enumerate(self.__trigrams))

        self.__classes_indexes = dict((c, i) for i, c in enumerate(self.__classes))
        self.__indexes_classes = dict((i, c) for i, c in enumerate(self.__classes)) 

def get_id_set(lang_codes):
    feature_database = np.load("family_features.npz")
    lang_codes = [ get_language_code(l, feature_database) for l in lang_codes ]
    all_languages = list(feature_database["langs"])
    feature_names = [ "ID_" + l.upper() for l in all_languages ]
    values = np.zeros((len(lang_codes), len(feature_names)))
    for i, lang_code in enumerate(lang_codes):
        feature_index = get_language_index(lang_code, feature_database)
        values[i, feature_index] = 1.0
    return feature_names, values 

def unpickle(file):
    import pickle
    fo = open(file, 'rb')
    dict = pickle.load(fo, encoding='latin1')
    fo.close()
    return dict 

def load_pkl(path):
  with open(path) as f:
    obj = cPickle.load(f)
    print(" [*] load %s" % path)
    return obj 

def load_npy(path):
  obj = np.load(path)
  print(" [*] load %s" % path)
  return obj 

def load(self, local_dir_=None):
        '''
        load dataset from local disk

        Args:
            local_dir_: string or None
                if None, will use default Dataset.DEFAULT_DIR
        ''' 

def load(self, local_dir_=None):
        if local_dir_ is None:
            local_dir = self.DEFAULT_DIR
        else:
            local_dir = Path(local_dir_)
        data_di = np.load(str(local_dir/'cifar10.npz'))
        self.datum[:] = data_di['images']
        self.labels[:] = data_di['labels'] 

def install(
        self, local_dst_dir_=None, local_src_dir_=None, clean_install_=False):
        '''
        Install the dataset into directly usable format,
        requires downloading for public dataset.

        Args:
            local_dst_dir_: string or None
                where to install the dataset, None -> "%(default_dir)s"
            local_src_dir_: string or None
                where to find the raw downloaded files, None -> "%(default_dir)s"
        '''
        local_dst_dir = self.DEFAULT_DIR if local_dst_dir_ is None else Path(local_dst_dir_)
        local_src_dir = self.DEFAULT_DIR if local_src_dir_ is None else Path(local_src_dir_)
        local_dst_dir.mkdir(parents=True, exist_ok=True)
        assert local_src_dir.exists()
        images = np.empty((60000,3,32,32), dtype=np.uint8)
        labels = np.empty((60000,), dtype=np.uint8)
        tarfile_name = str(local_src_dir / 'cifar-10-python.tar.gz')
        with tarfile.open(tarfile_name, 'r:gz') as tf:
            for i in range(5):
                with tf.extractfile('cifar-10-batches-py/data_batch_%d'%(i+1)) as f:
                    data_di = pickle.load(f, encoding='bytes')
                    images[(10000*i):(10000*(i+1))] = data_di[b'data'].reshape((10000,3,32,32))
                    labels[(10000*i):(10000*(i+1))] = np.asarray(data_di[b'labels'], dtype=np.uint8)
            with tf.extractfile('cifar-10-batches-py/test_batch') as f:
                data_di = pickle.load(f, encoding='bytes')
                images[50000:60000] = data_di[b'data'].reshape((10000,3,32,32))
                labels[50000:60000] = data_di[b'labels']
        np.savez_compressed(str(local_dst_dir / 'cifar10.npz'), images=images, labels=labels)

        if clean_install_:
            os.remove(tarfile_name) 

def load(self, local_dir_=None):
        if local_dir_ is None:
            local_dir = self.DEFAULT_DIR
        else:
            local_dir = Path(local_dir_)

        data = np.load(str(local_dir / 'mnist.npz'))
        self.labels = data['labels']
        self.datum = data['images']
        self.label_map = np.arange(10)
        self.imsize = (1,28,28) 

def load(self, local_dir_=None):
        # TODO
        raise NotImplementedError() 

def load_aggregate_masks_scans (masks_mnames, grids, upgrid_multis):
    
    scans = []
    masks = []
    
    igrid = 0
    for masks_names in masks_mnames:
        if (len(masks_names) > 0):      
            grid = grids[igrid]
            upgrid_multi = upgrid_multis[igrid]
            upgcount = upgrid_multi * upgrid_multi
            
            scans1 = []
            masks1 = []
            for masks_name in masks_names:
                print ("Loading: ", masks_name)
                masks0 =  np.load(''.join((masks_name, ".npz")))['arr_0']
                scans0 = np.load(''.join((masks_name.replace("masks_", "scans_", 1), ".npz")))['arr_0']
                masks1.append(masks0)
                scans1.append(scans0)
           
            scans1 = np.vstack(scans1)
            masks1 = np.vstack(masks1)
            if len(masks) > 0:
                scans1 = np.vstack([scans1, scans])
                masks1 = np.vstack([masks1, masks])
            
            lm = len(masks1) // upgcount * upgcount  
            scans1 = scans1[0:lm] # cut to multiples of upgcount
            masks1 = masks1[0:lm]
            index_shuf = np.arange(lm)
            np.random.shuffle(index_shuf)
            scans1 = scans1[index_shuf]
            masks1 = masks1[index_shuf]
            
            scans = data_from_grid_by_proximity(scans1, upgrid_multi, upgrid_multi, grid=grid)
            masks = data_from_grid_by_proximity(masks1, upgrid_multi, upgrid_multi, grid=grid)
        
        igrid += 1
        
    return masks, scans