Python graph.Graph() Examples

def main():
    '''
    Create render and a graph, get some edge calling some graph
    function and draw them, finally save the results in FILENAME

    Bergamo = [.337125 ,.245148]
    Roma = [.4936765,.4637286]
    Napoli = [.5936468,.5253573]
    '''
    render = Render()
    #coords = load_italy_coords()
    g = Graph(
        points=None, oriented=False, rand=True, n=300, max_neighbours=9)

    edges = g.tsp(0)

    render.draw_points(g.nodes)
    render.draw_lines(g.coords_edges(g.edges))
    render.sur.write_to_png(FILENAME)
    import pdb
    pdb.set_trace()
    render.draw_lines(g.coords_edges(edges), color=True, filename='tsp/tsp')

    #render.draw_lines(g.coords_edges(edges), color=True, filename='kruskal/kru')
    render.sur.write_to_png(FILENAME) 

def synthesize():
    if not os.path.exists(hp.sampledir): os.mkdir(hp.sampledir)

    # Load graph
    g = Graph(mode="synthesize"); print("Graph loaded")

    # Load data
    texts = load_data(mode="synthesize")

    saver = tf.train.Saver()
    with tf.Session() as sess:
        saver.restore(sess, tf.train.latest_checkpoint(hp.logdir)); print("Restored!")

        # Feed Forward
        ## mel
        y_hat = np.zeros((texts.shape[0], 200, hp.n_mels*hp.r), np.float32)  # hp.n_mels*hp.r
        for j in tqdm.tqdm(range(200)):
            _y_hat = sess.run(g.y_hat, {g.x: texts, g.y: y_hat})
            y_hat[:, j, :] = _y_hat[:, j, :]
        ## mag
        mags = sess.run(g.z_hat, {g.y_hat: y_hat})
        for i, mag in enumerate(mags):
            print("File {}.wav is being generated ...".format(i+1))
            audio = spectrogram2wav(mag)
            write(os.path.join(hp.sampledir, '{}.wav'.format(i+1)), hp.sr, audio) 

def init_graph(self):
        self.preprocess = TextProcessor()
        self.graph = Graph() 

def add_row( self, name, scale ):
        ln = Gtk.Label( name )
        vlabel = Gtk.Label( "0.00 / 0.00" )
        graph = Graph( scale )
    
        self.graphs.append( graph )
        self.values.append( vlabel )

        numrows = len(self.graphs)
        self.attach( ln, 0, 1, numrows, numrows+1, Gtk.AttachOptions.FILL, Gtk.AttachOptions.FILL )
        self.attach( graph, 1, 2, numrows, numrows+1 )
        self.attach( vlabel, 2, 3, numrows, numrows+1, Gtk.AttachOptions.FILL, Gtk.AttachOptions.FILL ) 

def train2():
    g = Graph("train2"); print("Training Graph loaded")

    with tf.Session() as sess:
        # Initialize all variables
        sess.run(tf.global_variables_initializer())

        # Restore
        logdir = hp.logdir + "/train1"
        var_list = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, 'net1')
        saver = tf.train.Saver(var_list=var_list)
        ckpt = tf.train.latest_checkpoint(logdir)
        if ckpt is not None: saver.restore(sess, ckpt)

        logdir = hp.logdir + "/train2"
        var_list = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, 'net2') +\
                   tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, 'training')
        saver2 = tf.train.Saver(var_list=var_list)
        ckpt = tf.train.latest_checkpoint(logdir)
        if ckpt is not None: saver2.restore(sess, ckpt)

        # Writer & Queue
        writer = tf.summary.FileWriter(logdir, sess.graph)
        coord = tf.train.Coordinator()
        threads = tf.train.start_queue_runners(coord=coord)

        while 1:
            for _ in tqdm(range(g.num_batch), total=g.num_batch, ncols=70, leave=False, unit='b'):
                gs, _ = sess.run([g.global_step, g.train_op])

            merged = sess.run(g.merged)
            writer.add_summary(merged, global_step=gs)

            # Save
            saver2.save(sess, logdir + '/model_gs'.format(gs))

        writer.close()
        coord.request_stop()
        coord.join(threads) 

def eval1():
    # Load data
    mfccs, phns = load_data(mode="eval1")

    # Graph
    g = Graph("eval1"); print("Evaluation Graph loaded")
    logdir = hp.logdir + "/train1"

    # Session
    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())

        # Restore saved variables
        var_list = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, 'net1') +\
                   tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, 'training')
        saver = tf.train.Saver(var_list=var_list)

        ckpt = tf.train.latest_checkpoint(logdir)
        if ckpt is not None: saver.restore(sess, ckpt)

        # Writer
        writer = tf.summary.FileWriter(logdir, sess.graph)

        # Evaluation
        merged, gs = sess.run([g.merged, g.global_step], {g.mfccs: mfccs, g.phones: phns})

        #  Write summaries
        writer.add_summary(merged, global_step=gs)
        writer.close() 

def convert():
    g = Graph("convert"); print("Training Graph loaded")
    mfccs = load_data("convert")

    with tf.Session() as sess:
        # Initialize all variables
        sess.run(tf.global_variables_initializer())

        # Restore
        logdir = hp.logdir + "/train1"
        var_list = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, 'net1')
        saver = tf.train.Saver(var_list=var_list)
        ckpt = tf.train.latest_checkpoint(logdir)
        if ckpt is not None: saver.restore(sess, ckpt)

        logdir = hp.logdir + "/train2"
        var_list = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, 'net2') +\
                   tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, 'training')
        saver2 = tf.train.Saver(var_list=var_list)
        ckpt = tf.train.latest_checkpoint(logdir)
        if ckpt is not None: saver2.restore(sess, ckpt)

        # Synthesize
        if not os.path.exists('50lang-output'): os.mkdir('50lang-output')

        mag_hats = sess.run(g.mag_hats, {g.mfccs: mfccs})
        for i, mag_hat in enumerate(mag_hats):
            wav = spectrogram2wav(mag_hat)
            write('50lang-output/{}.wav'.format(i+1), hp.sr, wav) 

def read_graph_from_adj(adj,dataset_name):
    '''Assume idx starts from *1* and are continuous. Edge shows up twice. Assume single connected component.'''
#    logging.info("Reading graph from metis...")
    with open("data/ind.{}.{}".format(dataset_name, 'graph'), 'rb') as f:
        if sys.version_info > (3, 0):
            in_file = pkl.load(f, encoding='latin1')
        else:
            in_file = pkl.load(f)
    weighted = False 
    node_num = adj.shape[0]
    edge_num = np.count_nonzero(adj.toarray()) * 2
    graph = Graph(node_num, edge_num)
    edge_cnt = 0
    graph.adj_idx[0] = 0
    for idx in range(node_num):
        graph.node_wgt[idx] = 1
        eles = in_file[idx]
        j = 0 
        while j < len(eles):
            neigh = int(eles[j])  #
            if weighted:
                wgt = float(eles[j+1])
            else:
                wgt = 1.0
            graph.adj_list[edge_cnt] = neigh # self-loop included.
            graph.adj_wgt[edge_cnt] = wgt
            graph.degree[idx] += wgt
            edge_cnt += 1
            if weighted:
                j += 2
            else:
                j += 1
        graph.adj_idx[idx+1] = edge_cnt
    graph.A = graph_to_adj(graph, self_loop=False)
    # check connectivity in debug mode
    # if ctrl.debug_mode:
    #     assert nx.is_connected(graph2nx(graph))
    return graph, None 

def __init__(self, word2vec=None):
        self.preprocess = TextProcessor()
        self.graph = Graph()
        if word2vec:
            print("Loading word2vec embedding...")
            self.word2vec = KeyedVectors.load_word2vec_format(word2vec, binary=True)
            print("Succesfully loaded word2vec embeddings!")
        else:
            self.word2vec = None 

def init_population(self, population_size, graph_max_layer, graph_min_layer):
        """
        initialize populations for evolution tuner
        """
        population = []
        graph = Graph(max_layer_num=graph_max_layer, min_layer_num=graph_min_layer,
                      inputs=[Layer(LayerType.input.value, output=[4, 5], size='x'), Layer(LayerType.input.value, output=[4, 5], size='y')],
                      output=[Layer(LayerType.output.value, inputs=[4], size='x'), Layer(LayerType.output.value, inputs=[5], size='y')],
                      hide=[Layer(LayerType.attention.value, inputs=[0, 1], output=[2]),
                            Layer(LayerType.attention.value, inputs=[1, 0], output=[3])])
        for _ in range(population_size):
            graph_tmp = copy.deepcopy(graph)
            graph_tmp.mutation()
            population.append(Individual(indiv_id=self.generate_new_id(), graph_cfg=graph_tmp, result=None))
        return population 

def mutation(self, indiv_id: int, graph_cfg: Graph = None, info=None):
        self.result = None
        if graph_cfg is not None:
            self.config = graph_cfg
        self.config.mutation()
        self.info = info
        self.parent_id = self.indiv_id
        self.indiv_id = indiv_id
        self.shared_ids.intersection_update({layer.hash_id for layer in self.config.layers if layer.is_delete is False}) 

def __init__(self, graph_cfg: Graph = None, info=None, result=None, indiv_id=None):
        self.config = graph_cfg
        self.result = result
        self.info = info
        self.indiv_id = indiv_id
        self.parent_id = None
        self.shared_ids = {layer.hash_id for layer in self.config.layers if layer.is_delete is False} 

def parse_profile_file_to_graph(profile_filename, directory):
    gr = graph.Graph()
    node_id = 0
    with open(profile_filename, 'r') as f:
        csv_reader = csv.reader(f)
        line_id = 0
        profile_data = []
        prev_node = None
        for line in csv_reader:
            if line_id == 0:
                header = line
                num_minibatches = None
                for header_elem in header:
                    if "Forward pass time" in header_elem:
                        num_minibatches = int(header_elem.split("(")[1].rstrip(")"))
            else:
                total_time = float(line[header.index("Total time")]) / num_minibatches
                for i in xrange(len(header)):
                    if "Output Size" in header[i]:
                        if line[i] == '':
                            output_size = 0
                        else:
                            output_size = float(line[i].replace(",", ""))
                        break
                parameter_size = float(line[header.index("Parameter Size (floats)")].replace(",", ""))
                node_desc = line[header.index("Layer Type")]
                node = graph.Node("node%d" % node_id, node_desc=node_desc,
                                  compute_time=total_time * 1000,
                                  parameter_size=(4.0 * parameter_size),
                                  activation_size=(output_size * 4.0))
                node_id += 1
                if prev_node is not None:
                    gr.add_edge(prev_node, node)
                prev_node = node
            line_id += 1

    gr.to_dot(os.path.join(directory, "graph.dot"))
    with open(os.path.join(directory, "graph.txt"), 'w') as f:
        f.write(str(gr))

    return gr 

def __init__(self, model, summary, module_whitelist):
        if isinstance(model, torch.nn.Module) is False:
            raise Exception("Not a valid model, please provide a 'nn.Module' instance.")

        self.model = model
        self.module_whitelist = module_whitelist
        self.summary = copy.deepcopy(summary)
        self.forward_original_methods = {}
        self.graph = graph.Graph()
        self.inputs = {} 

def modelFromCheckpoint():
    if FLAGS.baseline is not '':
        return None, None
    tf.reset_default_graph()
    g = graph.Graph()
    saver = tf.train.Saver()
    sess = tf.Session()
    saver.restore(sess, checkpointPath())
    return g, sess 

def create_coarse_graph(graph, groups, coarse_graph_size):
    '''create the coarser graph and return it based on the groups array and coarse_graph_size'''
    coarse_graph = Graph(coarse_graph_size, graph.edge_num)
    coarse_graph.finer = graph
    graph.coarser = coarse_graph
    cmap = graph.cmap
    adj_list = graph.adj_list
    adj_idx = graph.adj_idx
    adj_wgt = graph.adj_wgt
    node_wgt = graph.node_wgt

    coarse_adj_list = coarse_graph.adj_list
    coarse_adj_idx = coarse_graph.adj_idx
    coarse_adj_wgt = coarse_graph.adj_wgt
    coarse_node_wgt = coarse_graph.node_wgt
    coarse_degree = coarse_graph.degree

    coarse_adj_idx[0] = 0
    nedges = 0  # number of edges in the coarse graph
    for idx in range(len(groups)):  # idx in the graph
        coarse_node_idx = idx
        neigh_dict = dict()  # coarser graph neighbor node --> its location idx in adj_list.
        group = groups[idx]
        for i in range(len(group)):
            merged_node = group[i]
            if (i == 0):
                coarse_node_wgt[coarse_node_idx] = node_wgt[merged_node]
            else:
                coarse_node_wgt[coarse_node_idx] += node_wgt[merged_node]

            istart = adj_idx[merged_node]
            iend = adj_idx[merged_node + 1]
            for j in range(istart, iend):
                k = cmap[adj_list[
                    j]]  # adj_list[j] is the neigh of v; k is the new mapped id of adj_list[j] in coarse graph.
                if k not in neigh_dict:  # add new neigh
                    coarse_adj_list[nedges] = k
                    coarse_adj_wgt[nedges] = adj_wgt[j]
                    neigh_dict[k] = nedges
                    nedges += 1
                else:  # increase weight to the existing neigh
                    coarse_adj_wgt[neigh_dict[k]] += adj_wgt[j]
                # add weights to the degree. For now, we retain the loop. 
                coarse_degree[coarse_node_idx] += adj_wgt[j]

        coarse_node_idx += 1
        coarse_adj_idx[coarse_node_idx] = nedges

    coarse_graph.edge_num = nedges

    coarse_graph.resize_adj(nedges)
    C = cmap2C(cmap)  # construct the matching matrix.
    graph.C = C
    coarse_graph.A = C.transpose().dot(graph.A).dot(C)
    return coarse_graph 

def run(self):
        # parse config file to get components
        self.parse_configs()
        # convert components to nodes, inputs and outputs
        self.convert_components()
        # to build graph, graph will take over the converting work
        g = Graph(self._nodes,
                  self._inputs,
                  self._outputs,
                  self._batch,
                  self._chunk_size,
                  self._left_context,
                  self._right_context,
                  self._modulus,
                  self._input_dims,
                  self._subsample_factor,
                  self._nnet_type,
                  self._fuse_lstm,
                  self._fuse_stats)

        onnx_model = g.run()
        input_info, output_info, cache_info = g.model_interface_info()
        input_nodes_str, input_shapes_str = self.nodes_info_to_str(input_info)

        output_nodes_str, output_shapes_str = \
            self.nodes_info_to_str(output_info)

        left_context_conf = "--left-context=" + str(self._left_context) + "\n"
        right_context_conf = \
            "--right-context=" + str(self._right_context) + "\n"
        modulus_conf = "--modulus=" + str(self._modulus) + "\n"
        frames_per_chunk_conf = \
            "--frames-per-chunk=" + str(self._chunk_size) + "\n"
        subsample_factor_conf = \
            "--frame-subsampling-factor=" + str(self._subsample_factor) + "\n"
        conf_lines = [left_context_conf, right_context_conf, modulus_conf,
                      frames_per_chunk_conf, subsample_factor_conf]

        input_node_conf = "--input-nodes=" + input_nodes_str + "\n"
        input_shapes_conf = "--input-shapes=" + input_shapes_str + "\n"
        conf_lines.append(input_node_conf)
        conf_lines.append(input_shapes_conf)
        output_node_conf = "--output-nodes=" + output_nodes_str + "\n"
        output_shapes_conf = "--output-shapes=" + output_shapes_str + "\n"
        conf_lines.append(output_node_conf)
        conf_lines.append(output_shapes_conf)
        if len(cache_info) > 0:
            cache_nodes_str, cache_shapes_str =\
                self.nodes_info_to_str(cache_info)
            cache_node_conf = "--cache-nodes=" + cache_nodes_str + "\n"
            cache_shapes_conf = "--cache-shapes=" + cache_shapes_str + "\n"
            conf_lines.append(cache_node_conf)
            conf_lines.append(cache_shapes_conf)
        return onnx_model, conf_lines, self._transition_model 

def train1():
    g = Graph(); print("Training Graph loaded")
    logdir = hp.logdir + "/train1"

    # Session
    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())

        # Restore saved variables
        var_list = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, 'net1') + \
                   tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, 'training')
        saver = tf.train.Saver(var_list=var_list)
        ckpt = tf.train.latest_checkpoint(logdir)
        if ckpt is not None: saver.restore(sess, ckpt)

        # Writer & Queue
        writer = tf.summary.FileWriter(logdir, sess.graph)
        coord = tf.train.Coordinator()
        threads = tf.train.start_queue_runners(coord=coord)

        # Inspect variables
        saver.save(sess, logdir + '/model_gs_0')
        from tensorflow.python.tools.inspect_checkpoint import print_tensors_in_checkpoint_file
        print_tensors_in_checkpoint_file(file_name=hp.logdir+'/train1/model_gs_0', tensor_name='', all_tensors=False)

        # Training
        while 1:
            for _ in tqdm(range(g.num_batch), total=g.num_batch, ncols=70, leave=False, unit='b'):
                gs, _ = sess.run([g.global_step, g.train_op])

            # Write checkpoint files at every epoch
            merged = sess.run(g.merged)
            writer.add_summary(merged, global_step=gs)

            # evaluation
            with tf.Graph().as_default(): eval1()

            # Save
            saver.save(sess, logdir + '/model_gs_{}'.format(gs))

            if gs > 10000: break

        writer.close()
        coord.request_stop()
        coord.join(threads) 

def amr2graph(id, amr, full = False, reify = False, alignment = None):
    graph = Graph(id, flavor = 2, framework = "amr")
    node2id = {}
    i = 0
    for n, v, a in zip(amr.nodes, amr.node_values, amr.attributes):
        j = i
        node2id[n] = j
        top = False;
        for key, val in a:
            if key == "TOP":
                top = True;
        node = graph.add_node(j, label = v, top=top)
        i += 1
        for key, val in a:
            if key != "TOP" \
               and (key not in {"wiki"} or full):
                if val.endswith("¦"):
                    val = val[:-1];
                if reify:
                    graph.add_node(i, label=val)
                    graph.add_edge(j, i, key)
                    i += 1
                else:
                    node.set_property(key, val);

    for src, r in zip(amr.nodes, amr.relations):
        for label, tgt in r:
            normal = None;
            if label == "mod":
                normal = "domain";
            elif label.endswith("-of-of") \
                 or label.endswith("-of") \
                   and label not in {"consist-of" "subset-of"} \
                   and not label.startswith("prep-"):
                normal = label[:-3];
            graph.add_edge(node2id[src], node2id[tgt], label, normal)

    overlay = None;
    if alignment is not None:
        overlay = Graph(id, flavor = 2, framework = "alignment");
        for node in alignment:
            for path, span in alignment[node].items():
                if len(path) == 0:
                    node = overlay.add_node(node2id[node], label = tuple(span));
        for node in alignment:
            i = node2id[node];
            for path, span in alignment[node].items():
                if len(path) == 1:
                    node = overlay.find_node(i);
                    if node is None: node = overlay.add_node(i);
                    node.set_property(path[0], tuple(span));
                elif len(path) > 1:
                    print("amr2graph(): ignoring alignment path {} on node #{} ({})"
                          "".format(path, source, node));

    return graph, overlay; 

def evaluate():
    # Load graph
    g = Graph(mode="evaluate"); print("Graph loaded")

    # Load data
    fpaths, _, texts = load_data(mode="evaluate")
    lengths = [len(t) for t in texts]
    maxlen = sorted(lengths, reverse=True)[0]
    new_texts = np.zeros((len(texts), maxlen), np.int32)
    for i, text in enumerate(texts):
        new_texts[i, :len(text)] = [idx for idx in text]
    #new_texts = np.split(new_texts, 2)
    new_texts = new_texts[:evaluate_wav_num]
    half_size = int(len(fpaths)/2)
    print(half_size)
    #new_fpaths = [fpaths[:half_size], fpaths[half_size:]]
    fpaths = fpaths[:evaluate_wav_num]
    saver = tf.train.Saver()
    with tf.Session() as sess:
        saver.restore(sess, tf.train.latest_checkpoint(hp.logdir)); print("Evaluate Model Restored!")
        """
        err = 0.0

        for i, t_split in enumerate(new_texts):
            y_hat = np.zeros((t_split.shape[0], 200, hp.n_mels*hp.r), np.float32)  # hp.n_mels*hp.r
            for j in tqdm.tqdm(range(200)):
                _y_hat = sess.run(g.y_hat, {g.x: t_split, g.y: y_hat})
                y_hat[:, j, :] = _y_hat[:, j, :]

            mags = sess.run(g.z_hat, {g.y_hat: y_hat})
            for k, mag in enumerate(mags):
                fname, mel_ans, mag_ans = load_spectrograms(new_fpaths[i][k])
                print("File {} is being evaluated ...".format(fname))
                audio = spectrogram2wav(mag)
                audio_ans = spectrogram2wav(mag_ans)
                err += calculate_mse(audio, audio_ans)

        err = err/float(len(fpaths))
        print(err)

        """
        # Feed Forward
        ## mel
        y_hat = np.zeros((new_texts.shape[0], 200, hp.n_mels*hp.r), np.float32)  # hp.n_mels*hp.r
        for j in tqdm.tqdm(range(200)):
            _y_hat = sess.run(g.y_hat, {g.x: new_texts, g.y: y_hat})
            y_hat[:, j, :] = _y_hat[:, j, :]
        ## mag
        mags = sess.run(g.z_hat, {g.y_hat: y_hat})
        err = 0.0
        for i, mag in enumerate(mags):
            fname, mel_ans, mag_ans = load_spectrograms(fpaths[i])
            print("File {} is being evaluated ...".format(fname))
            #audio = spectrogram2wav(mag)
            #audio_ans = spectrogram2wav(mag_ans)
            #err += calculate_mse(audio, audio_ans)
            err += calculate_mse(mag, mag_ans)
        err = err/float(len(fpaths))
        print(err)
        opf.write(hp.logdir  + " spectrogram mse: " + str(err) + "\n")