Python seaborn.set() Examples

def sequencing_speed_over_time(dfs, path, figformat, title, plot_settings={}):
    time_duration = Plot(path=path + "TimeSequencingSpeed_ViolinPlot." + figformat,
                         title="Violin plot of sequencing speed over time")
    sns.set(style="white", **plot_settings)
    if "timebin" not in dfs:
        dfs['timebin'] = add_time_bins(dfs)
    mask = dfs['duration'] != 0
    ax = sns.violinplot(x=dfs.loc[mask, "timebin"],
                        y=dfs.loc[mask, "lengths"] / dfs.loc[mask, "duration"],
                        inner=None,
                        cut=0,
                        linewidth=0)
    ax.set(xlabel='Interval (hours)',
           ylabel="Sequencing speed (nucleotides/second)",
           title=title or time_duration.title)
    plt.xticks(rotation=45, ha='center', fontsize=8)
    time_duration.fig = ax.get_figure()
    time_duration.save(format=figformat)
    plt.close("all")
    return time_duration 

def quality_over_time(dfs, path, figformat, title, plot_settings={}):
    time_qual = Plot(path=path + "TimeQualityViolinPlot." + figformat,
                     title="Violin plot of quality over time")
    sns.set(style="white", **plot_settings)
    ax = sns.violinplot(x="timebin",
                        y="quals",
                        data=dfs,
                        inner=None,
                        cut=0,
                        linewidth=0)
    ax.set(xlabel='Interval (hours)',
           ylabel="Basecall quality",
           title=title or time_qual.title)
    plt.xticks(rotation=45, ha='center', fontsize=8)
    time_qual.fig = ax.get_figure()
    time_qual.save(format=figformat)
    plt.close("all")
    return time_qual 

def __init__(self, add_inputs, title='', **kwargs):
        super(OffshorePlot, self).__init__(**kwargs)
        self.fig = plt.figure(num=None, facecolor='w', edgecolor='k') #figsize=(13, 8), dpi=1000
        self.shape_plot = self.fig.add_subplot(121)
        self.objf_plot = self.fig.add_subplot(122)

        self.targname = add_inputs
        self.title = title

        # Adding automatically the inputs
        for i in add_inputs:
            self.add(i, Float(0.0, iotype='in'))

        #sns.set(style="darkgrid")
        #self.pal = sns.dark_palette("skyblue", as_cmap=True)
        plt.rc('lines', linewidth=1)
        plt.ion()
        self.force_execute = True
        if not pa('fig').exists():
            pa('fig').mkdir() 

def image(path: str, costs: Dict[str, int]) -> str:
    ys = ['0', '1', '2', '3', '4', '5', '6', '7+', 'X']
    xs = [costs.get(k, 0) for k in ys]
    sns.set_style('white')
    sns.set(font='Concourse C3', font_scale=3)
    g = sns.barplot(ys, xs, palette=['#cccccc'] * len(ys))
    g.axes.yaxis.set_ticklabels([])
    rects = g.patches
    sns.set(font='Concourse C3', font_scale=2)
    for rect, label in zip(rects, xs):
        if label == 0:
            continue
        height = rect.get_height()
        g.text(rect.get_x() + rect.get_width()/2, height + 0.5, label, ha='center', va='bottom')
    g.margins(y=0, x=0)
    sns.despine(left=True, bottom=True)
    g.get_figure().savefig(path, transparent=True, pad_inches=0, bbox_inches='tight')
    plt.clf() # Clear all data from matplotlib so it does not persist across requests.
    return path 

def yield_by_minimal_length_plot(array, name, path,
                                 title=None, color="#4CB391", figformat="png"):
    df = pd.DataFrame(data={"lengths": np.sort(array)[::-1]})
    df["cumyield_gb"] = df["lengths"].cumsum() / 10**9
    yield_by_length = Plot(
        path=path + "Yield_By_Length." + figformat,
        title="Yield by length")
    ax = sns.regplot(
        x='lengths',
        y="cumyield_gb",
        data=df,
        x_ci=None,
        fit_reg=False,
        color=color,
        scatter_kws={"s": 3})
    ax.set(
        xlabel='Read length',
        ylabel='Cumulative yield for minimal length',
        title=title or yield_by_length.title)
    yield_by_length.fig = ax.get_figure()
    yield_by_length.save(format=figformat)
    plt.close("all")
    return yield_by_length 

def make_biplot(self, pc_x=1, pc_y=2, outpath=None, dpi=150, custom_markers=None, custom_order=None):
        if not custom_order:
            custom_order = sorted(self.observations_df[self.observation_colname].unique().tolist())
        if not custom_markers:
            custom_markers = self.markers
        plot = sns.lmplot(data=self.principal_observations_df,
                              x=self.principal_observations_df.columns[pc_x - 1],
                              y=self.principal_observations_df.columns[pc_y - 1],
                              hue=self.observation_colname,
                              hue_order=custom_order,
                              fit_reg=False,
                              size=6,
                              markers=custom_markers,
                            scatter_kws={'alpha': 0.5})
        plot = (plot.set(title='PC{} vs. PC{}'.format(pc_x, pc_y)))
        if outpath:
            plot.savefig(outpath, dpi=dpi)
        else:
            plt.show()
        plt.close() 

def _plot_weights(self, title, file, layer_index=0, vmin=-5, vmax=5):
        import seaborn as sns
        sns.set_context("paper")

        layers = self.iwp.estimator.steps[-1][1].coefs_
        layer = layers[layer_index]
        f, ax = plt.subplots(figsize=(18, 12))
        weights = pd.DataFrame(layer)
        weights.index = self.iwp.inputs

        sns.set(font_scale=1.1)

        # Draw a heatmap with the numeric values in each cell
        sns.heatmap(
            weights, annot=True, fmt=".1f", linewidths=.5, ax=ax,
            cmap="difference", center=0, vmin=vmin, vmax=vmax,
            # annot_kws={"size":14},
        )
        ax.tick_params(labelsize=18)
        f.tight_layout()
        f.savefig(file) 

def enrich_activity(seqs, seqs_1hot, targets, activity_enrich, target_indexes):
  """ Filter data for the most active sequences in the set. """

  # compute the max across sequence lengths and mean across targets
  seq_scores = targets[:, :, target_indexes].max(axis=1).mean(
      axis=1, dtype='float64')

  # sort the sequences
  scores_indexes = [(seq_scores[si], si) for si in range(seq_scores.shape[0])]
  scores_indexes.sort(reverse=True)

  # filter for the top
  enrich_indexes = sorted(
      [scores_indexes[si][1] for si in range(seq_scores.shape[0])])
  enrich_indexes = enrich_indexes[:int(activity_enrich * len(enrich_indexes))]
  seqs = [seqs[ei] for ei in enrich_indexes]
  seqs_1hot = seqs_1hot[enrich_indexes]
  targets = targets[enrich_indexes]

  return seqs, seqs_1hot, targets 

def get_protein_feather_paths(protgroup, memornot, protgroup_dict, protein_feathers_dir, core_only_genes=None):
    """
    protgroup example: ('subsystem', 'cog_primary', 'H')
    memornot example: ('vizrecon', 'membrane')
    protgroup_dict example: {'databases': {'redoxdb': {'experimental_sensitive_cys': ['b2518','b3352','b2195','b4016'], ...}}}
    """
    prots_memornot = protgroup_dict['localization'][memornot[0]][memornot[1]]

    if protgroup[0] == 'localization':
        if protgroup[2] != 'all':
            if memornot[1] in ['membrane', 'inner_membrane', 'outer_membrane'] and protgroup[2] not in ['membrane', 'inner_membrane', 'outer_membrane']:
                return []
            if memornot[1] not in ['membrane', 'inner_membrane', 'outer_membrane'] and protgroup[2] in ['membrane', 'inner_membrane', 'outer_membrane']:
                return []

    prots_group = protgroup_dict[protgroup[0]][protgroup[1]][protgroup[2]]
    prots_filtered = list(set(prots_group).intersection(prots_memornot))
    if core_only_genes:
        prots_filtered = list(set(prots_filtered).intersection(core_only_genes))
    return [op.join(protein_feathers_dir, '{}_protein_strain_properties.fthr'.format(x)) for x in prots_filtered if op.exists(op.join(protein_feathers_dir, '{}_protein_strain_properties.fthr'.format(x)))] 

def scatter_plot_intent_dist(workspace_pd):
    """
    takes the workspace_pd and generate a scatter distribution of the intents
    :param workspace_pd:
    :return:
    """

    label_frequency = Counter(workspace_pd["intent"]).most_common()
    frequencies = list(reversed(label_frequency))
    counter_list = list(range(1, len(frequencies) + 1))
    df = pd.DataFrame(data=frequencies, columns=["Intent", "Number of User Examples"])
    df["Intent"] = counter_list

    sns.set(rc={"figure.figsize": (15, 10)})
    display(
        Markdown(
            '## <p style="text-align: center;">Sorted Distribution of User Examples \
                     per Intent</p>'
        )
    )

    plt.ylabel("Number of User Examples", fontdict=LABEL_FONT)
    plt.xlabel("Intent", fontdict=LABEL_FONT)
    ax = sns.scatterplot(x="Intent", y="Number of User Examples", data=df, s=100) 

def plot_wind_rose(wind_rose):
    fig = plt.figure(figsize=(12,5), dpi=1000)

    # Plotting the wind statistics
    ax1 = plt.subplot(121, polar=True)
    w = 2.*np.pi/len(wind_rose.frequency)
    b = ax1.bar(np.pi/2.0-np.array(wind_rose.wind_directions)/180.*np.pi - w/2.0, 
                np.array(wind_rose.frequency)*100, width=w)

    # Trick to set the right axes (by default it's not oriented as we are used to in the WE community)
    mirror = lambda d: 90.0 - d if d < 90.0 else 360.0 + (90.0 - d)
    ax1.set_xticklabels([u'%d\xb0'%(mirror(d)) for d in linspace(0.0, 360.0,9)[:-1]]);
    ax1.set_title('Wind direction frequency');

    # Plotting the Weibull A parameter
    ax2 = plt.subplot(122, polar=True)
    b = ax2.bar(np.pi/2.0-np.array(wind_rose.wind_directions)/180.*np.pi - w/2.0, 
                np.array(wind_rose.A), width=w)
    ax2.set_xticklabels([u'%d\xb0'%(mirror(d)) for d in linspace(0.0, 360.0,9)[:-1]]);
    ax2.set_title('Weibull A parameter per wind direction sectors'); 

def make_reddit_prop_plt():
    sns.set()
    prop_expt = pd.DataFrame(att.process_propensity_experiment())

    prop_expt = prop_expt[['exog', 'plugin', 'one_step_tmle', 'very_naive']]
    prop_expt = prop_expt.rename(index=str, columns={'exog': 'Exogeneity',
                                         'very_naive': 'Unadjusted',
                                         'plugin': 'Plug-in',
                                         'one_step_tmle': 'TMLE'})
    prop_expt = prop_expt.set_index('Exogeneity')

    plt.figure(figsize=(4.75, 3.00))
    # plt.figure(figsize=(2.37, 1.5))
    sns.scatterplot(data=prop_expt, legend='brief', s=75)
    plt.xlabel("Exogeneity", fontfamily='monospace')
    plt.ylabel("NDE Estimate", fontfamily='monospace')
    plt.tight_layout()

    fig_dir = '../output/figures'
    os.makedirs(fig_dir, exist_ok=True)
    plt.savefig(os.path.join(fig_dir,'reddit_propensity.pdf')) 

def plot_kernel(kernel_weights, out_pdf):
    depth, width = kernel_weights.shape
    fig_width = 2 + 1.5*np.log2(width)

    # normalize
    kernel_weights -= kernel_weights.mean(axis=0)

    # plot
    sns.set(font_scale=1.5)
    plt.figure(figsize=(fig_width, depth))
    sns.heatmap(kernel_weights, cmap='PRGn', linewidths=0.2, center=0)
    ax = plt.gca()
    ax.set_xticklabels(range(1,width+1))

    if depth == 4:
        ax.set_yticklabels('ACGT', rotation='horizontal')
    else:
        ax.set_yticklabels(range(1,depth+1), rotation='horizontal')

    plt.savefig(out_pdf)
    plt.close() 

def getEdgeHistogram(inGraph, refGraph):
    falsePositives = set(inGraph.edges()).difference(refGraph.edges())
    edgeHistogramCounts = {0:0}
    
    for fe in falsePositives:
        u,v = fe
        try:
            path = nx.dijkstra_path(refGraph,u,v)
            pathlength = len(path) -1 
            if pathlength in edgeHistogramCounts.keys():
                edgeHistogramCounts[pathlength] +=1
            else:
                edgeHistogramCounts[pathlength] = 0
            
        except nx.exception.NetworkXNoPath:
            edgeHistogramCounts[0] +=1
    return edgeHistogramCounts 

def main():
    args = parse_args()
    X, labels = np.loadtxt(args.embeddings_path), np.loadtxt(args.labels_path, dtype=np.str)
    tsne = TSNE(n_components=2, n_iter=10000, perplexity=5, init='pca', learning_rate=200, verbose=1)
    transformed = tsne.fit_transform(X)

    y = set(labels)
    labels = np.array(labels)
    plt.figure(figsize=(20, 14))
    colors = cm.rainbow(np.linspace(0, 1, len(y)))
    for label, color in zip(y, colors):
        points = transformed[labels == label, :]
        plt.scatter(points[:, 0], points[:, 1], c=[color], label=label, s=200, alpha=0.5)
        for p1, p2 in random.sample(list(zip(points[:, 0], points[:, 1])), k=min(1, len(points))):
            plt.annotate(label, (p1, p2), fontsize=30)

    plt.savefig('tsne_visualization.png', transparent=True, bbox_inches='tight', pad_inches=0)
    plt.show() 

def generate_speed_graph(data, filename="als_speed.png", keys=['gpu', 'cg2', 'cg3', 'cholesky'],
                         labels=None, colours=None):
    labels = labels or {}
    colours = colours or {}

    seaborn.set()
    fig, ax = plt.subplots()

    factors = data['factors']
    for key in keys:
        ax.plot(factors, data[key],
                color=colours.get(key, COLOURS.get(key)),
                marker='o', markersize=6)

        ax.text(factors[-1] + 5, data[key][-1], labels.get(key, LABELS[key]), fontsize=10)

    ax.set_ylabel("Seconds per Iteration")
    ax.set_xlabel("Factors")
    plt.savefig(filename, bbox_inches='tight', dpi=300) 

def main():
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('logdir', nargs='*')
    parser.add_argument('--legend', nargs='*')
    parser.add_argument('--value', default='AverageReturn', nargs='*')
    args = parser.parse_args()

    use_legend = False
    if args.legend is not None:
        assert len(args.legend) == len(args.logdir), \
            "Must give a legend title for each set of experiments."
        use_legend = True

    data = []
    if use_legend:
        for logdir, legend_title in zip(args.logdir, args.legend):
            data += get_datasets(logdir, legend_title)
    else:
        for logdir in args.logdir:
            data += get_datasets(logdir)

    if isinstance(args.value, list):
        values = args.value
    else:
        values = [args.value]
    for value in values:
        plot_data(data, value=value) 

def plot_data(data, value="AverageReturn"):
    if isinstance(data, list):
        data = pd.concat(data, ignore_index=True)
    sns.set(style="darkgrid", font_scale=1.5)
    sns.tsplot(data=data, time="Iteration", value=value, unit="Unit", condition="Condition")
    plt.legend(loc='best').draggable()
    plt.show() 

def main():
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('logdir', nargs='*')
    parser.add_argument('--legend', nargs='*')
    parser.add_argument('--value', default='AverageReturn', nargs='*')
    args = parser.parse_args()

    use_legend = False
    if args.legend is not None:
        assert len(args.legend) == len(args.logdir), \
            "Must give a legend title for each set of experiments."
        use_legend = True

    data = []
    if use_legend:
        for logdir, legend_title in zip(args.logdir, args.legend):
            data += get_datasets(logdir, legend_title)
    else:
        for logdir in args.logdir:
            data += get_datasets(logdir)

    if isinstance(args.value, list):
        values = args.value
    else:
        values = [args.value]
    for value in values:
        plot_data(data, value=value) 

def end_plotting(fig, ax, title=None, xlabel=None,
                 ylabel=None, xlim=None, ylim=None, filename=None,
                 xticklabel=None, xlabel_rotation=None,
                 yticklabel=None, ylabel_rotation=None, label_text=None,
                 xtickgap=None):
    '''A set of common operations after plotting.'''
    if title:
        ax.set_title(title)
    if xlabel:
        ax.set_xlabel(xlabel)
    if xticklabel:
        ax.set_xticks(xticklabel[0])
        ax.set_xticklabels(xticklabel[1], rotation=xlabel_rotation)
    elif xtickgap:
        xticks = ax.get_xticks()
        ax.set_xticks(list(frange(min(xticks), max(xticks) + 0.001, xtickgap)))
    else:
        ax.set_xticks(ax.get_xticks())
        ax.set_xticklabels(ax.get_xticks())
    if yticklabel:
        ax.set_yticks(yticklabel[0])
        ax.set_yticklabels(yticklabel[1], rotation=ylabel_rotation)
    else:
        ax.set_yticks(ax.get_yticks())
        ax.set_yticklabels(ax.get_yticks())
    if ylabel:
        ax.set_ylabel(ylabel)
    if xlim:
        ax.set_xlim(xlim)
    if ylim:
        ax.set_ylim(ylim)
    if label_text:
        for x, y, t in label_text:
            ax.text(x, y, t) 

def generate_unique_thresholds(sorted_results_tuples):
    """
    generate list of unique thresholds based off changes in confidence
    and sorted list of unique confidences
    :return: unique_thresholds
    """
    sort_uniq_confs = list(sorted(set([info[2] for info in sorted_results_tuples])))
    thresholds = [0]
    thresholds.extend(
        [
            (sort_uniq_confs[idx] + sort_uniq_confs[idx + 1]) / 2
            for idx in range(len(sort_uniq_confs) - 1)
        ]
    )
    return thresholds, sort_uniq_confs 

def __init__(self,
                 observation_space,
                 substation_layout=None,
                 radius_sub=25.,
                 load_prod_dist=70.,
                 bus_radius=4.):
        """

        Parameters
        ----------
        substation_layout: ``list``
            List of tupe given the position of each of the substation of the powergrid.

        observation_space: :class:`grid2op.Observation.ObservationSpace`
            BaseObservation space

        """
        BasePlot.__init__(self,
                          substation_layout=substation_layout,
                          observation_space=observation_space,
                          radius_sub=radius_sub,
                          load_prod_dist=load_prod_dist,
                          bus_radius=bus_radius)
        if not can_plot:
            raise PlotError("Impossible to plot as plotly cannot be imported. Please install \"plotly\" and "
                            "\"seaborn\" with \"pip install --update plotly seaborn\"")

        # define a color palette, whatever...
        sns.set()
        pal = sns.light_palette("darkred", 8)
        self.cols = pal.as_hex()[1:]

        self.col_line = "royalblue"
        self.col_sub = "red"
        self.col_load = "black"
        self.col_gen = "darkgreen"
        self.default_color = "black"
        self.type_fig_allowed = go.Figure 

def remove_correlated_feats(df):
    tmp = df.T
    # Remove columns with no variation
    nunique = tmp.apply(pd.Series.nunique)
    cols_to_drop = nunique[nunique == 1].index
    tmp.drop(cols_to_drop, axis=1, inplace=True)

    perc_spearman = scipy.stats.spearmanr(tmp)
    abs_corr = np.subtract(np.ones(shape=perc_spearman.correlation.shape),
                           np.absolute(perc_spearman.correlation))
    np.fill_diagonal(abs_corr, 0)
    abs_corr_clean = np.maximum(abs_corr,
                                abs_corr.transpose())  # some floating point mismatches, just make symmetric
    clustering = linkage(squareform(abs_corr_clean), method='average')
    clusters = fcluster(clustering, .1, criterion='distance')
    names = tmp.columns.tolist()
    names_to_cluster = list(zip(names, clusters))
    indices_to_keep = []
    ### Extract models closest to cluster centroids
    for x in range(1, len(set(clusters)) + 1):
        # Create mask from the list of assignments for extracting submatrix of the cluster
        mask = np.array([1 if i == x else 0 for i in clusters], dtype=bool)

        # Take the index of the column with the smallest sum of distances from the submatrix
        idx = np.argmin(sum(abs_corr_clean[:, mask][mask, :]))

        # Extract names of cluster elements from names_to_cluster
        sublist = [name for (name, cluster) in names_to_cluster if cluster == x]

        # Element closest to centroid
        centroid = sublist[idx]
        indices_to_keep.append(centroid)

    return df.loc[df.index.isin(indices_to_keep)] 

def get_proteome_counts_impute_missing(prots_filtered_feathers, outpath, length_filter_pid=None,
                                       copynum_scale=False, copynum_df=None,
                                       force_rerun=False):
    """Get counts, uses the mean feature vector to fill in missing proteins for a strain"""
    if ssbio.utils.force_rerun(flag=force_rerun, outfile=outpath):
        big_strain_counts_df = pd.DataFrame()
        first = True
        for feather in prots_filtered_feathers:
            loaded = load_feather(protein_feather=feather, length_filter_pid=length_filter_pid,
                                  copynum_scale=copynum_scale,
                                  copynum_df=copynum_df)
            if first:
                big_strain_counts_df = pd.DataFrame(index=_all_counts, columns=loaded.columns)
                first = False

            new_columns = list(set(loaded.columns.tolist()).difference(big_strain_counts_df.columns))
            if new_columns:
                for col in new_columns:
                    big_strain_counts_df[col] = big_strain_counts_df.mean(axis=1)

            not_in_loaded = list(set(big_strain_counts_df.columns).difference(loaded.columns.tolist()))
            if not_in_loaded:
                for col in not_in_loaded:
                    big_strain_counts_df[col] = big_strain_counts_df[col] + loaded.mean(axis=1)

            big_strain_counts_df = big_strain_counts_df.add(loaded, fill_value=0)

        if len(big_strain_counts_df) > 0:
            big_strain_counts_df.astype(float).reset_index().to_feather(outpath)
        return big_strain_counts_df
    else:
        return pd.read_feather(outpath).set_index('index') 

def test_option(self):
        sns.set(style="darkgrid") 

def pathStats(inGraph, refGraph):
    """
    Only returns TP, FP, numPredictions for each networks
    """
    falsePositives = set(inGraph.edges()).difference(refGraph.edges())
    truePositives = set(inGraph.edges()).intersection(refGraph.edges())
    numPredictions = len(inGraph.edges())
    nopath = 0
    yespath = 0
    edgeCounts = {0:0,2:0,3:0,4:0,5:0}    
    for fe in falsePositives:
        u,v = fe
        try:
            path = nx.dijkstra_path(refGraph,u,v)
            pathlength = len(path) -1
            yespath +=1
            if pathlength in edgeCounts.keys():
                edgeCounts[pathlength] +=1
            
        except nx.exception.NetworkXNoPath:
            nopath +=1

    edgeCounts['numPred'] = numPredictions
    edgeCounts['numTP'] = len(truePositives)
    edgeCounts['numFP_withPath'] = yespath
    edgeCounts['numFP_noPath'] = nopath
    return edgeCounts 

def getNetProp(inGraph):
    '''
    Function to compute properties
    of a given network.
    '''

    # number of weakly connected components in 
    # reference network
    numCC = len(list(nx.weakly_connected_components(inGraph)))
    
    # number of feedback loop 
    # in reference network
    allCyc = nx.simple_cycles(inGraph)
    cycSet = set()
    for cyc in allCyc:
        if len(cyc) == 3:
            cycSet.add(frozenset(cyc))
    
    numFB = len(cycSet)
    
    # number of feedfwd loops
    # in reference network
    allPaths = []
    allPathsSet = set()   
    for u,v in inGraph.edges():
        allPaths = nx.all_simple_paths(inGraph, u, v, cutoff=2)
        for p in allPaths:
            if len(p) >  2:
                allPathsSet.add(frozenset(p))
                
    numFF= len(allPathsSet)
    
    
    # number of mutual interactions
    numMI = 0.0
    for u,v in inGraph.edges():
        if (v,u) in inGraph.edges():
            numMI += 0.5

    return numCC, numFB, numFF, numMI 

def _set_nb_options(namespace):
    namespace["WIDGET_DEFAULTS"] = {
        "layout": widgets.Layout(width="95%"),
        "style": {"description_width": "initial"},
    }

    # Some of our dependencies (networkx) still use deprecated Matplotlib
    # APIs - we can't do anything about it, so suppress them from view
    warnings.simplefilter("ignore", category=MatplotlibDeprecationWarning)
    warnings.filterwarnings("ignore", category=DeprecationWarning)
    sns.set()
    pd.set_option("display.max_rows", 100)
    pd.set_option("display.max_columns", 50)
    pd.set_option("display.max_colwidth", 100)
    os.environ["KQLMAGIC_LOAD_MODE"] = "silent" 

def generate_graph(times, factors, filename="spark_speed.png"):
    seaborn.set()
    fig, ax = plt.subplots()
    for key in times:
        current = [times[key][f] for f in factors]
        ax.plot(factors, current, marker='o', markersize=6)
        ax.text(factors[-1] + 5, current[-1], key, fontsize=10)

    ax.set_ylabel("Seconds per Iteration")
    ax.set_xlabel("Factors")
    plt.savefig(filename, bbox_inches='tight', dpi=300) 

def benchmark_spark(ratings, factors, iterations=5):
    conf = (SparkConf()
            .setAppName("implicit_benchmark")
            .setMaster('local[*]')
            .set('spark.driver.memory', '16G')
            )
    context = SparkContext(conf=conf)
    spark = SparkSession(context)

    times = {}
    try:
        ratings = convert_sparse_to_dataframe(spark, context, ratings)

        for rank in factors:
            als = ALS(rank=rank, maxIter=iterations,
                      alpha=1, implicitPrefs=True,
                      userCol="row", itemCol="col", ratingCol="data")
            start = time.time()
            als.fit(ratings)
            elapsed = time.time() - start
            times[rank] = elapsed / iterations
            print("spark. factors=%i took %.3f" % (rank, elapsed/iterations))
    finally:
        spark.stop()

    return times