# -*- coding: utf-8 -*-
"""
This file contains are a series of helper functions that interact with
LSDTopoTools spatial data.
Written by Simon M. Mudd and Fiona J Clubb at the University of Edinburgh
Released under GPL3
22/06/2017
"""
import os
import pandas as pd
import fiona
from shapely.geometry import shape, Polygon, Point, LineString
#==============================================================================
def MapFigureSizer(figure_width_inches,aspect_ratio, cbar_loc = "None", title = "None",
cbar_width = 0.2,
cbar_text_width = 0.4,
cbar_padding = 0.1,
cbar_fraction = 1,
whitespace_padding = 0.2,
map_text_width = 0.65,
map_text_height = 0.45,
title_height=0.2):
"""This function takes a string size argument and calculates the size of the
various components of a plot based on a map image making up the centre of the
figure.
We use inches because bloody yanks wrote matplotlib and figures in matplotlib use inches.
Luckily we do not have to calculate bushels or furlongs of pixels somewhere,
and the inches stupidity is somewhat limited.
Args:
figure_width_inches (flt): The figure width in inches
aspect_ratio (flt): The width to height ratio of the data
cbar_loc (string): the location of the colourbar, either "left", "right", "top",
"bottom", or "none"
title (bool): if true then adjust the height of the figure to make space for a title
cbar_width (flt): the width of the colorbar
text_padding (list): the padding around the map from the whitespace and the axis/tick labels
Author: SMM
"""
# The text padding list holds the height and width, in inches of
# [0] = left/right tick marks+tick labels
# [1] = left/right text (i.e., and exis label)
# [2] = top/bottom tick marks+tick labels
# [3] = top/bottom text (i.e., and exis label)
# [4] = whitespace at the edge of the figure + between map and colourbar
# [5] = colorbar text (e.g. tick labels+axis label)
# This gets returned, we use it to make the figure.
#======================================================
# Now we need to figure out where the axis are. Sadly this requires
# a load of tedious conditional statments about the location of the axes
#Will's changes:
#1/ Make space for the colourbar label on the left
#2/ Double the cbar_padding to leave space for the colourbar values on the right.
# NB: this should later be a function of font size
#3/ Changed rotation of colourbar text to 90 and the labelpad to -75 in PlottingRaster.py
if cbar_loc == "left":
cbar_left_inches = whitespace_padding + cbar_text_width
#cbar_left_inches = whitespace_padding
map_left_inches = cbar_left_inches+cbar_width+map_text_width + 2*cbar_padding
#map_left_inches = cbar_left_inches+cbar_width+cbar_text_width+map_text_width+cbar_padding
map_width_inches = figure_width_inches-map_left_inches-whitespace_padding
map_height_inches = map_width_inches/aspect_ratio
map_bottom_inches = whitespace_padding+map_text_height
cbar_bottom_inches = map_bottom_inches
figure_height_inches = map_bottom_inches+map_height_inches+whitespace_padding
if title != "None":
# add some space for a title if needed. At the moment this is hard coded but
# we might want to change this for the font size.
figure_height_inches = figure_height_inches+title_height
fig_size_inches = [figure_width_inches,figure_height_inches]
print("cbar_left: "+str(cbar_left_inches)+" map left: "+str(map_left_inches))
print("cbar_bottom: "+str(cbar_bottom_inches)+ " map bottom: "+str(map_bottom_inches))
map_axes = [map_left_inches/figure_width_inches,
map_bottom_inches/figure_height_inches,
map_width_inches/figure_width_inches,
map_height_inches/figure_height_inches]
cbar_axes = [cbar_left_inches/figure_width_inches,
map_bottom_inches/figure_height_inches,
cbar_width/figure_width_inches,
cbar_fraction*(map_height_inches/figure_height_inches)]
elif cbar_loc == "right":
map_left_inches = whitespace_padding+map_text_width
cbar_left_inches= figure_width_inches-whitespace_padding-cbar_width-cbar_text_width
map_right_inches = cbar_left_inches-cbar_padding
map_width_inches = map_right_inches-map_left_inches
map_height_inches = map_width_inches/aspect_ratio
map_bottom_inches = whitespace_padding+map_text_height
cbar_bottom_inches = map_bottom_inches
figure_height_inches = map_bottom_inches+map_height_inches+whitespace_padding
if title != "None":
# add some space for a title if needed. At the moment this is hard coded but
# we might want to change this for the font size.
figure_height_inches = figure_height_inches+title_height
fig_size_inches = [figure_width_inches,figure_height_inches]
print("cbar_left: "+str(cbar_left_inches)+" map left: "+str(map_left_inches))
print("cbar_bottom: "+str(cbar_bottom_inches)+ " map bottom: "+str(map_bottom_inches))
map_axes = [map_left_inches/figure_width_inches,
map_bottom_inches/figure_height_inches,
map_width_inches/figure_width_inches,
map_height_inches/figure_height_inches]
cbar_axes = [cbar_left_inches/figure_width_inches,
map_bottom_inches/figure_height_inches,
cbar_width/figure_width_inches,
cbar_fraction*(map_height_inches/figure_height_inches)]
elif cbar_loc == "top":
print("I am placing the colourbar on the top")
map_left_inches = whitespace_padding+map_text_width
map_right_inches = figure_width_inches-whitespace_padding
map_width_inches = map_right_inches-map_left_inches
map_height_inches = map_width_inches/aspect_ratio
cbar_left_inches= map_left_inches
map_bottom_inches = whitespace_padding+map_text_height
cbar_bottom_inches = map_bottom_inches+map_height_inches+cbar_padding+cbar_text_width
figure_height_inches = cbar_bottom_inches+cbar_width+whitespace_padding
if title != "None":
# add some space for a title if needed. At the moment this is hard coded but
# we might want to change this for the font size.
title_height = 0.5
figure_height_inches = figure_height_inches+title_height
fig_size_inches = [figure_width_inches,figure_height_inches]
print("cbar_left: "+str(cbar_left_inches)+" map left: "+str(map_left_inches))
print("cbar_bottom: "+str(cbar_bottom_inches)+ " map bottom: "+str(map_bottom_inches))
map_axes = [map_left_inches/figure_width_inches,
map_bottom_inches/figure_height_inches,
map_width_inches/figure_width_inches,
map_height_inches/figure_height_inches]
cbar_axes = [cbar_left_inches/figure_width_inches,
cbar_bottom_inches/figure_height_inches,
cbar_fraction*(map_width_inches/figure_width_inches),
cbar_width/figure_height_inches]
elif cbar_loc == "bottom":
print("I am placing the colourbar on the bottom")
map_left_inches = whitespace_padding+map_text_width
map_right_inches = figure_width_inches-whitespace_padding
map_width_inches = map_right_inches-map_left_inches
map_height_inches = map_width_inches/aspect_ratio
cbar_left_inches= map_left_inches
cbar_bottom_inches = whitespace_padding+cbar_text_width
map_bottom_inches = cbar_bottom_inches+cbar_width+cbar_padding+map_text_height
whitespace_padding+map_text_height
figure_height_inches = map_bottom_inches+map_height_inches+whitespace_padding
if title != "None":
# add some space for a title if needed. At the moment this is hard coded but
# we might want to change this for the font size.
figure_height_inches = figure_height_inches+title_height
fig_size_inches = [figure_width_inches,figure_height_inches]
print("cbar_left: "+str(cbar_left_inches)+" map left: "+str(map_left_inches))
print("cbar_bottom: "+str(cbar_bottom_inches)+ " map bottom: "+str(map_bottom_inches))
map_axes = [map_left_inches/figure_width_inches,
map_bottom_inches/figure_height_inches,
map_width_inches/figure_width_inches,
map_height_inches/figure_height_inches]
cbar_axes = [cbar_left_inches/figure_width_inches,
cbar_bottom_inches/figure_height_inches,
cbar_fraction*(map_width_inches/figure_width_inches),
cbar_width/figure_height_inches]
else:
print("No colourbar")
map_left_inches = whitespace_padding+map_text_width
map_right_inches = figure_width_inches-whitespace_padding
map_width_inches = map_right_inches-map_left_inches
map_height_inches = map_width_inches/aspect_ratio
map_bottom_inches = whitespace_padding+map_text_height
figure_height_inches = map_bottom_inches+map_height_inches+whitespace_padding
if title != "None":
# add some space for a title if needed. At the moment this is hard coded but
# we might want to change this for the font size.
figure_height_inches = figure_height_inches+title_height
fig_size_inches = [figure_width_inches,figure_height_inches]
map_axes = [map_left_inches/figure_width_inches,
map_bottom_inches/figure_height_inches,
map_width_inches/figure_width_inches,
map_height_inches/figure_height_inches]
cbar_axes = None
print("The figure size is: ")
print(fig_size_inches)
print("Map axes are:")
print(map_axes)
print("cbar_axes are:")
print(cbar_axes)
return fig_size_inches, map_axes, cbar_axes
#==============================================================================
# Formats ticks for an imshow plot in UTM
# Filename is the name of the file with full path
# x_max, x_min, y_max, y_min are the extent of the plotting area (NOT the DEM)
# n_target ticks are the number of ticks for plotting
#------------------------------------------------------------------------------
def GetTicksForUTMNoInversion(FileName,x_max,x_min,y_max,y_min,n_target_tics):
"""This fuction is used to set tick locations for UTM maps. It tries to optimise the spacing of these ticks.
Args:
x_min (float): The minimum value on the x axis (in metres).
x_max (float): The maximum value on the x axis (in metres).
y_min (float): The minimum value on the y axis (in metres).
y_max (float): The maximum value on the y axis (in metres).
n_target_ticks (int): The number of ticks you want on the axis (this is optimised so you may not get exactly this number)
Returns:
new_xlocs (float list): List of locations of the ticks in metres.
new_x_labels (str list): List of strings for ticks, will be location in kilometres.
new_ylocs (float list): List of locations of the ticks in metres.
new_y_labels (str list): List of strings for ticks, will be location in kilometres.
Author: SMM
"""
CellSize,XMin,XMax,YMin,YMax = LSDMap_IO.GetUTMMaxMin(FileName)
NDV, xsize, ysize, GeoT, Projection, DataType = LSDMap_IO.GetGeoInfo(FileName)
#print("Getting ticks. YMin: "+str(YMin)+" and YMax: "+str(YMax))
xmax_UTM = XMax
xmin_UTM = XMin
ymax_UTM = YMax
ymin_UTM = YMin
dy_fig = ymax_UTM-ymin_UTM
dx_fig = xmax_UTM-xmin_UTM
dx_spacing = dx_fig/n_target_tics
dy_spacing = dy_fig/n_target_tics
if (dx_spacing>dy_spacing):
dy_spacing = dx_spacing
str_dy = str(dy_spacing)
str_dy = str_dy.split('.')[0]
n_digits = str_dy.__len__()
nd = int(n_digits)
first_digit = float(str_dy[0])
dy_spacing_rounded = first_digit*pow(10,(nd-1))
str_xmin = str(xmin_UTM)
str_ymin = str(ymin_UTM)
str_xmin = str_xmin.split('.')[0]
str_ymin = str_ymin.split('.')[0]
xmin_UTM = float(str_xmin)
ymin_UTM = float(str_ymin)
print("minimum values are x: "+str(xmin_UTM)+ " and y: "+str(ymin_UTM))
n_digx = str_xmin.__len__()
n_digy = str_ymin.__len__()
if (n_digx-nd+1) >= 1:
front_x = str_xmin[:(n_digx-nd+1)]
else:
front_x = str_xmin
if (n_digy-nd+1) >= 1:
front_y = str_ymin[:(n_digy-nd+1)]
else:
front_y = str_ymin
round_xmin = float(front_x)*pow(10,nd-1)
round_ymin = float(front_y)*pow(10,nd-1)
#print("UTM y in: "+str(ymin_UTM)+" and rounded min: "+str(round_ymin))
# now we need to figure out where the xllocs and ylocs are
xUTMlocs = np.zeros(2*n_target_tics)
yUTMlocs = np.zeros(2*n_target_tics)
xlocs = np.zeros(2*n_target_tics)
ylocs = np.zeros(2*n_target_tics)
new_x_labels = []
new_y_labels = []
for i in range(0,2*n_target_tics):
# Note we use dy spacing here in both x and y directions since we want
# Ticks spaced the same in each direction
xUTMlocs[i] = round_xmin+(i)*dy_spacing_rounded
yUTMlocs[i] = round_ymin+(i)*dy_spacing_rounded
xlocs[i] = xUTMlocs[i]
ylocs[i] = yUTMlocs[i]
new_x_labels.append( str(xUTMlocs[i]).split(".")[0] )
new_y_labels.append( str(yUTMlocs[i]).split(".")[0] )
new_xlocs = []
new_xUTMlocs = []
x_labels = []
# Now loop through these to get rid of those not in range
for index,xloc in enumerate(xlocs):
if (xloc < XMax and xloc > XMin):
new_xlocs.append(xloc)
new_xUTMlocs.append(xUTMlocs[index])
x_labels.append(new_x_labels[index])
new_ylocs = []
new_yUTMlocs = []
y_labels = []
for index,yloc in enumerate(ylocs):
if (yloc < YMax and yloc > YMin):
new_ylocs.append(yloc)
new_yUTMlocs.append(yUTMlocs[index])
y_labels.append(new_y_labels[index])
#return xlocs,ylocs,new_x_labels,new_y_labels
return new_xlocs,new_ylocs,x_labels,y_labels
#==============================================================================
#=============================================================================
# CSV READERS
# Read in the csv files to pandas dataframes
#=============================================================================
def ReadBaselevelKeysCSV(DataDirectory, fname_prefix):
"""
This function reads in the file with the suffix '__BaselevelKeys.csv'
to a pandas dataframe
Args:
DataDirectory: the data directory
fname_prefix: the file name prefix
Returns:
pandas dataframe with the csv file
Author: FJC
"""
# get the csv filename
baselevel_suffix = '_BaselevelKeys.csv'
fname = fname_prefix+baselevel_suffix
# read in the dataframe using pandas
df = pd.read_csv(DataDirectory+fname)
return df
def ReadSourceKeysCSV(DataDirectory, fname_prefix):
"""
This function reads in the file with the suffix '_SourceKeys.csv'
to a pandas dataframe
Args:
DataDirectory: the data directory
fname_prefix: the file name prefix
Returns:
pandas dataframe with the csv file
Author: FJC
"""
# get the csv filename
source_keys_suffix = '_SourceKeys.csv'
fname = fname_prefix+source_keys_suffix
# read in the dataframe using pandas
df = pd.read_csv(DataDirectory+fname)
return df
def ReadBasinInfoCSV(DataDirectory, fname_prefix):
"""
This function reads in the file with the suffix '_AllBasinsInfo.csv'
to a pandas dataframe
Args:
DataDirectory: the data directory
fname_prefix: the file name prefix
Returns:
pandas dataframe with the csv file
Author: FJC
"""
# get the csv filename
basin_suffix = '_AllBasinsInfo.csv'
fname = fname_prefix+basin_suffix
# read in the dataframe using pandas
df = pd.read_csv(DataDirectory+fname)
return df
def ReadFullStatsCSV(DataDirectory, fname_prefix, m_over_n):
"""
This function reads in the file with the suffix '_fullstats.csv'
to a pandas dataframe. Must specify the m/n value as an argument
Args:
DataDirectory: the data directory
fname_prefix: the file name prefix
m_over_n: the m/n value
Returns:
pandas dataframe with the csv file
Author: FJC
"""
# get the csv filename
fullstats_suffix = '_movernstats_%s_fullstats.csv' % m_over_n
fname = fname_prefix+fullstats_suffix
# read in the dataframe using pandas
df = pd.read_csv(DataDirectory+fname)
return df
def ReadChiProfileCSV(DataDirectory, fname_prefix):
"""
This function reads in the file with the suffix '_movern.csv', which
contains the data for the full chi profiles, to a pandas dataframe.
Args:
DataDirectory: the data directory
fname_prefix: the file name prefix
Returns:
pandas dataframe with the csv file
Author: FJC
"""
import os
# get the csv filename
profile_suffix = '_burned_movern.csv'
fname = fname_prefix+profile_suffix
# check if this exists. if not then use the burned one.
if not os.path.isfile(DataDirectory+fname):
print ("Reading the csv...")
fname = fname_prefix+'_movern.csv'
else:
print("Reading the burned csv...")
# read in the dataframe using pandas
df = pd.read_csv(DataDirectory+fname)
return df
def ReadBasinStatsCSV(DataDirectory, fname_prefix):
"""
This function reads in the file with the suffix '_disorder_basinstats.csv'
to a pandas dataframe
Args:
DataDirectory: the data directory
fname_prefix: the file name prefix
Returns:
pandas dataframe with the csv file
Author: FJC
"""
# get the csv filename
basin_stats_suffix = '_disorder_basinstats.csv'
fname = fname_prefix+basin_stats_suffix
# read in the dataframe using pandas
df = pd.read_csv(DataDirectory+fname)
return df
def AppendBasinStatsCSVs(DataDirectory, FilenamePrefix):
"""
This function reads in the files with the prefic "basin"
and the suffix '_movernstats_basinstats.csv'
to a pandas dataframe, for use with parallel analysis
Args:
DataDirectory: the data directory
Returns:
pandas dataframe with the csv file
Author: FJC, MDH
"""
# get the csv filename
basin_stats_suffix = '_disorder_basinstats.csv'
# decclare empty data frame
MasterDF = pd.DataFrame()
# get the list of basins as a dict
basin_dict = MapBasinsToKeysFromJunctionList(DataDirectory,FilenamePrefix)
# loop through and get each basin csv
for outlet_jn, basin_key in basin_dict.iteritems():
print ((outlet_jn, basin_key))
this_fname = "basin"+str(outlet_jn)+basin_stats_suffix
df = pd.read_csv(DataDirectory+this_fname)
df = df[df['basin_key'] == 0]
df['basin_key'] = basin_key
MasterDF = MasterDF.append(df, ignore_index = True)
return MasterDF
def ReadBasinStatsPointCSV(DataDirectory, fname_prefix):
"""
This function reads in the file with the suffix '_point_movernstats_basinstats.csv'
to a pandas dataframe
Args:
DataDirectory: the data directory
fname_prefix: the file name prefix
Returns:
pandas dataframe with the csv file
Author: FJC
"""
# get the csv filename
csv_suffix = '_point_movernstats_basinstats.csv'
fname = fname_prefix+csv_suffix
# read in the dataframe using pandas
df = pd.read_csv(DataDirectory+fname)
return df
def ReadChainCSV(DataDirectory, fname_prefix, basin_key):
"""
This function reads in the file with the suffix '_BasinX_chain.csv'
to a pandas dataframe, where X is the basin key
Args:
DataDirectory: the data directory
fname_prefix: the file name prefix
basin_key: the basin key
Returns:
pandas dataframe with the csv file
Author: FJC
"""
# get the csv filename
chain_suffix = '_Basin%s_chain.csv' %str(basin_key)
fname = fname_prefix+chain_suffix
# read in the dataframe using pandas
df = pd.read_csv(DataDirectory+fname)
return df
def ReadMCPointsCSV(DataDirectory, fname_prefix):
"""
This function reads in the file with the suffix
'_MCpoint__points_MC_basinstats.csv'
Args:
DataDirectory: the data directory
fname_prefix: the file name prefix
Returns:
pandas dataframe with the csv file
Author: FJC
"""
# get the csv filename
mc_points_suffix = '_MCpoint_points_MC_basinstats.csv'
fname = fname_prefix+mc_points_suffix
# read in the dataframe using pandas
df = pd.read_csv(DataDirectory+fname)
return df
def ReadMChiSegCSV(DataDirectory, fname_prefix, type = "Normal"):
"""
This function reads in the file with the suffix
'_MChiSegmented.csv'
This file holds the MCHI segmented data
Args:
DataDirectory: the data directory
fname_prefix: the file name prefix
Returns:
pandas dataframe with the csv file
Author: BG
"""
# get the csv filename depending of what you need
# The "knickpoint" type is a special M_Chi file generated with the exact degmented elevation
if(type == "Normal"):
suffix = '_MChiSegmented.csv'
elif(type == "knickpoint"):
suffix = "_ksnkp_mchi.csv"
fname = fname_prefix+suffix
# read in the dataframe using pandas
df = pd.read_csv(DataDirectory+fname)
# Getting rid of NoData
df = df[df["chi"] >= 0]
return df
def ReadDisorderCSV(DataDirectory, fname_prefix):
"""
Function to read in the CSV from the chi disorder
analysis
Args:
DataDirectory: the data directory
fname_prefix: the file name prefix
Returns:
pandas dataframe with the csv file
Author: FJC
"""
# get the csv filename
mc_points_suffix = '_disorder_movernstats_disorder_basinstats.csv'
fname = fname_prefix+mc_points_suffix
# read in the dataframe using pandas
df = pd.read_csv(DataDirectory+fname)
return df
def ReadDisorderUncertCSV(DataDirectory, fname_prefix):
"""
Function to read in the CSV from the chi disorder
analysis
Args:
DataDirectory: the data directory
fname_prefix: the file name prefix
Returns:
pandas dataframe with the csv file
Author: FJC
"""
# get the csv filename
mc_points_suffix = '_fullstats_disorder_uncert.csv'
fname = fname_prefix+mc_points_suffix
# read in the dataframe using pandas
df = pd.read_csv(DataDirectory+fname)
return df
def AppendDisorderCSV(DataDirectory, FilenamePrefix):
"""
This function reads in a series of disorder csvs and appends them together
into one function for plotting
Args:
DataDirectory (str): the data DataDirectory
Returns:
pandas dataframe with the appended fullstats csvs
Author: FJC
"""
# get the csv filename
csv_suffix = '_fullstats_disorder_uncert.csv'
MasterDF = pd.DataFrame()
basin_dict = MapBasinsToKeysFromJunctionList(DataDirectory, FilenamePrefix)
# loop through and get each basin csv
for outlet_jn, basin_key in basin_dict.iteritems():
this_fname = "basin"+str(outlet_jn)+csv_suffix
# append to master DF and change the basin key and the junction
df = pd.read_csv(DataDirectory+this_fname)
df = df[df['basin_key'] == 0]
df['basin_key'] = basin_key
MasterDF = MasterDF.append(df, ignore_index = True)
return MasterDF
def readSKKPstats(DataDirectory, fname_prefix):
"""
This function reads in the file with the suffix
'_KsnKn.csv'
This file holds the MCHI segmented data
Args:
DataDirectory: the data directory
fname_prefix: the file name prefix
Returns:
pandas dataframe with the csv file
Author: BG
"""
# get the csv filename
suffix = '_ksnkp_SK.csv'
fname = fname_prefix+suffix
# read in the dataframe using pandas
df = pd.read_csv(DataDirectory+fname)
return df
def ReadKnickpointCSV(DataDirectory, fname_prefix, ftype = "normal"):
"""
This function reads in the file with the suffix
'_KsnKn.csv'
This file holds the MCHI segmented data
Args:
DataDirectory: the data directory
fname_prefix: the file name prefix
Returns:
pandas dataframe with the csv file
Author: BG
"""
# get the csv filename
if(ftype == "raw"):
suffix = '_ksnkp_raw.csv'
else:
suffix = '_ksnkp.csv'
fname = fname_prefix+suffix
# read in the dataframe using pandas
df = pd.read_csv(DataDirectory+fname)
return df
def ReadKnickzoneCSV(DataDirectory, fname_prefix):
"""
This function reads in the file with the suffix
'_KsnKn.csv'
This file holds the MCHI segmented data
Args:
DataDirectory: the data directory
fname_prefix: the file name prefix
Returns:
pandas dataframe with the csv file
Author: BG
"""
# get the csv filename
suffix = '_KsnKz.csv'
fname = fname_prefix+suffix
# read in the dataframe using pandas
df = pd.read_csv(DataDirectory+fname)
return df
def ReadChiResidualsCSVs(DataDirectory, fname_prefix):
"""
This function reads in the 3 CSV files for the residuals analysis
They have the format:
"_residual_movernstats_movern_residuals_median.csv"
"_residual_movernstats_movern_residuals_Q1.csv"
"_residual_movernstats_movern_residuals_Q3.csv"
Args:
DataDirectory: the data directory
fname_prefix: the file name prefix
Returns:
list of pandas dataframes with the csv files. List format is:
0 = medians
1 = 1st quartile
2 = 3rd quartile
Author: FJC
"""
# get the csv filename
fnames = ["_residual_movernstats_movern_residuals_median.csv","_residual_movernstats_movern_residuals_Q1.csv","_residual_movernstats_movern_residuals_Q3.csv"]
#print "The fnames are: ", fnames
dfs = []
for f in fnames:
fname = fname_prefix+f
dfs.append(pd.read_csv(DataDirectory+fname))
return dfs
def ReadRawSAData(DataDirectory, fname_prefix):
"""
This function reads in the raw SA data to a pandas dataframe
Args:
DataDirectory: the data directory
fname_prefix: the file name prefix
Returns:
pandas dataframe with the raw SA data
Author: FJC
"""
# get the csv filename
fname_suffix = "_SAvertical.csv"
fname = fname_prefix+fname_suffix
df = pd.read_csv(DataDirectory+fname)
return df
def AppendRawSAData(DataDirectory, FilenamePrefix):
"""
This function reads in the raw SA data to a pandas dataframe
from multiple CSV files with the filename prefix "basin"
For use with parallelised methods
Args:
DataDirectory: the data directory
Returns:
pandas dataframe with the raw SA data
Author: MDH, FJC
"""
# get the csv filename
csv_suffix = "_SAvertical.csv"
MasterDF = pd.DataFrame()
basin_dict = MapBasinsToKeysFromJunctionList(DataDirectory, FilenamePrefix)
# loop through and get each basin csv
for outlet_jn, basin_key in basin_dict.iteritems():
this_fname = "basin"+str(outlet_jn)+csv_suffix
df = pd.read_csv(DataDirectory+this_fname)
df = df[df['basin_key'] == 0]
df['basin_key'] = basin_key
MasterDF = MasterDF.append(df, ignore_index = True)
return MasterDF
def ReadSegmentedSAData(DataDirectory, fname_prefix):
"""
This function reads in the segmented SA data to a pandas dataframe
Args:
DataDirectory: the data directory
fname_prefix: the file name prefix
Returns:
pandas dataframe with the segmented SA data
Author: FJC
"""
# get the csv filename
fname_suffix = "_SAsegmented.csv"
fname = fname_prefix+fname_suffix
df = pd.read_csv(DataDirectory+fname)
return df
def ReadBinnedSAData(DataDirectory, fname_prefix):
"""
This function reads in the binned SA data to a pandas dataframe
csv with the suffix "_SAbinned.csv"
Args:
DataDirectory: the data directory
fname_prefix: the file name prefix
Returns:
pandas dataframe with the segmented SA data
Author: FJC
"""
# get the csv filename
fname_suffix = "_SAbinned.csv"
fname = fname_prefix+fname_suffix
df = pd.read_csv(DataDirectory+fname)
return df
def ReadMOverNSummaryCSV(DataDirectory, fname_prefix):
"""
This function reads in the summary csv with the best fit movern info
to a pandas dataframe
Args:
DataDirectory: the data directory
fname_prefix: the file name prefix
Returns:
pandas dataframe with the segmented SA data
Author: FJC
"""
# get the csv filename
fname_suffix = "_movern_summary.csv"
fname = fname_prefix+fname_suffix
df = pd.read_csv(DataDirectory+fname)
return df
def ReadChannelNetworkCSV(DataDirectory, fname_prefix):
"""
This function reads in the channel network csv to a df
Args:
DataDirectory: the data directory
fname_prefix: the file name prefix
Returns:
pandas dataframe with the channel network
Author: FJC
"""
# get the csv filename
fname_suffix = "_CN.csv"
fname = fname_prefix+fname_suffix
df = pd.read_csv(DataDirectory+fname)
return df
def ReadChiDataMapCSV(DataDirectory, fname_prefix):
"""
This function reads in the chi data map csv to a df
Args:
DataDirectory: the data directory
fname_prefix: the file name prefix
Returns:
pandas dataframe with the chi map
Author: FJC
"""
# get the csv filename
fname_suffix = "_chi_data_map.csv"
fname = fname_prefix+fname_suffix
df = pd.read_csv(DataDirectory+fname)
return df
#--------------------------------------------------------------------------------#
# Terraces
#--------------------------------------------------------------------------------#
def read_terrace_csv(DataDirectory,fname_prefix):
"""
This function reads in the csv file with the extension "_terrace_info.csv"
and returns it as a pandas dataframe
Args:
DataDirectory (str): the data directory
fname_prefix (str): the name of the DEM
Returns:
pandas dataframe with the terrace info
Author: FJC
"""
csv_suffix = '_terrace_info.csv'
fname = DataDirectory+fname_prefix+csv_suffix
df = pd.read_csv(fname)
return df
def read_channel_csv(DataDirectory,fname_prefix):
"""
This function reads in the csv file with the extension "_baseline_channel_info.csv"
and returns it as a pandas dataframe
Args:
DataDirectory (str): the data directory
fname_prefix (str): the name of the DEM
Returns:
pandas dataframe with the channel info
Author: FJC
"""
csv_suffix = '_baseline_channel_info.csv'
fname = DataDirectory+fname_prefix+csv_suffix
df = pd.read_csv(fname)
return df
def read_index_channel_csv(DataDirectory,fname_prefix):
"""
This function reads in the csv file with the extension "_index_chan.csv"
and returns it as a pandas dataframe
Args:
DataDirectory (str): the data directory
fname_prefix (str): the name of the DEM
Returns:
pandas dataframe with the channel info
Author: FJC
"""
csv_suffix = '_index_chan.csv'
fname = DataDirectory+fname_prefix+csv_suffix
df = pd.read_csv(fname)
return df
def read_terrace_shapefile(DataDirectory, shapefile_name):
"""
This function reads in a shapefile of digitised terraces
using shapely and fiona
Args:
DataDirectory (str): the data directory
shapefile_name (str): the name of the shapefile
Returns: shapely polygons with terraces
Author: FJC
"""
Polygons = {}
with fiona.open(DataDirectory+shapefile_name, 'r') as input:
for f in input:
this_shape = Polygon(shape(f['geometry']))
this_id = f['properties']['id']
Polygons[this_id] = this_shape
return Polygons
def read_terrace_centrelines(DataDirectory, shapefile_name):
"""
This function reads in a shapefile of terrace centrelines
using shapely and fiona
Args:
DataDirectory (str): the data directory
shapefile_name (str): the name of the shapefile
Returns: shapely polygons with terraces
Author: FJC
"""
Lines = {}
with fiona.open(DataDirectory+shapefile_name, 'r') as input:
for f in input:
this_line = LineString(shape(f['geometry']))
this_id = f['properties']['id']
Lines[this_id] = this_line
return Lines
def ReadModelCSV(DataDirectory, Base_file):
"""
This function reads in the csv file from the model run to a pandas dataframe
Args:
DataDirectory (str): the data directory
Base_file (str): the base file prefix
Returns:
pandas dataframe with the csv file info
Author: FJC
"""
# get the csv filename
csv_suffix = '_model_info.csv'
fname = Base_file+csv_suffix
# read in the dataframe using pandas
df = pd.read_csv(DataDirectory+fname)
return df
#-----------------------------------------------------------------------------#
# Drainage capture metrics
#-----------------------------------------------------------------------------#
def ReadPerimeterCSV(DataDirectory, fname_prefix):
"""
This function reads in the csv file with the perimeter info
Args:
DataDirectory (str): the data directory
fname_prefix (str): the base file prefix
Returns:
pandas dataframe with the csv file info
Author: FJC
"""
csv_suffix = '_Perimeters.csv'
df = pd.read_csv(DataDirectory+fname_prefix+csv_suffix)
return df
#-----------------------------------------------------------------------------#
# Functions for appending csvs together for parallel basin running
# FJC 19/10/17
#-----------------------------------------------------------------------------#
def AppendBasinCSVs(DataDirectory, FilenamePrefix):
"""
This function reads in a series of basin csv files and appends them together
into one function for plotting
Args:
DataDirectory (str): the data DataDirectory
Returns:
pandas dataframe with the appended basin csvs
Author: FJC
"""
# get the csv filename
csv_suffix = "_movernstats_basinstats.csv"
MasterDF = pd.DataFrame()
basin_dict = MapBasinsToKeysFromJunctionList(DataDirectory,FilenamePrefix)
# loop through and get each basin csv
for outlet_jn, basin_key in basin_dict.iteritems():
this_fname = "basin"+str(outlet_jn)+csv_suffix
# append to master DF and change the basin key and the junction
df = pd.read_csv(DataDirectory+this_fname)
df.ix[0, 'basin_key'] = basin_key
df.ix[0, 'outlet_jn'] = outlet_jn
MasterDF = MasterDF.append(df.iloc[0], ignore_index = True)
return MasterDF
def AppendFullStatsCSVs(DataDirectory, m_over_n, FilenamePrefix):
"""
This function reads in a series of full stats csvs and appends them together
into one function for plotting
Args:
DataDirectory (str): the data DataDirectory
m_over_n (float): the m/n value
Returns:
pandas dataframe with the appended fullstats csvs
Author: FJC
"""
# get the csv filename
csv_suffix = '_movernstats_%s_fullstats.csv' % m_over_n
MasterDF = pd.DataFrame()
basin_dict = MapBasinsToKeysFromJunctionList(DataDirectory, FilenamePrefix)
# loop through and get each basin csv
for outlet_jn, basin_key in basin_dict.iteritems():
this_fname = "basin"+str(outlet_jn)+csv_suffix
# append to master DF and change the basin key and the junction
df = pd.read_csv(DataDirectory+this_fname)
df = df[df['basin_key'] == 0]
df['basin_key'] = basin_key
MasterDF = MasterDF.append(df, ignore_index = True)
return MasterDF
def ReadMovernCSV(DataDirectory, fname_prefix):
"""
This function reads in a the movern csv with the suffix "_movern"
Args:
DataDirectory (str): the data DataDirectory
fname_prefix
Returns:
pandas dataframe with the appended movern csvs
Author: MDH
"""
# get the csv filename
csv_suffix = '_movern.csv'
df = pd.read_csv(DataDirectory+fname_prefix+csv_suffix)
return df
def AppendMovernCSV(DataDirectory, FilenamePrefix):
"""
This function reads in a series of csvs with the suffix "_movern"
and appends them together into one function for plotting
Args:
DataDirectory (str): the data DataDirectory
Returns:
pandas dataframe with the appended movern csvs
Author: FJC
"""
# get the csv filename
csv_suffix = '_movern.csv'
MasterDF = pd.DataFrame()
basin_dict = MapBasinsToKeysFromJunctionList(DataDirectory,FilenamePrefix)
# loop through and get each basin csv
for outlet_jn, basin_key in basin_dict.iteritems():
this_fname = "basin"+str(outlet_jn)+csv_suffix
# append to master DF and change the basin key and the junction
df = pd.read_csv(DataDirectory+this_fname)
df = df[df['basin_key'] == 0]
df['basin_key'] = basin_key
MasterDF = MasterDF.append(df, ignore_index = True)
return MasterDF
def AppendBasinInfoCSVs(DataDirectory, FilenamePrefix):
"""
This function reads in a series of csvs with the suffix "_AllBasinsInfo"
and appends them together into one function for plotting
Args:
DataDirectory (str): the data DataDirectory
Returns:
pandas dataframe with the appended movern csvs
Author: FJC
"""
# get the csv filename
csv_suffix = '_AllBasinsInfo.csv'
MasterDF = pd.DataFrame()
basin_dict = MapBasinsToKeysFromJunctionList(DataDirectory, FilenamePrefix)
# loop through and get each basin csv
for outlet_jn, basin_key in basin_dict.iteritems():
this_fname = "basin"+str(outlet_jn)+csv_suffix
# append to master DF and change the basin key and the junction
df = pd.read_csv(DataDirectory+this_fname)
df = df[df['basin_key'] == 0]
df['basin_key'] = basin_key
df['outlet_junction'] = outlet_jn
MasterDF = MasterDF.append(df, ignore_index = True)
return MasterDF
def AppendChiDataMapCSVs(DataDirectory, FilenamePrefix):
"""
This function reads in a series of csvs with the suffix "_chi_data_map"
and appends them together into one function for plotting
Args:
DataDirectory (str): the data DataDirectory
Returns:
pandas dataframe with the appended csvs
Author: FJC
"""
# get the csv filename
csv_suffix = '_chi_data_map.csv'
MasterDF = pd.DataFrame()
basin_dict = MapBasinsToKeysFromJunctionList(DataDirectory, FilenamePrefix)
# loop through and get each basin csv
for outlet_jn, basin_key in basin_dict.iteritems():
this_fname = "basin"+str(outlet_jn)+csv_suffix
# append to master DF and change the basin key and the junction
df = pd.read_csv(DataDirectory+this_fname)
df = df[df['basin_key'] == 0]
df['basin_key'] = basin_key
MasterDF = MasterDF.append(df, ignore_index = True)
return MasterDF
def AppendSABinnedCSVs(DataDirectory, fname_prefix):
"""
This function reads in a series of csvs with the suffix "_SAbinned"
and appends them together into one function for plotting
Args:
DataDirectory (str): the data directory
Returns:
pandas dataframe with the appended csvs
Author: FJC
"""
# get the csv filename
csv_suffix = '_SAbinned.csv'
MasterDF = pd.DataFrame()
basin_dict = MapBasinsToKeysFromJunctionList(DataDirectory, fname_prefix)
# loop through and get each basin csv
for outlet_jn, basin_key in basin_dict.iteritems():
this_fname = "basin"+str(outlet_jn)+csv_suffix
# append to master DF and change the basin key and the junction
df = pd.read_csv(DataDirectory+this_fname)
df = df[df['basin_key'] == 0]
df['basin_key'] = basin_key
MasterDF = MasterDF.append(df, ignore_index = True)
# write to a new csv
MasterDF.to_csv(DataDirectory+fname_prefix+csv_suffix)
return MasterDF
def AppendSASegmentedCSVs(DataDirectory, fname_prefix):
"""
This function reads in a series of csvs with the suffix "_SAsegmented"
and appends them together into one function for plotting
Args:
DataDirectory (str): the data directory
Returns:
pandas dataframe with the appended csvs
Author: FJC
"""
# get the csv filename
csv_suffix = '_SAsegmented.csv'
MasterDF = pd.DataFrame()
basin_dict = MapBasinsToKeysFromJunctionList(DataDirectory, fname_prefix)
# loop through and get each basin csv
for outlet_jn, basin_key in basin_dict.iteritems():
this_fname = "basin"+str(outlet_jn)+csv_suffix
# append to master DF and change the basin key and the junction
df = pd.read_csv(DataDirectory+this_fname)
df = df[df['basin_key'] == 0]
df['basin_key'] = basin_key
MasterDF = MasterDF.append(df, ignore_index = True)
# write to a new csv
MasterDF.to_csv(DataDirectory+fname_prefix+csv_suffix)
return MasterDF
def AppendSAVerticalCSVs(DataDirectory, fname_prefix):
"""
This function reads in a series of csvs with the suffix "_SAvertical"
and appends them together into one function for plotting
Args:
DataDirectory (str): the data directory
Returns:
pandas dataframe with the appended csvs
Author: FJC
"""
# get the csv filename
csv_suffix = '_SAvertical.csv'
MasterDF = pd.DataFrame()
basin_dict = MapBasinsToKeysFromJunctionList(DataDirectory, fname_prefix)
# loop through and get each basin csv
for outlet_jn, basin_key in basin_dict.iteritems():
this_fname = "basin"+str(outlet_jn)+csv_suffix
# append to master DF and change the basin key and the junction
df = pd.read_csv(DataDirectory+this_fname)
df = df[df['basin_key'] == 0]
df['basin_key'] = basin_key
MasterDF = MasterDF.append(df, ignore_index = True)
# write to a new csv
MasterDF.to_csv(DataDirectory+fname_prefix+csv_suffix)
return MasterDF
def AppendBasinPointCSVs(DataDirectory, FilenamePrefix):
"""
This function reads in a series of csvs with the suffix
"_MCpoint_points_MC_basinstats" and appends them together
into one function for plotting
Args:
DataDirectory (str): the data directory
Returns:
pandas dataframe with the appended csvs
Author: FJC
"""
# get the csv filename
csv_suffix = '_MCpoint_points_MC_basinstats.csv'
MasterDF = pd.DataFrame()
basin_dict = MapBasinsToKeysFromJunctionList(DataDirectory, FilenamePrefix)
# loop through and get each basin csv
for outlet_jn, basin_key in basin_dict.iteritems():
this_fname = "basin"+str(outlet_jn)+csv_suffix
# append to master DF and change the basin key and the junction
df = pd.read_csv(DataDirectory+this_fname)
df = df[df['basin_key'] == 0]
df['basin_key'] = basin_key
df['outlet_jn'] = outlet_jn
MasterDF = MasterDF.append(df, ignore_index = True)
return MasterDF
def AppendChiResidualsCSVs(DataDirectory, FilenamePrefix):
"""
This function reads in a series of 3 csvs with the residuals data
and appends them together into one function for plotting
Args:
DataDirectory (str): the data directory
Returns:
pandas dataframe with the appended csvs
Author: FJC
"""
# get the csv filename
fnames = ["_residual_movernstats_movern_residuals_median.csv","_residual_movernstats_movern_residuals_Q1.csv","_residual_movernstats_movern_residuals_Q3.csv"]
MasterDFs = []
MasterDF = pd.DataFrame()
basin_dict = MapBasinsToKeysFromJunctionList(DataDirectory, FilenamePrefix)
for f in fnames:
# loop through and get each basin csv
for outlet_jn, basin_key in basin_dict.iteritems():
this_fname = "basin"+str(outlet_jn)+f
# append to master DF and change the basin key and the junction
df = pd.read_csv(DataDirectory+this_fname)
df = df[df['basin_key'] == 0]
df['basin_key'] = basin_key
df['outlet_jn'] = outlet_jn
MasterDF = MasterDF.append(df, ignore_index = True)
MasterDFs.append(MasterDF)
return MasterDFs
def MapBasinsToKeys(DataDirectory):
"""
This function reads in all of the basin files in a directory
and assigns a basin key to each basin based on the outlet junction.
This is used when appending these all together so that each basin still
has a unique ID
Args:
DataDirectory (str): the data directory
Returns:
dictionary where key is the outlet junction and value is the assigned
basin ID
Author: FJC
"""
import os
# get the csv filename
csv_suffix = "_movernstats_basinstats.csv"
basin_dict = {}
key = 0
# loop through the directory and get each basin stats file
for fname in os.listdir(DataDirectory):
if fname.startswith("basin"):
if fname.endswith(csv_suffix):
# get this basin junction and map to a key
fname = fname.split("/")[-1]
fname = fname.split("_")[0]
fname = fname.split("n")[-1] #stupid way of getting just the basin junction number
basin_dict[fname] = key
key+=1
return basin_dict
def MapBasinsToKeysFromJunctionList(DataDirectory,FilenamePrefix):
"""
Function to map the basins to keys in an order specified by then
original junction list.
Args:
DataDirectory (str): the data directory
FilenamePrefix (str): prefix of the DEM, should be the same as the
junctions.list file.
Returns:
dictionary where key is the outlet junction and value is the assigned
basin ID
Author: FJC
"""
import csv
basin_dict = {}
key = 0
# read in the space delimited junctons.list file
reader = csv.reader(open(DataDirectory+FilenamePrefix+'_junctions.list'), delimiter=" ")
for row in reader:
for jn in row:
basin_dict[jn] = key
key+=1
return basin_dict
