Python Code Examples for read csv

60 Python code examples are found related to "read csv". These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example.
Example 1
Project: PyU4V   Author: dell   File: common.py    License: Apache License 2.0 6 votes vote down vote up
def read_csv_values(file_name, delimiter=',', quotechar='|'):
        """Read any csv file with headers.

        DEPRECATION NOTICE: CommonFunctions.read_csv_values() will be
        refactored in PyU4V version 9.3 in favour of
        utils.file_handler.read_csv_values(). For further information please
        consult PyU4V 9.1 release notes.

        You can extract the multiple lists from the headers in the CSV file.
        In your own script, call this function and assign to data variable,
        then extract the lists to the variables.

        :param file_name: path to the file -- str
        :param delimiter: delimiter kwarg for csv DictReader object -- str
        :param quotechar: quotechar kwarg for csv DictReader object -- str
        :returns: file contents -- dict
        """
        return file_handler.read_csv_values(
            file_name, delimiter=delimiter, quotechar=quotechar) 
Example 2
Project: btgym   Author: Kismuz   File: multi.py    License: GNU Lesser General Public License v3.0 6 votes vote down vote up
def read_csv(self, data_filename=None, force_reload=False):
        # Load:
        indexes = []
        for stream in self.data.values():
            stream.read_csv(force_reload=force_reload)
            indexes.append(stream.data.index)

        # Get indexes intersection:
        if len(indexes) > 1:
            idx_intersected = indexes[0]
            for i in range(1, len(indexes)):
                idx_intersected = idx_intersected.intersection(indexes[i])

            # Truncate data to common index:
            for stream in self.data.values():
                stream.data = stream.data.loc[idx_intersected] 
Example 3
Project: gempy   Author: cgre-aachen   File: gempy_api.py    License: GNU Lesser General Public License v3.0 6 votes vote down vote up
def read_csv(geo_model: Project, path_i=None, path_o=None, **kwargs):
    """
    Args:
        geo_model (Project):
        path_i:
        path_o:
        **kwargs:
    """
    if path_i is not None or path_o is not None:
        try:
            geo_model.read_data(path_i, path_o, **kwargs)
        except KeyError:
            raise KeyError('Loading of CSV file failed. Check if you use commas '
                           'to separate your data.')
    return True


# endregion


# region Computing the model 
Example 4
Project: pyRevit   Author: eirannejad   File: script.py    License: GNU General Public License v3.0 6 votes vote down vote up
def read_csv_typed_data(csv_file):
    """Read Revit property data from the given CSV file."""
    # open file
    with codecs.open(csv_file, 'rb', encoding='utf-8') as csvfile:
        # read lines
        csv_lines = list(csv.reader(csvfile, delimiter=',', quotechar='\"'))
        # grab the first line, extract field names
        # if field definition include the type, grab the associated
        # DB.ParameterType as well
        # https://www.apidocs.co/apps/revit/2019/f38d847e-207f-b59a-3bd6-ebea80d5be63.htm
        # https://support.spatialkey.com/providing-data-types-in-csv-headers/
        field_defs = []
        for field_def in csv_lines[0]:
            parts = field_def.split('|')
            parts_count = len(parts)
            if parts_count == 1:
                if parts[0]:
                    field_defs.append((parts[0], DB.ParameterType.Text))
            elif parts_count == 2:
                field_defs.append((parts[0],
                                   coreutils.get_enum_value(DB.ParameterType,
                                                            parts[1])))
    # return field definitions, and data
    return (field_defs, csv_lines[1:]) 
Example 5
Project: tiny_python_projects   Author: kyclark   File: solution2.py    License: MIT License 6 votes vote down vote up
def read_csv(fh):
    """Read the CSV input"""

    exercises = []
    for row in csv.DictReader(fh, delimiter=','):
        name, reps = row.get('exercise'), row.get('reps')
        if name and reps:
            match = re.match(r'(\d+)-(\d+)', reps)
            if match:
                low, high = map(int, match.groups())
                exercises.append((name, low, high))

    return exercises


# -------------------------------------------------- 
Example 6
Project: evo   Author: MichaelGrupp   File: file_interface.py    License: GNU General Public License v3.0 6 votes vote down vote up
def read_euroc_csv_trajectory(file_path):
    """
    parses ground truth trajectory from EuRoC MAV state estimate .csv
    :param file_path: <sequence>/mav0/state_groundtruth_estimate0/data.csv
    :return: trajectory.PoseTrajectory3D object
    """
    raw_mat = csv_read_matrix(file_path, delim=",", comment_str="#")
    error_msg = ("EuRoC MAV state ground truth must have 17 entries per row "
                 "and no trailing delimiter at the end of the rows (comma)")
    if len(raw_mat) > 0 and len(raw_mat[0]) != 17:
        raise FileInterfaceException(error_msg)
    try:
        mat = np.array(raw_mat).astype(float)
    except ValueError:
        raise FileInterfaceException(error_msg)
    stamps = np.divide(mat[:, 0], 1e9)  # n x 1  -  nanoseconds to seconds
    xyz = mat[:, 1:4]  # n x 3
    quat = mat[:, 4:8]  # n x 4
    logger.debug("Loaded {} stamps and poses from: {}".format(
        len(stamps), file_path))
    return PoseTrajectory3D(xyz, quat, stamps) 
Example 7
Project: pyDataverse   Author: AUSSDA   File: utils.py    License: MIT License 6 votes vote down vote up
def read_file_csv(filename):
    """Read in CSV file.

    See more at `csv.reader() <https://docs.python.org/3.5/library/csv.html>`_.

    Parameters
    ----------
    filename : string
        Full filename with path of file.

    Returns
    -------
    reader
        Reader object, which can be iterated over.

    """
    try:
        with open(filename, newline='') as csvfile:
            return csv.reader(csvfile, delimiter=',', quotechar='"')
    except Exception as e:
        raise e
    finally:
        csvfile.close() 
Example 8
Project: pudl   Author: catalyst-cooperative   File: epacems.py    License: MIT License 6 votes vote down vote up
def read_cems_csv(filename):
    """
    Read a CEMS CSV file, compressed or not, into a :class:`pandas.DataFrame`.

    Note that some columns are not read. See
    :mod:`pudl.constants.epacems_columns_to_ignore`. Data types for the columns
    are specified in :mod:`pudl.constants.epacems_csv_dtypes` and names of the
    output columns are set by :mod:`pudl.constants.epacems_rename_dict`.

    Args:
        filename (str): The name of the file to be read

    Returns:
        pandas.DataFrame: A DataFrame containing the contents of the
        CSV file.

    """
    df = pd.read_csv(
        filename,
        index_col=False,
        usecols=lambda col: col not in pc.epacems_columns_to_ignore,
        dtype=pc.epacems_csv_dtypes,
    ).rename(columns=pc.epacems_rename_dict)
    return df 
Example 9
Project: isitfit   Author: autofitcloud   File: tagsPush.py    License: Apache License 2.0 6 votes vote down vote up
def read_csv(self):
    import pandas as pd
    try:
      # read all fields as string
      self.csv_df = pd.read_csv(self.csv_fn, dtype=str)
    except pd.errors.EmptyDataError as e_info:
      raise IsitfitCliError("Error reading csv: %s"%str(e_info), self.ctx)

    if self.csv_df.shape[0]==0:
      raise IsitfitCliError("Tags csv file is empty", self.ctx)

    if 'instance_id' not in self.csv_df.columns:
      raise IsitfitCliError("Missing column instance_id", self.ctx)

    # sort by instance ID
    self.csv_df = self.csv_df.sort_values('instance_id', ascending=True)

    # fill na with ''
    self.csv_df = self.csv_df.fillna(value='') 
Example 10
Project: fileflow   Author: industrydive   File: task_runner.py    License: Apache License 2.0 6 votes vote down vote up
def read_upstream_pandas_csv(self, data_dependency_key, dag_id=None, encoding='utf-8'):
        """
        Reads a csv file from upstream into a pandas DataFrame. Specifically
        reads a csv into memory as a pandas dataframe in a standard
        manner. Reads the data in from a file output by a previous task.

        :param str data_dependency_key: The key (business logic name) for the
            upstream dependency. This will get the value from the
            self.data_dependencies dictionary to determine the file to read
            from.
        :param str dag_id: Defaults to the current DAG id.
        :param str encoding: The file encoding to use. Defaults to 'utf-8'.
        :return: The pandas dataframe.
        :rtype: :py:obj:`pd.DataFrame`
        """
        # Read the upstream file as a stream, abstracting away storage concerns
        input_stream = self.get_upstream_stream(data_dependency_key, dag_id)

        return read_and_clean_csv_to_dataframe(
            filename_or_stream=input_stream,
            encoding=encoding
        ) 
Example 11
Project: BlenderProc   Author: DLR-RM   File: LabelIdMapping.py    License: GNU General Public License v3.0 6 votes vote down vote up
def read_csv_mapping(path):
		""" Loads an idset mapping from a csv file, assuming the rows are sorted by their ids.
		
		:param path: Path to csv file
		"""

		with open(path, 'r') as csvfile:
				reader = csv.DictReader(csvfile)
				new_id_label_map = []
				new_label_id_map = {}

				for row in reader:
					new_id_label_map.append(row["name"])
					new_label_id_map[row["name"]] = int(row["id"])

				return new_id_label_map, new_label_id_map 
Example 12
Project: LSDMappingTools   Author: LSDtopotools   File: PlottingHelpers.py    License: MIT License 6 votes vote down vote up
def ReadChiDataMapCSV(DataDirectory, fname_prefix):
    """
    This function reads in the chi data map csv to a df

    Args:
        DataDirectory: the data directory
        fname_prefix: the file name prefix

    Returns:
        pandas dataframe with the chi map

    Author: FJC
    """
    # get the csv filename
    fname_suffix = "_chi_data_map.csv"
    fname = fname_prefix+fname_suffix
    df = pd.read_csv(DataDirectory+fname)

    return df


#--------------------------------------------------------------------------------#
# Terraces
#--------------------------------------------------------------------------------# 
Example 13
Project: DeepMoon   Author: silburt   File: input_data_gen.py    License: MIT License 6 votes vote down vote up
def ReadHeadCraterCSV(filename="catalogues/HeadCraters.csv", sortlat=True):
    """Reads Head et al. 2010 >= 20 km diameter crater catalogue.

    Parameters
    ----------
    filename : str, optional
        Filepath and name of Head et al. csv file.  Defaults to the one in
        the current folder.
    sortlat : bool, optional
        If `True` (default), order catalogue by latitude.

    Returns
    -------
    craters : pandas.DataFrame
        Craters data frame.
    """
    craters = pd.read_csv(filename, header=0,
                          names=['Long', 'Lat', 'Diameter (km)'])
    if sortlat:
        craters.sort_values(by='Lat', inplace=True)
        craters.reset_index(inplace=True, drop=True)

    return craters 
Example 14
Project: LSDMappingTools   Author: LSDtopotools   File: PlottingHelpers.py    License: MIT License 6 votes vote down vote up
def ReadDisorderUncertCSV(DataDirectory, fname_prefix):
    """
    Function to read in the CSV from the chi disorder
    analysis
    Args:
        DataDirectory: the data directory
        fname_prefix: the file name prefix

    Returns:
        pandas dataframe with the csv file

    Author: FJC
    """
    # get the csv filename
    mc_points_suffix = '_fullstats_disorder_uncert.csv'
    fname = fname_prefix+mc_points_suffix
    # read in the dataframe using pandas
    df = pd.read_csv(DataDirectory+fname)

    return df 
Example 15
Project: anndata   Author: theislab   File: read.py    License: BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
def read_csv(
    filename: Union[PathLike, Iterator[str]],
    delimiter: Optional[str] = ",",
    first_column_names: Optional[bool] = None,
    dtype: str = "float32",
) -> AnnData:
    """\
    Read `.csv` file.

    Same as :func:`~anndata.read_text` but with default delimiter `','`.

    Parameters
    ----------
    filename
        Data file.
    delimiter
        Delimiter that separates data within text file.
        If `None`, will split at arbitrary number of white spaces,
        which is different from enforcing splitting at single white space `' '`.
    first_column_names
        Assume the first column stores row names.
    dtype
        Numpy data type.
    """
    return read_text(filename, delimiter, first_column_names, dtype) 
Example 16
Project: LSDMappingTools   Author: LSDtopotools   File: PlottingHelpers.py    License: MIT License 6 votes vote down vote up
def ReadChannelNetworkCSV(DataDirectory, fname_prefix):
    """
    This function reads in the channel network csv to a df

    Args:
        DataDirectory: the data directory
        fname_prefix: the file name prefix

    Returns:
        pandas dataframe with the channel network

    Author: FJC
    """
    # get the csv filename
    fname_suffix = "_CN.csv"
    fname = fname_prefix+fname_suffix
    df = pd.read_csv(DataDirectory+fname)

    return df 
Example 17
Project: LSDMappingTools   Author: LSDtopotools   File: PlottingHelpers.py    License: MIT License 6 votes vote down vote up
def ReadPerimeterCSV(DataDirectory, fname_prefix):
    """
    This function reads in the csv file with the perimeter info

    Args:
        DataDirectory (str): the data directory
        fname_prefix (str): the base file prefix

    Returns:
        pandas dataframe with the csv file info

    Author: FJC
    """
    csv_suffix = '_Perimeters.csv'
    df = pd.read_csv(DataDirectory+fname_prefix+csv_suffix)
    return df

#-----------------------------------------------------------------------------#
# Functions for appending csvs together for parallel basin running
# FJC 19/10/17
#-----------------------------------------------------------------------------# 
Example 18
Project: graph-partition-neural-network-samples   Author: microsoft   File: reader_helper.py    License: MIT License 6 votes vote down vote up
def read_csv_file(file_name):
  with open(file_name, "r") as ff:
    count = 0

    for line in ff:
      line_str = line.rstrip().split(",")

      if count == 0:
        num_col = len(line_str)
        results = [[] for _ in xrange(num_col)]

      for ii, xx in enumerate(line_str):
        results[ii] += [int(xx)]

      count += 1

  return results 
Example 19
Project: waldo   Author: waldo-seg   File: scoring.py    License: Apache License 2.0 6 votes vote down vote up
def read_csv_as_dict(csv_file):
    """
    This function accepts a csv file and returns a run-length encoding (rle) dictionary, where
    the key is the image_id and the value is a matrix. Each row in this matrix is the rle of 
    an object.
    """
    rle_dict = {}
    with open(csv_file, 'r') as csv_fh:
        csv_reader = csv.reader(csv_fh)
        for row in csv_reader:  # each row represents an object
            image_id = row[0]
            if image_id == 'ImageId':  # skip header row
                continue
            encoded_pixels = row[1].split()
            encoded_pixels = list(map(int, encoded_pixels))
            if image_id not in rle_dict:
                rle_dict[image_id] = [encoded_pixels]
            else:
                rle_dict[image_id].append(encoded_pixels)

    return rle_dict 
Example 20
Project: DeepMoon   Author: silburt   File: input_data_gen.py    License: MIT License 6 votes vote down vote up
def ReadLROCCraterCSV(filename="catalogues/LROCCraters.csv", sortlat=True):
    """Reads LROC 5 - 20 km crater catalogue CSV.

    Parameters
    ----------
    filename : str, optional
        Filepath and name of LROC csv file.  Defaults to the one in the current
        folder.
    sortlat : bool, optional
        If `True` (default), order catalogue by latitude.

    Returns
    -------
    craters : pandas.DataFrame
        Craters data frame.
    """
    craters = pd.read_csv(filename, header=0, usecols=list(range(2, 6)))
    if sortlat:
        craters.sort_values(by='Lat', inplace=True)
        craters.reset_index(inplace=True, drop=True)

    return craters 
Example 21
Project: robotframework-CSVLibrary   Author: s4int   File: __init__.py    License: Apache License 2.0 6 votes vote down vote up
def read_csv_file_to_associative(self, filename, delimiter=',', fieldnames=None, **kwargs):
        """Read CSV file and return its content as a Python list of dictionaries.
        
        - ``filename``:  name of csv file
        - ``delimiter``: Default: `,`
        - ``fieldnames``: list of column names
        - ``line_numbers``: List of linenumbers to read. Default None
        - ``quoting`` (int):
          _0_: QUOTE_MINIMAL
          _1_: QUOTE_ALL
          _2_: QUOTE_NONNUMERIC
          _3_: QUOTE_NONE
        """
        csv_dict = self._open_csv_file_for_read(
            filename,
            csv_reader=csv.DictReader,
            delimiter=str(delimiter),
            fieldnames=fieldnames,
            **kwargs
        )
        return [item for item in csv_dict] 
Example 22
Project: luminol   Author: linkedin   File: utils.py    License: Apache License 2.0 6 votes vote down vote up
def read_csv(csv_name):
    """
    Read data from a csv file into a dictionary.
    :param str csv_name: path to a csv file.
    :return dict: a dictionary represents the data in file.
    """
    data = {}
    if int(sys.version[0]) == 2:
        str_types = (str, unicode)
    else:
        str_types = (bytes, str)
    if not isinstance(csv_name, str_types):
        raise exceptions.InvalidDataFormat('luminol.utils: csv_name has to be a string!')
    with open(csv_name, 'r') as csv_data:
        reader = csv.reader(csv_data, delimiter=',', quotechar='|')
        for row in reader:
            try:
                key = to_epoch(row[0])
                value = float(row[1])
                data[key] = value
            except ValueError:
                pass
    return data 
Example 23
Project: climatelearn   Author: Ambrosys   File: read.py    License: GNU General Public License v2.0 6 votes vote down vote up
def read_csv(path, sep=None, feat_drop=None, date_key=None, dropna=False, drop_axis=None):
    """
    Wraps the pandas.read_csv function adding extra features.

    :path: string
        Path to the csv file to be read.
    :sep: char
        Same argument as pandas.read_csv function.
    :feat_drop: list of strings
        Features to drop
    :date_key: string
        The name of the key representing the date in the dataset.
    :returns: Pandas DataFrame
        The pandas dataframe created from the data.
    """
    df = pd.read_csv(path, sep=sep, index_col=date_key)
    if dropna:
        for d in drop_axis:
            df = df.dropna(axis=d)
    if feat_drop is not None:
        df = df.drop(feat_drop, axis=1)
    return df 
Example 24
Project: mentornet   Author: google   File: data_generator.py    License: Apache License 2.0 6 votes vote down vote up
def read_from_csv(input_csv_file):
  """Reads Data from an input CSV file.

  Args:
    input_csv_file: the path of the CSV file.

  Returns:
    a numpy array with different data at each index:
  """
  data = {}
  with open(input_csv_file, 'r') as csv_file_in:
    reader = csv.reader(csv_file_in)
    for row in reader:
      for (_, cell) in enumerate(row):
        rdata = cell.strip().split(' ')
        rid = rdata[0]
        rdata = [float(t) for t in rdata[1:]]
        data[rid] = rdata
    csv_file_in.close()
  return data 
Example 25
Project: robotframework-CSVLibrary   Author: s4int   File: __init__.py    License: Apache License 2.0 6 votes vote down vote up
def read_csv_file_to_list(self, filename, delimiter=',', **kwargs):
        """Read CSV file and return its content as a Python list of tuples.
        
        - ``filename``:  name of csv file
        - ``delimiter``: Default: `,`
        - ``line_numbers``: List of linenumbers to read. Default None
        - ``quoting`` (int):
          _0_: QUOTE_MINIMAL
          _1_: QUOTE_ALL
          _2_: QUOTE_NONNUMERIC
          _3_: QUOTE_NONE
        """
        csv_list = self._open_csv_file_for_read(
            filename,
            csv_reader=csv.reader,
            delimiter=str(delimiter),
            **kwargs
        )
        return [tuple(row) for row in csv_list] 
Example 26
Project: e2e-nlg-challenge-2017   Author: UKPLab   File: __init__.py    License: Apache License 2.0 6 votes vote down vote up
def read_csv_test(self, fname):
        raw_data_x = []
        lexicalizations = []

        with open(fname, 'r') as csv_file:
            reader = csv.reader(csv_file, delimiter=',', quotechar='"')
            header = next(reader)
            for row in list(reader):
                mr = row[0]
                this_src, this_lex = self.process_e2e_mr(mr)

                # add raw data instance
                raw_data_x.append(this_src)
                lexicalizations.append(this_lex)

        return raw_data_x, None, lexicalizations 
Example 27
Project: LSDMappingTools   Author: LSDtopotools   File: PlottingHelpers.py    License: MIT License 6 votes vote down vote up
def ReadBasinStatsCSV(DataDirectory, fname_prefix):
    """
    This function reads in the file with the suffix '_disorder_basinstats.csv'
    to a pandas dataframe

    Args:
        DataDirectory: the data directory
        fname_prefix: the file name prefix

    Returns:
        pandas dataframe with the csv file

    Author: FJC
    """
    
    # get the csv filename
    basin_stats_suffix = '_disorder_basinstats.csv'
    fname = fname_prefix+basin_stats_suffix
    # read in the dataframe using pandas
    df = pd.read_csv(DataDirectory+fname)

    return df 
Example 28
Project: CASED-Tensorflow   Author: taki0112   File: convert_luna_to_npy.py    License: MIT License 6 votes vote down vote up
def read_csv(filename):
    lines = []
    with open(filename, 'r') as f:
        csvreader = csv.reader(f)
        for line in csvreader:
            lines.append(line)

    lines = lines[1:] # remove csv headers
    annotations_dict = {}
    for i in lines:
        series_uid, x, y, z, diameter = i
        value = {'position':[float(x),float(y),float(z)],
                 'diameter':float(diameter)}
        if series_uid in annotations_dict.keys():
            annotations_dict[series_uid].append(value)
        else:
            annotations_dict[series_uid] = [value]

    return annotations_dict 
Example 29
Project: geoist   Author: igp-gravity   File: utils.py    License: MIT License 6 votes vote down vote up
def read_csv(csv_name):
    """
    Read data from a csv file into a dictionary.
    :param str csv_name: path to a csv file.
    :return dict: a dictionary represents the data in file.
    """
    data = {}
    if int(sys.version[0]) == 2:
        str_types = (str, unicode)
    else:
        str_types = (bytes, str)
    if not isinstance(csv_name, str_types):
        raise exceptions.InvalidDataFormat('geoist.snoopy.utils: csv_name has to be a string!')
    with open(csv_name, 'r') as csv_data:
        reader = csv.reader(csv_data, delimiter=',', quotechar='|')
        for row in reader:
            try:
                key = to_epoch(row[0])
                value = float(row[1])
                data[key] = value
            except ValueError:
                pass
    return data 
Example 30
Project: CTGAN   Author: sdv-dev   File: data.py    License: MIT License 6 votes vote down vote up
def read_csv(csv_filename, meta_filename=None, header=True, discrete=None):

    data = pd.read_csv(csv_filename, header='infer' if header else None)

    if meta_filename:
        with open(meta_filename) as meta_file:
            metadata = json.load(meta_file)

        discrete_columns = [
            column['name']
            for column in metadata['columns']
            if column['type'] != 'continuous'
        ]

    elif discrete:
        discrete_columns = discrete.split(',')
        if not header:
            discrete_columns = [int(i) for i in discrete_columns]

    else:
        discrete_columns = []

    return data, discrete_columns 
Example 31
Project: CleverCSV   Author: alan-turing-institute   File: wrappers.py    License: MIT License 6 votes vote down vote up
def read_csv(
    filename, dialect=None, encoding=None, num_chars=None, verbose=False,
):
    """This function is deprecated, use read_table instead."""
    warnings.warn(
        "'read_csv' was renamed to 'read_table' in version "
        "0.6.3 and will be removed in 0.7.0.",
        FutureWarning,
    )
    return read_table(
        filename,
        dialect=dialect,
        encoding=encoding,
        num_chars=num_chars,
        verbose=verbose,
    ) 
Example 32
Project: 3d-dl   Author: 921kiyo   File: train_keras_retinanet.py    License: MIT License 6 votes vote down vote up
def read_class_csv(csv_class_file):
    """
    function that reads in a class csv file that is used as an input to the retinanet training routine, and outputs
    a list of class names which is the format required by Keras ImageGenerator
    :param csv_class_file: the filename of the csv containing class names and indices
    :return: list of classnames
    """
    try:
        with csv_generator._open_for_csv(csv_class_file) as file:
            classes = csv_generator._read_classes(csv.reader(file, delimiter=','))
            class_list = list(classes.keys())
            for c in classes:
                class_idx = classes[c]
                class_list[class_idx] = c
            return class_list
    except ValueError as e:
        raise_from(ValueError('invalid CSV class file: {}: {}'.format(csv_class_file, e)), None) 
Example 33
Project: train-procgen   Author: openai   File: graph_util.py    License: MIT License 6 votes vote down vote up
def read_csv(filename, key_name):
    with open(filename) as csv_file:
        csv_reader = csv.reader(csv_file, delimiter=',')
        key_index = -1

        values = []

        for line_num, row in enumerate(csv_reader):
            row = [x.lower() for x in row]
            if line_num == 0:
                idxs = [i for i, val in enumerate(row) if val == key_name]
                key_index = idxs[0]
            else:
                values.append(row[key_index])

    return np.array(values, dtype=np.float32) 
Example 34
Project: pipeline   Author: ooni   File: utils.py    License: BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
def read_fingerprints_csv():
    with open("fingerprints.csv", newline="", encoding="utf-8") as f:
        reader = csv.reader(f)
        fingerprints = {}
        for row in reader:
            num, cc, body_match, header_name, header_prefix, header_full = row
            if cc not in fingerprints:
                fingerprints[cc] = []

            d = {}
            if body_match:
                d["body_match"] = body_match
            else:
                d["header_name"] = header_name
                if header_full:
                    d["header_full"] = header_full
                else:
                    d["header_prefix"] = header_prefix
            fingerprints[cc].append(d)
        print(fingerprints) 
Example 35
Project: LSDMappingTools   Author: LSDtopotools   File: PlottingHelpers.py    License: MIT License 6 votes vote down vote up
def read_index_channel_csv(DataDirectory,fname_prefix):
    """
    This function reads in the csv file with the extension "_index_chan.csv"
    and returns it as a pandas dataframe

    Args:
        DataDirectory (str): the data directory
        fname_prefix (str): the name of the DEM

    Returns:
        pandas dataframe with the channel info

    Author: FJC
    """
    csv_suffix = '_index_chan.csv'
    fname = DataDirectory+fname_prefix+csv_suffix

    df = pd.read_csv(fname)

    return df 
Example 36
Project: hwrt   Author: MartinThoma   File: filter_dataset.py    License: MIT License 6 votes vote down vote up
def read_csv(filepath: str) -> Sequence[Dict[Any, Any]]:
    """
    Read a CSV into a list of dictionarys. The first line of the CSV determines
    the keys of the dictionary.

    Parameters
    ----------
    filepath : str

    Returns
    -------
    symbols : List[Dict]
    """
    symbols = []
    with open(filepath) as csvfile:
        spamreader = csv.DictReader(csvfile, delimiter=",", quotechar='"')
        for row in spamreader:
            symbols.append(row)
    return symbols 
Example 37
Project: neural_graph_evolution   Author: WilsonWangTHU   File: visual_util.py    License: MIT License 6 votes vote down vote up
def read_csv(filepath, skipfirst=True, convert2num=True):
    ''' read a csv file and return its content
    '''
    data = []
    with open(filepath, newline='') as csvfile:
        reader = csv.reader(csvfile)

        for row in reader:
            data.append( row )

    if skipfirst: start = 1
    else: start = 0
    data = data[start:]

    if convert2num:
        for i, row in enumerate(data):
            data[i] = [float(x) for x in row]

    return data 
Example 38
Project: pointnet-registration-framework   Author: vinits5   File: plot_threshold_vs_success.py    License: MIT License 6 votes vote down vote up
def read_csv(folder_name):
	data = []
	# Each folder having results contain test.csv file with all the log.
	# Read all data from the csv file.
	with open(os.path.join(folder_name, 'test.csv')) as csvfile:
		csvreader = csv.reader(csvfile)
		for row in csvreader:
			row = [float(x) for x in row]
			data.append(row)

	rot_err, trans_err = [], []

	# Log stored is as per following sequence in csv files:
	# Sr. No. [0], time taken [1], number of iterations [2], translation error [3], rotation error [4].
	if folder_name[5:9]=='PNLK':
		for data_i in data:
			rot_err.append(data_i[2])
			trans_err.append(data_i[1])
	else:	
		for data_i in data:
			rot_err.append(data_i[4])
			trans_err.append(data_i[3])
	return rot_err, trans_err

# It will count the total number of test cases having rotation error below certain threshold. 
Example 39
Project: h2o4gpu   Author: h2oai   File: __init__.py    License: Apache License 2.0 6 votes vote down vote up
def readTensorFromCSV(datasetFileName, allowOneColumn=False):
    dataSource = FileDataSource(datasetFileName,
                                DataSourceIface.doAllocateNumericTable,
                                DataSourceIface.doDictionaryFromContext)
    dataSource.loadDataBlock()

    nt = dataSource.getNumericTable()
    size = nt.getNumberOfRows()
    block = BlockDescriptor()
    nt.getBlockOfRows(0, size, readOnly, block)
    blockData = block.getArray().flatten()

    dims = [size]
    if nt.getNumberOfColumns() > 1 or allowOneColumn:
        dims.append(nt.getNumberOfColumns())
        size *= dims[1]

    tensorData = np.array(blockData, dtype=np.float32)

    nt.releaseBlockOfRows(block)

    tensorData.shape = dims
    tensor = HomogenTensor(tensorData, ntype=np.float32)

    return tensor 
Example 40
Project: anomalydetector   Author: microsoft   File: utils.py    License: MIT License 6 votes vote down vote up
def read_csv_kpi(path):
    tm = []
    vl = []
    lb = []
    with open(path) as f:
        input = csv.reader(f, delimiter=',')
        cnt = 0
        for row in input:
            if cnt == 0:
                cnt += 1
                continue
            tm.append(int(row[0]))
            vl.append(float(row[1]))
            lb.append(int(row[2]))
            cnt += 1
        f.close()
    return tm, vl, lb 
Example 41
Project: LSDMappingTools   Author: LSDtopotools   File: PlottingHelpers.py    License: MIT License 6 votes vote down vote up
def ReadBasinStatsPointCSV(DataDirectory, fname_prefix):
    """
    This function reads in the file with the suffix '_point_movernstats_basinstats.csv'
    to a pandas dataframe

    Args:
        DataDirectory: the data directory
        fname_prefix: the file name prefix

    Returns:
        pandas dataframe with the csv file

    Author: FJC
    """
    # get the csv filename
    csv_suffix = '_point_movernstats_basinstats.csv'
    fname = fname_prefix+csv_suffix
    # read in the dataframe using pandas
    df = pd.read_csv(DataDirectory+fname)

    return df 
Example 42
Project: nlp-architect   Author: NervanaSystems   File: data.py    License: Apache License 2.0 6 votes vote down vote up
def read_csv_file_data(input_path):
    """
    Read csv file to a list

    Args:
        input_path (str): read csv file from this local file path

    Returns:
        list(str): A list where each item is a row in the csv file
    """
    # 1. read csv file
    if not os.path.isabs(input_path):
        # handle case using default value\relative paths
        input_path = os.path.join(os.path.dirname(__file__), input_path)
    with open(input_path, "r", encoding="utf-8-sig") as input_file:
        reader = csv.reader((line.replace("\0", "") for line in input_file))
        reader_list = list(reader)
    return reader_list 
Example 43
Project: pycsvw   Author: bloomberg   File: generator_utils.py    License: Apache License 2.0 6 votes vote down vote up
def read_csv(handle):
    """ Read CSV file
    :param handle: File-like object of the CSV file
    :return: csv.reader object
    """

    # These functions are to handle unicode in Python 2 as described in:
    # https://docs.python.org/2/library/csv.html#examples
    def unicode_csv_reader(unicode_csv_data, dialect=csv.excel, **kwargs):
        """ csv.py doesn't do Unicode; encode temporarily as UTF-8."""
        csv_reader = csv.reader(utf_8_encoder(unicode_csv_data),
                                dialect=dialect, **kwargs)
        for row in csv_reader:
            # decode UTF-8 back to Unicode, cell by cell:
            yield [unicode(cell, 'utf-8') for cell in row]

    def utf_8_encoder(unicode_csv_data):
        """ Encode with UTF-8."""
        for line in unicode_csv_data:
            yield line.encode('utf-8')

    return unicode_csv_reader(handle) if PY2 else csv.reader(handle) 
Example 44
Project: sqlite_bro   Author: stonebig   File: sqlite_bro.py    License: MIT License 6 votes vote down vote up
def read_this_csv(csv_file, encoding, delimiter , quotechar, header, decim):
    """yield csv data records from a file """
    # handle Python 2/3
    try:
        reader = csv.reader(open(csv_file, 'r', encoding=encoding),
                            delimiter=delimiter, quotechar=quotechar)
    except:  # minimal hack for 2.7
        reader = csv.reader(open(csv_file, 'r'),
                            delimiter=str(delimiter), quotechar=str(quotechar))
    # handle header
    if header:
        next(reader)
    # otherwise handle special decimal treatment
    for row in reader:
        if decim != "." and not isinstance(row, (type('e'), type(u'e'))):
                for i in range(len(row)):
                    row[i] = row[i].replace(decim, ".")
        yield(row) 
Example 45
Project: LSDMappingTools   Author: LSDtopotools   File: PlottingHelpers.py    License: MIT License 6 votes vote down vote up
def ReadMovernCSV(DataDirectory, fname_prefix):
    """
    This function reads in a the movern csv with the suffix "_movern"

    Args:
        DataDirectory (str): the data DataDirectory
        fname_prefix
    Returns:
        pandas dataframe with the appended movern csvs

    Author: MDH
    """

    # get the csv filename
    csv_suffix = '_movern.csv'

    df = pd.read_csv(DataDirectory+fname_prefix+csv_suffix)

    return df 
Example 46
Project: LSDMappingTools   Author: LSDtopotools   File: PlottingHelpers.py    License: MIT License 6 votes vote down vote up
def ReadKnickpointCSV(DataDirectory, fname_prefix, ftype = "normal"):
    """
    This function reads in the file with the suffix
    '_KsnKn.csv'
    This file holds the MCHI segmented data

    Args:
        DataDirectory: the data directory
        fname_prefix: the file name prefix

    Returns:
        pandas dataframe with the csv file

    Author: BG
    """
    # get the csv filename
    if(ftype == "raw"):
        suffix = '_ksnkp_raw.csv'
    else:
        suffix = '_ksnkp.csv'
    fname = fname_prefix+suffix
    # read in the dataframe using pandas
    df = pd.read_csv(DataDirectory+fname)

    return df 
Example 47
Project: edgar-10k-mda   Author: tzuhsial   File: edgar.py    License: MIT License 6 votes vote down vote up
def read_url_from_combined_csv(csv_path):
    """ Reads url from csv file
    Args:
        csv_path (str): path to index file
    Returns
        urls: urls in combined csv
    """
    urls = []
    with open(csv_path, 'r') as fin:
        reader = csv.reader(fin, delimiter=",",
                            quotechar='\"', quoting=csv.QUOTE_ALL)
        # Skip header
        next(reader)
        for row in reader:
            url = row[-1]
            urls.append(url)
    return urls 
Example 48
Project: simcoin   Author: sbaresearch   File: utils.py    License: MIT License 6 votes vote down vote up
def read_csv(file_name):
    if os.path.isfile(file_name):
        with open(file_name, 'r') as file:
            try:
                reader = csv.reader(file)
                Object = namedtuple("Object", next(reader))
                objects = []
                for line in reader:
                    for i, var in enumerate(line):
                        try:
                            line[i] = literal_eval(var)
                        except ValueError:
                            pass
                        except SyntaxError:
                            pass
                    objects.append(Object._make(line))
                return objects
            except StopIteration:
                logging.debug('File={} has not enough lines'.format(config.args_csv))
                return []
    else:
        return [] 
Example 49
Project: football_predictions   Author: DOsinga   File: calculate_model.py    License: Apache License 2.0 6 votes vote down vote up
def read_csv(path, base_weight):
  reader = csv.DictReader(file(path))
  games = list(reader)
  sum_diff = 0.0
  sum_count = 0
  year = 2016
  prev_month = None
  now = datetime.datetime.now()
  for game in games:
    goals1, goals2 = map(int, game['result'].split(':'))
    game['goals1'] = goals1
    game['goals2'] = goals2
    day, month, _ = game['date'].split('.')
    day = int(day)
    month = int(month)
    if prev_month and prev_month < month:
      year -= 1
    prev_month = month
    game['weight'] = base_weight / max((now - datetime.datetime(year, month, day)).days / 180.0, 1.0)
    game['date'] = '%d-%02d-%02d' % (year, month, day)
    sum_count += 1
    if 'Netherlands' in (game['team1'], game['team2']):
      print '%(team1)s - %(team2)s  %(goals1)d - %(goals2)d   (%(weight)2.2f)' % game
  return games, sum_diff, sum_count 
Example 50
Project: LSDMappingTools   Author: LSDtopotools   File: PlottingHelpers.py    License: MIT License 6 votes vote down vote up
def ReadModelCSV(DataDirectory, Base_file):
    """
    This function reads in the csv file from the model run to a pandas dataframe

    Args:
        DataDirectory (str): the data directory
        Base_file (str): the base file prefix

    Returns:
        pandas dataframe with the csv file info

    Author: FJC
    """
    # get the csv filename
    csv_suffix = '_model_info.csv'

    fname = Base_file+csv_suffix
    # read in the dataframe using pandas
    df = pd.read_csv(DataDirectory+fname)
    return df

#-----------------------------------------------------------------------------#
# Drainage capture metrics
#-----------------------------------------------------------------------------# 
Example 51
Project: blink-docker   Author: plaperdr   File: mainContainer.py    License: MIT License 6 votes vote down vote up
def readCsvFile(path):
        #########
        # Format
        # 1 - Name of font/plugin
        # 2 - Name of file
        # 3 - Weight
        #########
        l = []
        with open(path, newline='') as csvFile:
            reader = csv.reader(csvFile, delimiter=',')
            for row in reader:
                l.append((row[0],row[1],int(row[2])))
        return l


############### Main 
Example 52
Project: toil   Author: DataBiosphere   File: wdl_functions.py    License: Apache License 2.0 6 votes vote down vote up
def read_csv(f):
    '''
    Take a csv filepath and return an array; e.g. [[],[],[]].

    For example, a file containing:

    1,2,3
    4,5,6
    7,8,9

    would return the array: [['1','2','3'], ['4','5','6'], ['7','8','9']]

    :param csv_filepath:
    :return: csv_array
    '''
    return read_tsv(f, delimiter=",") 
Example 53
Project: SPN.pytorch   Author: yeezhu   File: voc.py    License: MIT License 6 votes vote down vote up
def read_object_labels_csv(file, header=True):
    images = []
    num_categories = 0
    print('[dataset] read', file)
    with open(file, 'r') as f:
        reader = csv.reader(f)
        rownum = 0
        for row in reader:
            if header and rownum == 0:
                header = row
            else:
                if num_categories == 0:
                    num_categories = len(row) - 1
                name = row[0]
                labels = (np.asarray(row[1:num_categories + 1])).astype(np.float32)
                labels = torch.from_numpy(labels)
                item = (name, labels)
                images.append(item)
            rownum += 1
    return images 
Example 54
Project: LSDMappingTools   Author: LSDtopotools   File: PlottingHelpers.py    License: MIT License 6 votes vote down vote up
def ReadFullStatsCSV(DataDirectory, fname_prefix, m_over_n):
    """
    This function reads in the file with the suffix '_fullstats.csv'
    to a pandas dataframe. Must specify the m/n value as an argument

    Args:
        DataDirectory: the data directory
        fname_prefix: the file name prefix
        m_over_n: the m/n value

    Returns:
        pandas dataframe with the csv file

    Author: FJC
    """
    
    # get the csv filename
    fullstats_suffix = '_movernstats_%s_fullstats.csv' % m_over_n
    fname = fname_prefix+fullstats_suffix
    # read in the dataframe using pandas
    df = pd.read_csv(DataDirectory+fname)

    return df 
Example 55
Project: LSDMappingTools   Author: LSDtopotools   File: PlottingHelpers.py    License: MIT License 6 votes vote down vote up
def ReadSourceKeysCSV(DataDirectory, fname_prefix):
    """
    This function reads in the file with the suffix '_SourceKeys.csv'
    to a pandas dataframe

    Args:
        DataDirectory: the data directory
        fname_prefix: the file name prefix

    Returns:
        pandas dataframe with the csv file

    Author: FJC
    """
    
    # get the csv filename
    source_keys_suffix = '_SourceKeys.csv'
    fname = fname_prefix+source_keys_suffix
    # read in the dataframe using pandas
    df = pd.read_csv(DataDirectory+fname)

    return df 
Example 56
Project: pcrnet   Author: vinits5   File: plot_threshold_vs_success.py    License: MIT License 6 votes vote down vote up
def read_csv(folder_name):
	data = []
	# Each folder having results contain test.csv file with all the log.
	# Read all data from the csv file.
	with open(os.path.join(folder_name, 'test.csv')) as csvfile:
		csvreader = csv.reader(csvfile)
		for row in csvreader:
			row = [float(x) for x in row]
			data.append(row)

	rot_err, trans_err = [], []

	# Log stored is as per following sequence in csv files:
	# Sr. No. [0], time taken [1], number of iterations [2], translation error [3], rotation error [4].
	if folder_name[5:9]=='PNLK':
		for data_i in data:
			rot_err.append(data_i[2])
			trans_err.append(data_i[1])
	else:	
		for data_i in data:
			rot_err.append(data_i[4])
			trans_err.append(data_i[3])
	return rot_err, trans_err

# It will count the total number of test cases having rotation error below certain threshold. 
Example 57
Project: quantrocket-client   Author: quantrocket-llc   File: moonshot.py    License: Apache License 2.0 6 votes vote down vote up
def read_moonshot_csv(filepath_or_buffer):
    """
    Load a Moonshot backtest CSV into a DataFrame.

    This is a light wrapper around pd.read_csv that handles setting index
    columns and casting to proper data types.

    Parameters
    ----------
    filepath_or_buffer : string or file-like, required
        path to CSV

    Returns
    -------
    DataFrame
        a multi-index (Field, Date[, Time]) DataFrame of backtest
        results, with sids or strategy codes as columns

    Examples
    --------
    >>> results = read_moonshot_csv("moonshot_backtest.csv")
    >>> returns = results.loc["Return"]
    """
    return _read_moonshot_or_pnl_csv(filepath_or_buffer) 
Example 58
Project: open-context-py   Author: ekansa   File: api.py    License: GNU General Public License v3.0 6 votes vote down vote up
def get_read_csv(self, url):
        """
        gets json daa from a geonames_uri
        """
        if self.delay_before_request > 0:
            # default to sleep BEFORE a request is sent, to
            # give the remote service a break.
            sleep(self.delay_before_request)
        try:
            gapi = GeneralAPI()
            r = requests.get(url,
                             timeout=240,
                             headers=gapi.client_headers)
            r.raise_for_status()
            csvfile = r.text.split('\n')
            self.csv_data = csv.reader(csvfile)
        except:
            self.csv_data = False
        return self.csv_data 
Example 59
Project: LSDMappingTools   Author: LSDtopotools   File: PlottingHelpers.py    License: MIT License 6 votes vote down vote up
def ReadKnickzoneCSV(DataDirectory, fname_prefix):
    """
    This function reads in the file with the suffix
    '_KsnKn.csv'
    This file holds the MCHI segmented data

    Args:
        DataDirectory: the data directory
        fname_prefix: the file name prefix

    Returns:
        pandas dataframe with the csv file

    Author: BG
    """
    # get the csv filename
    suffix = '_KsnKz.csv'
    fname = fname_prefix+suffix
    # read in the dataframe using pandas
    df = pd.read_csv(DataDirectory+fname)

    return df 
Example 60
Project: LSDMappingTools   Author: LSDtopotools   File: PlottingHelpers.py    License: MIT License 6 votes vote down vote up
def read_channel_csv(DataDirectory,fname_prefix):
    """
    This function reads in the csv file with the extension "_baseline_channel_info.csv"
    and returns it as a pandas dataframe

    Args:
        DataDirectory (str): the data directory
        fname_prefix (str): the name of the DEM

    Returns:
        pandas dataframe with the channel info

    Author: FJC
    """
    csv_suffix = '_baseline_channel_info.csv'
    fname = DataDirectory+fname_prefix+csv_suffix

    df = pd.read_csv(fname)

    return df