Python csv.reader() Examples

The following are 30 code examples for showing how to use csv.reader(). These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example.

You may check out the related API usage on the sidebar.

You may also want to check out all available functions/classes of the module csv , or try the search function .

Example 1
Project: incubator-spot   Author: apache   File: geoloc.py    License: Apache License 2.0 7 votes vote down vote up
def get_ip_geo_localization(self, ip):

        self._logger.debug("Getting {0} geo localization ".format(ip))
        if ip.strip() != "" and ip is not None:

            result = linecache.getline(self._ip_localization_file, bisect.bisect(self._ip_localization_ranges, Util.ip_to_int(ip)))
            result.strip('\n')

            reader = csv.reader([result])
            row = reader.next()

            geo_loc = ";".join(row[4:6]) + " " + ";".join(row[8:9])            
            domain = row[9:10][0]

            result = {"geo_loc": geo_loc, "domain": domain}

        return result 
Example 2
Project: indras_net   Author: gcallah   File: barter_model.py    License: GNU General Public License v3.0 6 votes vote down vote up
def fetch_agents_from_file(self, filenm, agent_type):
        """
        Read in a list of bartering agents from a csv file
        """

        max_detect = self.props.get("max_detect",
                                    ebm.GLOBAL_KNOWLEDGE)
        with open(filenm) as f:
            reader = csv.reader(f)
            for row in reader:
                agent = agent_type(row[0], max_detect=max_detect)
                self.add_agent(agent)
                for i in range(1, len(row) - 2, STEP):
                    good = row[i]
                    self.market.add_good(good)
                    agent.endow(good,
                                int(row[i + 1]),
                                eval("lambda qty: "
                                     + row[i + 2]))
        logging.info("Goods = " + str(self.market)) 
Example 3
Project: vergeml   Author: mme   File: __init__.py    License: MIT License 6 votes vote down vote up
def load_predictions(env, nclasses):
    path = os.path.join(env.stats_dir(), "predictions.csv")

    if not os.path.exists(path):
        raise FileExistsError(path)

    with open(path, newline='') as csvfile:
        y_score = []
        y_test = []
        csv_reader = csv.reader(csvfile, dialect="excel")
        for row in csv_reader:
            assert len(row) == nclasses * 2
            y_score.append(list(map(float, row[:nclasses])))
            y_test.append(list(map(float, row[nclasses:])))
        
        y_score = np.array(y_score)
        y_test = np.array(y_test)

        return y_test, y_score 
Example 4
def __init__(self, filename):
    """Initializes instance of DatasetMetadata."""
    self._true_labels = {}
    self._target_classes = {}
    with open(filename) as f:
      reader = csv.reader(f)
      header_row = next(reader)
      try:
        row_idx_image_id = header_row.index('ImageId')
        row_idx_true_label = header_row.index('TrueLabel')
        row_idx_target_class = header_row.index('TargetClass')
      except ValueError:
        raise IOError('Invalid format of dataset metadata.')
      for row in reader:
        if len(row) < len(header_row):
          # skip partial or empty lines
          continue
        try:
          image_id = row[row_idx_image_id]
          self._true_labels[image_id] = int(row[row_idx_true_label])
          self._target_classes[image_id] = int(row[row_idx_target_class])
        except (IndexError, ValueError):
          raise IOError('Invalid format of dataset metadata') 
Example 5
def load_images(input_dir, metadata_file_path, batch_shape):
    """Retrieve numpy arrays of images and labels, read from a directory."""
    num_images = batch_shape[0]
    with open(metadata_file_path) as input_file:
        reader = csv.reader(input_file)
        header_row = next(reader)
        rows = list(reader)

    row_idx_image_id = header_row.index('ImageId')
    row_idx_true_label = header_row.index('TrueLabel')
    images = np.zeros(batch_shape)
    labels = np.zeros(num_images, dtype=np.int32)
    for idx in xrange(num_images):
        row = rows[idx]
        filepath = os.path.join(input_dir, row[row_idx_image_id] + '.png')

        with tf.gfile.Open(filepath, 'rb') as f:
            image = np.array(
                Image.open(f).convert('RGB')).astype(np.float) / 255.0
        images[idx, :, :, :] = image
        labels[idx] = int(row[row_idx_true_label])
    return images, labels 
Example 6
Project: dynamic-training-with-apache-mxnet-on-aws   Author: awslabs   File: Train.py    License: Apache License 2.0 6 votes vote down vote up
def accumulate_result(validate_lst, prob):
    sum_result = {}
    cnt_result = {}
    size = prob.shape[0]
    fi = csv.reader(open(validate_lst))
    for i in range(size):
        line = fi.__next__() # Python2: line = fi.next()
        idx = int(line[0])
        if idx not in cnt_result:
            cnt_result[idx] = 0.
            sum_result[idx] = np.zeros((1, prob.shape[1]))
        cnt_result[idx] += 1
        sum_result[idx] += prob[i, :]
    for i in cnt_result.keys():
        sum_result[i][:] /= cnt_result[i]
    return sum_result


# In[9]: 
Example 7
Project: DOTA_models   Author: ringringyi   File: graphs.py    License: Apache License 2.0 6 votes vote down vote up
def _get_vocab_freqs():
  """Returns vocab frequencies.

  Returns:
    List of integers, length=FLAGS.vocab_size.

  Raises:
    ValueError: if the length of the frequency file is not equal to the vocab
      size, or if the file is not found.
  """
  path = FLAGS.vocab_freq_path or os.path.join(FLAGS.data_dir, 'vocab_freq.txt')

  if tf.gfile.Exists(path):
    with tf.gfile.Open(path) as f:
      # Get pre-calculated frequencies of words.
      reader = csv.reader(f, quoting=csv.QUOTE_NONE)
      freqs = [int(row[-1]) for row in reader]
      if len(freqs) != FLAGS.vocab_size:
        raise ValueError('Frequency file length %d != vocab size %d' %
                         (len(freqs), FLAGS.vocab_size))
  else:
    if FLAGS.vocab_freq_path:
      raise ValueError('vocab_freq_path not found')
    freqs = [1] * FLAGS.vocab_size

  return freqs 
Example 8
Project: InsightAgent   Author: insightfinder   File: get_metadata.py    License: Apache License 2.0 6 votes vote down vote up
def parse_topology_file(self):
        topology_list = []
        if os.path.isfile(self.file_path):
            with open(self.file_path) as topology_file:
                topology_file_csv = csv.reader(topology_file)
                for row in topology_file_csv:
                    map_size = len(bytearray(json.dumps(topology_list)))
                    if map_size >= BYTES_PER_FLUSH:
                        self._send_data(topology_list)
                        topology_list = []
                    if topology_file_csv.line_num == 1:
                        continue
                    key = ""
                    for index in xrange(len(row)):
                        if index == 0:
                            key = row[index]
                            continue
                        value1 = str(key) + "@@@@" + str(row[index])
                        value2 = str(row[index]) + "@@@@" + str(key)
                        if value1 not in topology_list:
                            topology_list.append(value1)
                        if value2 not in topology_list:
                            topology_list.append(value2)
                self._send_data(topology_list) 
Example 9
Project: fine-lm   Author: akzaidi   File: common_voice.py    License: MIT License 6 votes vote down vote up
def _collect_data(directory):
  """Traverses directory collecting input and target files.

  Args:
   directory: base path to extracted audio and transcripts.
  Returns:
   list of (media_base, media_filepath, label) tuples
  """
  # Returns:
  data_files = []
  transcripts = [
      filename for filename in os.listdir(directory)
      if filename.endswith(".csv")
  ]
  for transcript in transcripts:
    transcript_path = os.path.join(directory, transcript)
    with open(transcript_path, "r") as transcript_file:
      transcript_reader = csv.reader(transcript_file)
      _ = transcript_reader.next()  # Skip headers.
      for transcript_line in transcript_reader:
        media_name, label = transcript_line[0:2]
        filename = os.path.join(directory, media_name)
        data_files.append((media_name, filename, label))
  return data_files 
Example 10
Project: URS   Author: JosephLai241   File: test_Export.py    License: MIT License 6 votes vote down vote up
def test_write_csv(self):
        filename = os.path.join(sys.path[0], "test_csv_writing.csv")
        overview = {
            "this": [1, 2],
            "is": [3, 4],
            "a": [5, 6],
            "test": [7, 8]}

        Export.Export._write_csv(filename, overview)

        with open(filename, "r") as test_csv:
            reader = csv.reader(test_csv)
            test_dict = dict((header, []) for header in next(reader))
            for row in reader:
                for row_index, key in enumerate(test_dict.keys()):
                    test_dict[key].append(int(row[row_index]))

        assert test_dict == overview
        os.remove(filename) 
Example 11
Project: labuildings   Author: osmlab   File: osm_tags.py    License: BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
def csv_to_json(mapping_name, csv_file):
    reader = csv.reader(csv_file)
    reader.next()  # skip header row
    mappings[mapping_name] = {}
    for row in reader:
        # print row
        if row[1] != '' and row[2] != '':
            mappings[mapping_name][row[0]] = {
                'key1': row[1],
                'val1': row[2]
            }
            if len(row) > 4:
                if row[3] != '' and row[4] != '':
                    mappings[mapping_name][row[0]]['key2'] = row[3];
                    mappings[mapping_name][row[0]]['val2'] = row[4]; 
            if len(row) > 6:
                if row[5] != '' and row[6] != '':
                    mappings[mapping_name][row[0]]['key3'] = row[5];
                    mappings[mapping_name][row[0]]['val3'] = row[6]; 
    # print mappings 
Example 12
Project: aws-ops-automator   Author: awslabs   File: report_output_writer.py    License: Apache License 2.0 6 votes vote down vote up
def csv_to_dict_list(s):

    if s is None:
        return None

    result = []
    cols = None
    try:
        reader = csv.reader(io.StringIO(s))
        cols = next(reader)
        row = next(reader)

        while True:
            result.append({cols[i]: row[i] for i in list(range(0, len(cols)))})
            row = next(reader)

    except StopIteration:
        if cols is None:
            return None
        else:
            return result


# noinspection PyMethodMayBeStatic 
Example 13
Project: ernest   Author: amplab   File: predictor.py    License: Apache License 2.0 6 votes vote down vote up
def __init__(self, training_data_in=[], data_file=None):
    ''' 
        Initiliaze the Predictor with some training data
        The training data should be a list of [mcs, input_fraction, time]
    '''
    self.training_data = []
    self.training_data.extend(training_data_in)
    if data_file:
      with open(data_file, 'rb') as csvfile:
        reader = csv.reader(csvfile, delimiter=' ')
        for row in reader:
          if row[0][0] != '#':
            parts = row[0].split(',')
            mc = int(parts[0])
            scale = float(parts[1])
            time = float(parts[2])
            self.training_data.append([mc, scale, time]) 
Example 14
Project: xalpha   Author: refraction-ray   File: info.py    License: MIT License 6 votes vote down vote up
def _basic_init(self):
        raw = rget(self._url)
        cr = csv.reader(raw.text.splitlines(), delimiter=",")
        my_list = list(cr)
        factor = float(my_list[-1][3])
        dd = {
            "date": [
                dt.datetime.strptime(my_list[i + 1][0], "%Y-%m-%d")
                for i in range(len(my_list) - 1)
            ],
            "netvalue": [
                float(my_list[i + 1][3]) / factor for i in range(len(my_list) - 1)
            ],
            "totvalue": [float(my_list[i + 1][3]) for i in range(len(my_list) - 1)],
            "comment": [0 for _ in range(len(my_list) - 1)],
        }
        index = pd.DataFrame(data=dd)
        index = index.iloc[::-1]
        index = index.reset_index(drop=True)
        self.price = index[index["date"].isin(opendate)]
        self.price = self.price[self.price["date"] <= yesterdaydash()]
        self.name = my_list[-1][2] 
Example 15
Project: gazetteer   Author: LibraryOfCongress   File: imports.py    License: MIT License 6 votes vote down vote up
def import_gazetteer(f, limit):
    t = csv.reader(f, delimiter="\t")
    i = 0
    for row in t:
        ft = Feature()
        if Feature.objects.filter(url=row[0]).count() > 0:
            print "duplicate row " + row[0]
        else:
            ft.url = row[0]
            ft.preferred_name = row[1]

            try:
                fcode = FeatureType.objects.get(code=row[2])
            except:
                fcode = None

            ft.feature_type = fcode
            ft.admin1 = row[4]
            ft.admin2 = row[5]
            ft.geometry = Point(float(row[7]), float(row[6]))
            ft.save()
            print "saved " + ft.preferred_name
            i += 1
            if i > limit:
                break 
Example 16
Project: gazetteer   Author: LibraryOfCongress   File: parse_geoplanet.py    License: MIT License 6 votes vote down vote up
def parse_geoplanet_places_csv(csv_file):
    csv_reader = csv.reader(open(csv_file, 'rb'), dialect='excel-tab', quoting=csv.QUOTE_NONE)
    for row in csv_reader:
            out_line = ['P', row[0], row[1], row[6], row[7], row[8], row[10], row[18]+" 00:00:00+00", "POINT("+row[5]+" "+row[4]+")" ] 
            print "\t".join(out_line)
            
    return csv_file


 #* WOE_ID                      0- primary "place" key
  #* ISO                         1- ISO 3166-1 country/territory code
  #* State                       2- WOEID of admin state
  #* County                      3- WOEID of admin county
  #* Local_Admin                 4- WOEID of local admin
  #* Country                     5- WOEID of country
  #* Continent                   6- WOEID of continent 
Example 17
Project: ICDAR-2019-SROIE   Author: zzzDavid   File: prepare_dataset.py    License: MIT License 6 votes vote down vote up
def get_data():
    filenames = [os.path.splitext(f)[0] for f in glob.glob("original/*.jpg")]
    jpg_files = [s + ".jpg" for s in filenames]
    txt_files = [s + ".txt" for s in filenames]

    for file in txt_files:
        boxes = []
        with open(file, "r", encoding="utf-8", newline="") as lines:
            for line in csv.reader(lines):
                boxes.append([line[0], line[1], line[6], line[7]])
        with open('mlt/label/' + file.split('/')[1], "w+") as labelFile:
            wr = csv.writer(labelFile)
            wr.writerows(boxes)

    for jpg in jpg_files:
        shutil.copy(jpg, 'mlt/image/') 
Example 18
Project: ICDAR-2019-SROIE   Author: zzzDavid   File: main.py    License: MIT License 6 votes vote down vote up
def process_txt():
    filenames = [os.path.splitext(f)[0] for f in glob.glob("test_result/*.txt")]
    old_files = [s + ".txt" for s in filenames]
    for old_file in old_files:
        new = []
        with open(old_file, "r") as old:
            for line in csv.reader(old):
                if not line:
                    continue
                if not line[0]:
                    continue
                if line[0][0] == ' ' or line[0][-1] == ' ':
                    line[0] = line[0].strip()
                if ' ' in line[0]:
                    line = line[0].split(' ')
                new.append(line)
        with open('task2_result/' + old_file.split('/')[1], "w+") as newfile:
            wr = csv.writer(newfile, delimiter = '\n')
            new = [[s[0].upper()] for s in new]
            wr.writerows(new) 
Example 19
Project: ICDAR-2019-SROIE   Author: zzzDavid   File: main.py    License: MIT License 6 votes vote down vote up
def for_task3():
    filenames = [os.path.splitext(f)[0] for f in glob.glob("boundingbox/*.txt")]
    box_files = [s + ".txt" for s in filenames]
    for boxfile in box_files:
        box = []
        with open(boxfile,'r') as boxes:
            for line in csv.reader(boxes):
                box.append([int(string, 10) for string in line[0:8]])
        words = []
        with open('test_result/'+ boxfile.split('/')[1], 'r') as prediction:
            for line in csv.reader(prediction):
                words.append(line)
        words = [s if len(s)!=0 else [' '] for s in words]
        new = []
        for line in zip(box,words):
            a,b = line
            new.append(a+b)
        with open('for_task3/'+ boxfile.split('/')[1], 'w+') as newfile:
            csv_out = csv.writer(newfile)
            for line in new:
                csv_out.writerow(line) 
Example 20
Project: ICDAR-2019-SROIE   Author: zzzDavid   File: main.py    License: MIT License 6 votes vote down vote up
def draw():
    filenames = [os.path.splitext(f)[0] for f in glob.glob("for_task3/*.txt")]
    txt_files = [s + ".txt" for s in filenames]
    for txt in txt_files:
        image = cv2.imread('test_original/'+ txt.split('/')[1].split('.')[0]+'.jpg', cv2.IMREAD_COLOR)
        with open(txt, 'r') as txt_file:
            for line in csv.reader(txt_file):
                box = [int(string, 10) for string in line[0:8]]
                if len(line) < 9:
                    print(txt)
                cv2.rectangle(image, (box[0], box[1]), (box[4], box[5]), (0,255,0), 2)
                font = cv2.FONT_HERSHEY_SIMPLEX
                cv2.putText(image, line[8].upper(), (box[0],box[1]), font, 0.5, (0, 0, 255), 1, cv2.LINE_AA)
        cv2.imwrite('task2_result_draw/'+ txt.split('/')[1].split('.')[0]+'.jpg', image) 
Example 21
Project: pointnet-registration-framework   Author: vinits5   File: plot_threshold_vs_success.py    License: MIT License 6 votes vote down vote up
def read_csv(folder_name):
	data = []
	# Each folder having results contain test.csv file with all the log.
	# Read all data from the csv file.
	with open(os.path.join(folder_name, 'test.csv')) as csvfile:
		csvreader = csv.reader(csvfile)
		for row in csvreader:
			row = [float(x) for x in row]
			data.append(row)

	rot_err, trans_err = [], []

	# Log stored is as per following sequence in csv files:
	# Sr. No. [0], time taken [1], number of iterations [2], translation error [3], rotation error [4].
	if folder_name[5:9]=='PNLK':
		for data_i in data:
			rot_err.append(data_i[2])
			trans_err.append(data_i[1])
	else:	
		for data_i in data:
			rot_err.append(data_i[4])
			trans_err.append(data_i[3])
	return rot_err, trans_err

# It will count the total number of test cases having rotation error below certain threshold. 
Example 22
Project: BERT-Classification-Tutorial   Author: Socialbird-AILab   File: run_classifier.py    License: Apache License 2.0 5 votes vote down vote up
def _read_tsv(cls, input_file, quotechar=None):
        """Reads a tab separated value file."""
        with tf.gfile.Open(input_file, "r") as f:
            reader = csv.reader(f, delimiter="\t", quotechar=quotechar)
            lines = []
            for line in reader:
                lines.append(line)
            return lines 
Example 23
Project: decisiontrees   Author: jayelm   File: dtree.py    License: MIT License 5 votes vote down vote up
def parse_csv(self, dependent_index=-1):
        """
        Set the object's attributes and data, where attributes is a list of
        attributes and data is an array of row dictionaries keyed by attribute.

        Also sets the dependent variable, which defaults to the last one. An
        option to change the position of this dependent variable has not yet
        been implemented.

        Args:
            dependent_index: the index to be specified as the dependent
                variable (default -1).
        Raises:
            NotImplementedError: If dependent_index is specified, since I
                haven't implemented that yet.

        """
        if dependent_index != -1:
            raise NotImplementedError

        reader = csv.reader(self.training_file)
        attributes = reader.next()
        data = []
        for row in reader:
            row = dict(zip(attributes, row))
            data.append(row)
        self.training_file.close()

        self.dependent = attributes[dependent_index]
        self.attributes = [a for a in attributes if a != self.dependent]
        self.all_attributes = attributes
        self.data = data 
Example 24
Project: incubator-spot   Author: apache   File: flow.py    License: Apache License 2.0 5 votes vote down vote up
def add_geospatial_info(iploc,inbound,outbound,twoway):
    iplist = ''
    if os.path.isfile(iploc):
        iplist = np.loadtxt(iploc,dtype=np.uint32,delimiter=',',usecols={0},\
        converters={0: lambda s: np.uint32(s.replace('"',''))})
    else:
        print "No iploc.csv file was found, Map View map won't be created"


    # get geospatial info, only when iplocation file is available
    if iplist != '':
        for srcip in outbound:
            reader = csv.reader([linecache.getline(\
            iploc, bisect.bisect(iplist,outbound[srcip]['ip_int'])).replace('\n','')])

            outbound[srcip]['geo'] = reader.next()
            reader = csv.reader([linecache.getline(\
            iploc, bisect.bisect(iplist,outbound[srcip]['dst_ip_int'])).replace('\n','')])
            outbound[srcip]['geo_dst'] = reader.next()

        for dstip in twoway:
            reader = csv.reader([linecache.getline(\
            iploc,bisect.bisect(iplist,twoway[dstip]['ip_int'])).replace('\n','')])
            twoway[dstip]['geo'] = reader.next()

        for srcip in inbound:
            reader = csv.reader([linecache.getline(\
            iploc, bisect.bisect(iplist,inbound[srcip]['ip_int'])).replace('\n','')])

            inbound[srcip]['geo'] = reader.next()
            reader = csv.reader([linecache.getline(\
            iploc, bisect.bisect(iplist,inbound[srcip]['src_ip_int'])).replace('\n','')])
            inbound[srcip]['geo_src'] = reader.next()

    return inbound,outbound,twoway 
Example 25
Project: incubator-spot   Author: apache   File: utils.py    License: Apache License 2.0 5 votes vote down vote up
def read_results(cls, file, limit, delimiter=','):

        # read csv results.
        result_rows = []
        with open(file, 'rb') as results_file:
            csv_reader = csv.reader(results_file, delimiter=delimiter)
            for i in range(0, int(limit)):
                try:
                    row = csv_reader.next()
                except StopIteration:
                    return result_rows
                result_rows.append(row)
        return result_rows 
Example 26
Project: incubator-spot   Author: apache   File: network_context.py    License: Apache License 2.0 5 votes vote down vote up
def _init_dicts(self):    
		if os.path.isfile(self._nc_file_path):    			
			with open(self._nc_file_path) as nc_file:
				csv_reader = csv.reader(nc_file)
				csv_reader.next()
				nc_rows = list(csv_reader)
				if len(nc_rows) > 0:
					self._nc_dict = dict([(x[0], x[1]) if len(x) > 2 else ("-", "-") for x in nc_rows]) 
Example 27
Project: incubator-spot   Author: apache   File: iana_transform.py    License: Apache License 2.0 5 votes vote down vote up
def _init_dicts(self):
        if os.path.isfile(self._qclass_file_path):
            with open(self._qclass_file_path, 'rb') as qclass_file:
                csv_reader = csv.reader(qclass_file)
                csv_reader.next()
                qclass_rows = list(csv_reader)
                d1 = dict([(x[0],x[2]) for x in qclass_rows])
                d2 = dict([(x[1],x[2]) for x in qclass_rows])
                self._qclass_dict.update(d1)
                self._qclass_dict.update(d2)
        if os.path.isfile(self._qtype_file_path):
            with open(self._qtype_file_path, 'rb') as qtype_file:
                csv_reader = csv.reader(qtype_file)
                csv_reader.next()
                qtype_rows = list(csv_reader)
                self._qtype_dict = dict([(x[1],x[0]) for x in qtype_rows])
        if os.path.isfile(self._rcode_file_path):
            with open(self._rcode_file_path) as rcode_file:
                csv_reader = csv.reader(rcode_file)
                csv_reader.next()
                rcode_rows = list(csv_reader)
                self._rcode_dict = dict([(x[0],x[1]) for x in rcode_rows])
        if os.path.isfile(self._http_rcode_file_path):
            with open(self._http_rcode_file_path) as http_resp_code:
                csv_reader = csv.reader(http_resp_code)
                csv_reader.next()
                presp_rows = list(csv_reader)
                self._http_rcode_dict = dict([(x[0],x[1]) for x in presp_rows]) 
Example 28
def load_defense_output(filename):
  """Loads output of defense from given file."""
  result = {}
  with open(filename) as f:
    for row in csv.reader(f):
      try:
        image_filename = row[0]
        if not image_filename.endswith('.png'):
          image_filename += '.png'
        label = int(row[1])
      except (IndexError, ValueError):
        continue
      result[image_filename] = label
  return result 
Example 29
def load_target_class(input_dir):
  """Loads target classes."""
  with tf.gfile.Open(os.path.join(input_dir, 'target_class.csv')) as f:
    return {row[0]: int(row[1]) for row in csv.reader(f) if len(row) >= 2} 
Example 30
def load_target_class(input_dir):
  """Loads target classes."""
  with tf.gfile.Open(os.path.join(input_dir, 'target_class.csv')) as f:
    return {row[0]: int(row[1]) for row in csv.reader(f) if len(row) >= 2}