Python json.items() Examples

The following are code examples for showing how to use json.items(). They are from open source Python projects. You can vote up the examples you like or vote down the ones you don't like.

Example 1
Project: razzy-spinner   Author: rafasashi   File: common.py    GNU General Public License v3.0 6 votes vote down vote up
def _get_entity_recursive(json, entity):
    if not json:
        return None
    elif isinstance(json, dict):
        for key, value in json.items():
            if key == entity:
                return value
            # 'entities' and 'extended_entities' are wrappers in Twitter json
            # structure that contain other Twitter objects. See:
            # https://dev.twitter.com/overview/api/entities-in-twitter-objects

            if key == 'entities' or key == 'extended_entities':
                candidate = _get_entity_recursive(value, entity)
                if candidate is not None:
                    return candidate
        return None
    elif isinstance(json, list):
        for item in json:
            candidate = _get_entity_recursive(item, entity)
            if candidate is not None:
                return candidate
        return None
    else:
        return None 
Example 2
Project: OpenBottle   Author: xiaozhuchacha   File: common.py    MIT License 6 votes vote down vote up
def _get_entity_recursive(json, entity):
    if not json:
        return None
    elif isinstance(json, dict):
        for key, value in json.items():
            if key == entity:
                return value
            # 'entities' and 'extended_entities' are wrappers in Twitter json
            # structure that contain other Twitter objects. See:
            # https://dev.twitter.com/overview/api/entities-in-twitter-objects

            if key == 'entities' or key == 'extended_entities':
                candidate = _get_entity_recursive(value, entity)
                if candidate is not None:
                    return candidate
        return None
    elif isinstance(json, list):
        for item in json:
            candidate = _get_entity_recursive(item, entity)
            if candidate is not None:
                return candidate
        return None
    else:
        return None 
Example 3
Project: OpenBottle   Author: xiaozhuchacha   File: common.py    MIT License 6 votes vote down vote up
def _get_entity_recursive(json, entity):
    if not json:
        return None
    elif isinstance(json, dict):
        for key, value in json.items():
            if key == entity:
                return value
            # 'entities' and 'extended_entities' are wrappers in Twitter json
            # structure that contain other Twitter objects. See:
            # https://dev.twitter.com/overview/api/entities-in-twitter-objects

            if key == 'entities' or key == 'extended_entities':
                candidate = _get_entity_recursive(value, entity)
                if candidate is not None:
                    return candidate
        return None
    elif isinstance(json, list):
        for item in json:
            candidate = _get_entity_recursive(item, entity)
            if candidate is not None:
                return candidate
        return None
    else:
        return None 
Example 4
Project: Health-Checker   Author: KriAga   File: common.py    MIT License 6 votes vote down vote up
def _get_entity_recursive(json, entity):
    if not json:
        return None
    elif isinstance(json, dict):
        for key, value in json.items():
            if key == entity:
                return value
            # 'entities' and 'extended_entities' are wrappers in Twitter json
            # structure that contain other Twitter objects. See:
            # https://dev.twitter.com/overview/api/entities-in-twitter-objects

            if key == 'entities' or key == 'extended_entities':
                candidate = _get_entity_recursive(value, entity)
                if candidate is not None:
                    return candidate
        return None
    elif isinstance(json, list):
        for item in json:
            candidate = _get_entity_recursive(item, entity)
            if candidate is not None:
                return candidate
        return None
    else:
        return None 
Example 5
Project: deca   Author: kk49   File: vfs_db.py    MIT License 6 votes vote down vote up
def dump_status(self):
        self.logger.log('hashes: {}, mappings missing: {}, mappings present {}, mapping conflict {}'.format(
            len(self.hash_present),
            len(self.hash_map_missing),
            len(self.hash_map_present),
            len(self.hash_map_conflict)))

        for k, vs in self.adf_missing_types.items():
            for v in vs:
                vps = self.map_hash_to_vpath.get(v, {})
                for vp in vps:
                    self.logger.log('Missing Type {:08x} in {:08X} {}'.format(k, v, vp))

        for vid in self.hash_map_conflict:
            for vps in self.map_hash_to_vpath[vid]:
                for vp in vps:
                    self.logger.log('CONFLICT: {:08X} {}'.format(vid, vp)) 
Example 6
Project: deca   Author: kk49   File: vfs_db.py    MIT License 6 votes vote down vote up
def find_vpath_json(self, vpath_map):
        self.logger.log('PROCESS JSONs: look for hashable strings in json files')
        json_done = set()
        for idx in range(len(self.table_vfsnode)):
            node = self.table_vfsnode[idx]
            if node.is_valid() and node.ftype == FTYPE_TXT and node.vhash not in json_done:
                with self.file_obj_from(node) as f:
                    buffer = f.read(node.size_u)

                json = load_json(buffer)
                if json is not None:
                    json_done.add(node.vhash)

                # Parse {"0":[]. "1":[]}
                if isinstance(json, dict) and '0' in json and '1' in json:
                    for k, v in json.items():
                        for l in v:
                            vpath_map.propose(l, [FTYPE_TXT, node])
        self.logger.log('PROCESS JSONs: Total JSON files {}'.format(len(json_done))) 
Example 7
Project: deca   Author: kk49   File: vfs_db.py    MIT License 6 votes vote down vote up
def find_vpath_by_assoc(self, vpath_map):
        self.logger.log('STRINGS BY FILE NAME ASSOCIATION: epe/ee, blo/bl/nl/fl/nl.mdic/fl.mdic, mesh*/model*, avtx/atx?]')
        pair_exts = self.game_info.file_assoc()

        assoc_strings = {}
        for k, v in vpath_map.nodes.items():
            file_ext = os.path.splitext(k.decode('utf-8'))
            if len(file_ext[0]) > 0 and len(file_ext[1]) > 0:
                file = file_ext[0]
                ext = file_ext[1]
                for pe in pair_exts:
                    if ext in pe:
                        for pk, pv in pe.items():
                            assoc_strings[file + pk] = pv

        for k, v in assoc_strings.items():
            fn = k
            fn = fn.encode('ascii')
            fh = hash_little(fn)
            if fh in self.hash_present:
                vpath_map.propose(fn, ['ASSOC', None], possible_ftypes=v)

        self.logger.log('STRINGS BY FILE NAME ASSOCIATION: Found {}'.format(len(assoc_strings))) 
Example 8
Project: honours_project   Author: JFriel   File: common.py    GNU General Public License v3.0 6 votes vote down vote up
def _get_entity_recursive(json, entity):
    if not json:
        return None
    elif isinstance(json, dict):
        for key, value in json.items():
            if key == entity:
                return value
            # 'entities' and 'extended_entities' are wrappers in Twitter json
            # structure that contain other Twitter objects. See:
            # https://dev.twitter.com/overview/api/entities-in-twitter-objects

            if key == 'entities' or key == 'extended_entities':
                candidate = _get_entity_recursive(value, entity)
                if candidate is not None:
                    return candidate
        return None
    elif isinstance(json, list):
        for item in json:
            candidate = _get_entity_recursive(item, entity)
            if candidate is not None:
                return candidate
        return None
    else:
        return None 
Example 9
Project: honours_project   Author: JFriel   File: common.py    GNU General Public License v3.0 6 votes vote down vote up
def _get_entity_recursive(json, entity):
    if not json:
        return None
    elif isinstance(json, dict):
        for key, value in json.items():
            if key == entity:
                return value
            # 'entities' and 'extended_entities' are wrappers in Twitter json
            # structure that contain other Twitter objects. See:
            # https://dev.twitter.com/overview/api/entities-in-twitter-objects

            if key == 'entities' or key == 'extended_entities':
                candidate = _get_entity_recursive(value, entity)
                if candidate is not None:
                    return candidate
        return None
    elif isinstance(json, list):
        for item in json:
            candidate = _get_entity_recursive(item, entity)
            if candidate is not None:
                return candidate
        return None
    else:
        return None 
Example 10
Project: aop-helpFinder   Author: jecarvaill   File: common.py    GNU General Public License v3.0 6 votes vote down vote up
def _get_entity_recursive(json, entity):
    if not json:
        return None
    elif isinstance(json, dict):
        for key, value in json.items():
            if key == entity:
                return value
            # 'entities' and 'extended_entities' are wrappers in Twitter json
            # structure that contain other Twitter objects. See:
            # https://dev.twitter.com/overview/api/entities-in-twitter-objects

            if key == 'entities' or key == 'extended_entities':
                candidate = _get_entity_recursive(value, entity)
                if candidate is not None:
                    return candidate
        return None
    elif isinstance(json, list):
        for item in json:
            candidate = _get_entity_recursive(item, entity)
            if candidate is not None:
                return candidate
        return None
    else:
        return None 
Example 11
Project: rules_web   Author: quittle   File: generate_variables.py    Apache License 2.0 6 votes vote down vote up
def stringify_scss(json):
    if isinstance(json, list):
        return ('(' +
                ', '.join(stringify_scss(value) for value in json) +
                ')')
    elif isinstance(json, dict):
        return ('(' +
                ', '.join(stringify_scss(key) + ': ' + stringify_scss(value) for key, value in json.items()) +
                ')')
    elif isinstance(json, bool):
        return 'true' if json else 'false'
    elif isinstance(json, str):
        return '\'' + json + '\''
    else:
        return str(json)


# noinspection PyInterpreter 
Example 12
Project: serverless-chatbots-workshop   Author: datteswararao   File: common.py    Apache License 2.0 6 votes vote down vote up
def _get_entity_recursive(json, entity):
    if not json:
        return None
    elif isinstance(json, dict):
        for key, value in json.items():
            if key == entity:
                return value
            # 'entities' and 'extended_entities' are wrappers in Twitter json
            # structure that contain other Twitter objects. See:
            # https://dev.twitter.com/overview/api/entities-in-twitter-objects

            if key == 'entities' or key == 'extended_entities':
                candidate = _get_entity_recursive(value, entity)
                if candidate is not None:
                    return candidate
        return None
    elif isinstance(json, list):
        for item in json:
            candidate = _get_entity_recursive(item, entity)
            if candidate is not None:
                return candidate
        return None
    else:
        return None 
Example 13
Project: serverless-chatbots-workshop   Author: datteswararao   File: common.py    Apache License 2.0 6 votes vote down vote up
def _get_entity_recursive(json, entity):
    if not json:
        return None
    elif isinstance(json, dict):
        for key, value in json.items():
            if key == entity:
                return value
            # 'entities' and 'extended_entities' are wrappers in Twitter json
            # structure that contain other Twitter objects. See:
            # https://dev.twitter.com/overview/api/entities-in-twitter-objects

            if key == 'entities' or key == 'extended_entities':
                candidate = _get_entity_recursive(value, entity)
                if candidate is not None:
                    return candidate
        return None
    elif isinstance(json, list):
        for item in json:
            candidate = _get_entity_recursive(item, entity)
            if candidate is not None:
                return candidate
        return None
    else:
        return None 
Example 14
Project: razzy-spinner   Author: rafasashi   File: common.py    GNU General Public License v3.0 5 votes vote down vote up
def _write_to_file(object_fields, items, entity_fields, writer):
    if not items:
        # it could be that the entity is just not present for the tweet
        # e.g. tweet hashtag is always present, even as [], however
        # tweet media may not be present
        return
    if isinstance(items, dict):
        # this happens e.g. for "place" of a tweet
        row = object_fields
        # there might be composed keys in de list of required fields
        entity_field_values = [x for x in entity_fields if not _is_composed_key(x)]
        entity_field_composed = [x for x in entity_fields if _is_composed_key(x)]
        for field in entity_field_values:
            value = items[field]
            if isinstance(value, list):
                row += value
            else:
                row += [value]
        # now check required dictionaries
        for d in entity_field_composed:
            kd, vd = _get_key_value_composed(d)
            json_dict = items[kd]
            if not isinstance(json_dict, dict):
                raise RuntimeError("""Key {0} does not contain a dictionary
                in the json file""".format(kd))
            row += [json_dict[vd]]
        writer.writerow(row)
        return
    # in general it is a list
    for item in items:
        row = object_fields + extract_fields(item, entity_fields)
        writer.writerow(row) 
Example 15
Project: OpenBottle   Author: xiaozhuchacha   File: common.py    MIT License 5 votes vote down vote up
def _write_to_file(object_fields, items, entity_fields, writer):
    if not items:
        # it could be that the entity is just not present for the tweet
        # e.g. tweet hashtag is always present, even as [], however
        # tweet media may not be present
        return
    if isinstance(items, dict):
        # this happens e.g. for "place" of a tweet
        row = object_fields
        # there might be composed keys in de list of required fields
        entity_field_values = [x for x in entity_fields if not _is_composed_key(x)]
        entity_field_composed = [x for x in entity_fields if _is_composed_key(x)]
        for field in entity_field_values:
            value = items[field]
            if isinstance(value, list):
                row += value
            else:
                row += [value]
        # now check required dictionaries
        for d in entity_field_composed:
            kd, vd = _get_key_value_composed(d)
            json_dict = items[kd]
            if not isinstance(json_dict, dict):
                raise RuntimeError("""Key {0} does not contain a dictionary
                in the json file""".format(kd))
            row += [json_dict[vd]]
        writer.writerow(row)
        return
    # in general it is a list
    for item in items:
        row = object_fields + extract_fields(item, entity_fields)
        writer.writerow(row) 
Example 16
Project: Health-Checker   Author: KriAga   File: common.py    MIT License 5 votes vote down vote up
def _write_to_file(object_fields, items, entity_fields, writer):
    if not items:
        # it could be that the entity is just not present for the tweet
        # e.g. tweet hashtag is always present, even as [], however
        # tweet media may not be present
        return
    if isinstance(items, dict):
        # this happens e.g. for "place" of a tweet
        row = object_fields
        # there might be composed keys in de list of required fields
        entity_field_values = [x for x in entity_fields if not _is_composed_key(x)]
        entity_field_composed = [x for x in entity_fields if _is_composed_key(x)]
        for field in entity_field_values:
            value = items[field]
            if isinstance(value, list):
                row += value
            else:
                row += [value]
        # now check required dictionaries
        for d in entity_field_composed:
            kd, vd = _get_key_value_composed(d)
            json_dict = items[kd]
            if not isinstance(json_dict, dict):
                raise RuntimeError("""Key {0} does not contain a dictionary
                in the json file""".format(kd))
            row += [json_dict[vd]]
        writer.writerow(row)
        return
    # in general it is a list
    for item in items:
        row = object_fields + extract_fields(item, entity_fields)
        writer.writerow(row) 
Example 17
Project: lite-internal-frontend   Author: uktrade   File: context_variables.py    MIT License 5 votes vote down vote up
def _get_valid_class_types(variables):
    valid_variables = {}
    for key, data in variables.items():
        if data["valid"] == "True":
            valid_variables[key] = data
    return valid_variables 
Example 18
Project: lite-internal-frontend   Author: uktrade   File: context_variables.py    MIT License 5 votes vote down vote up
def _get_build_order(json):
    keys = []
    for key, value in json.items():
        if "extends" in value:
            keys.append(key)
    return keys 
Example 19
Project: lite-internal-frontend   Author: uktrade   File: context_variables.py    MIT License 5 votes vote down vote up
def _clean_dict(dictionary):
    dictionary = _remove_hidden_variables(dictionary)
    if "variables" in dictionary:
        for var in dictionary["variables"]:
            dictionary[var] = ""
        del dictionary["variables"]
    for key, value in dictionary.items():
        if isinstance(value, dict):
            dictionary[key] = _clean_dict(value)
    return dictionary 
Example 20
Project: lite-internal-frontend   Author: uktrade   File: context_variables.py    MIT License 5 votes vote down vote up
def get_context_variables():
    with open(JSON_PATH, "r") as f:
        variables = json.load(f)

    # Populate classes with any base class variables
    variables = _add_base_class_variables(variables)

    # Extract all object types we want to give to the user (using 'valid' key)
    variables = _get_valid_class_types(variables)

    # Remove all hidden variables and format as fully nested dict
    for key, value in variables.items():
        variables[key] = _clean_dict(value)

    return variables 
Example 21
Project: lite-internal-frontend   Author: uktrade   File: context_variables.py    MIT License 5 votes vote down vote up
def flatten_dict(dictionary, path):
    if path != "":
        path += "."
    for key, value in dictionary.items():
        if isinstance(value, dict):
            flatten_dict(value, path + key)
        else:
            for variable in dictionary:
                flattened_context_variables.append(path + variable) 
Example 22
Project: honours_project   Author: JFriel   File: common.py    GNU General Public License v3.0 5 votes vote down vote up
def _write_to_file(object_fields, items, entity_fields, writer):
    if not items:
        # it could be that the entity is just not present for the tweet
        # e.g. tweet hashtag is always present, even as [], however
        # tweet media may not be present
        return
    if isinstance(items, dict):
        # this happens e.g. for "place" of a tweet
        row = object_fields
        # there might be composed keys in de list of required fields
        entity_field_values = [x for x in entity_fields if not _is_composed_key(x)]
        entity_field_composed = [x for x in entity_fields if _is_composed_key(x)]
        for field in entity_field_values:
            value = items[field]
            if isinstance(value, list):
                row += value
            else:
                row += [value]
        # now check required dictionaries
        for d in entity_field_composed:
            kd, vd = _get_key_value_composed(d)
            json_dict = items[kd]
            if not isinstance(json_dict, dict):
                raise RuntimeError("""Key {0} does not contain a dictionary
                in the json file""".format(kd))
            row += [json_dict[vd]]
        writer.writerow(row)
        return
    # in general it is a list
    for item in items:
        row = object_fields + extract_fields(item, entity_fields)
        writer.writerow(row) 
Example 23
Project: icdc-docker   Author: CBIIT   File: client.py    Apache License 2.0 5 votes vote down vote up
def json_dumps(data):
    return json.dumps({k: v for (k, v) in data.items() if v is not None}) 
Example 24
Project: icdc-docker   Author: CBIIT   File: client.py    Apache License 2.0 5 votes vote down vote up
def get_with_params(self, params=None):
        """
        Return a document object corresponding to the supplied parameters, such
        as ``{'hashes': {'md5': '...'}, 'size': '...', 'metadata': {'file_state': '...'}}``.
        """
        # need to include all the hashes in the request
        # index client like signpost or indexd will need to handle the
        # query param `'hash': 'hash_type:hash'`
        params_copy = copy.deepcopy(params) or {}
        if 'hashes' in params_copy:
            params_copy['hash'] = params_copy.pop('hashes')
        reformatted_params = dict()
        for param in ['hash', 'metadata']:
            if param in params_copy:
                reformatted_params[param] = []
                for k, v in params_copy[param].items():
                    reformatted_params[param].append(str(k) + ':' + str(v))
                del params_copy[param]
        reformatted_params.update(params_copy)
        reformatted_params['limit'] = 1

        try:
            response = self._get('index', params=reformatted_params)
        except requests.HTTPError as e:
            if e.response.status_code == 404:
                return None
            else:
                raise e
        if not response.json()['records']:
            return None
        json = response.json()['records'][0]
        did = json['did']
        return Document(self, did, json=json) 
Example 25
Project: icdc-docker   Author: CBIIT   File: client.py    Apache License 2.0 5 votes vote down vote up
def list_versions(self, did):
        # type: (str) -> list[Document]
        versions_dict = self._get("index", did, "versions").json()  # type: dict
        versions = []

        for _, version in versions_dict.items():
            versions.append(Document(self, version["did"], version))
        return versions 
Example 26
Project: icdc-docker   Author: CBIIT   File: client.py    Apache License 2.0 5 votes vote down vote up
def _load(self, json=None):
        """ Load the document contents from the server or from the provided dictionary """
        self._check_deleted()
        json = json or self.client._get("index", self.did).json()
        # set attributes to current Document
        for k,v in json.items():
            self.__dict__[k] = v
        self._attrs = json.keys()
        self._fetched = True 
Example 27
Project: aop-helpFinder   Author: jecarvaill   File: common.py    GNU General Public License v3.0 5 votes vote down vote up
def _write_to_file(object_fields, items, entity_fields, writer):
    if not items:
        # it could be that the entity is just not present for the tweet
        # e.g. tweet hashtag is always present, even as [], however
        # tweet media may not be present
        return
    if isinstance(items, dict):
        # this happens e.g. for "place" of a tweet
        row = object_fields
        # there might be composed keys in de list of required fields
        entity_field_values = [x for x in entity_fields if not _is_composed_key(x)]
        entity_field_composed = [x for x in entity_fields if _is_composed_key(x)]
        for field in entity_field_values:
            value = items[field]
            if isinstance(value, list):
                row += value
            else:
                row += [value]
        # now check required dictionaries
        for d in entity_field_composed:
            kd, vd = _get_key_value_composed(d)
            json_dict = items[kd]
            if not isinstance(json_dict, dict):
                raise RuntimeError("""Key {0} does not contain a dictionary
                in the json file""".format(kd))
            row += [json_dict[vd]]
        writer.writerow(row)
        return
    # in general it is a list
    for item in items:
        row = object_fields + extract_fields(item, entity_fields)
        writer.writerow(row) 
Example 28
Project: rules_web   Author: quittle   File: generate_variables.py    Apache License 2.0 5 votes vote down vote up
def stringify_css(json):
    if isinstance(json, list):
        return ('[' +
                ', '.join(stringify_css(value) for value in json) +
                ']')
    elif isinstance(json, dict):
        return ('{' +
                ', '.join(key + ': ' + stringify_css(value) for key, value in json.items()) +
                '}')
    elif isinstance(json, bool):
        return 'true' if json else 'false'
    else:
        return str(json) 
Example 29
Project: rules_web   Author: quittle   File: generate_variables.py    Apache License 2.0 5 votes vote down vote up
def write(self):
        if self.uses_iteration():
            for key, value in self._config.items():
                self._file.write(self.write_iteration(key, value))
        else:
            self._file.write(self.write_whole(config)) 
Example 30
Project: serverless-chatbots-workshop   Author: datteswararao   File: common.py    Apache License 2.0 5 votes vote down vote up
def _write_to_file(object_fields, items, entity_fields, writer):
    if not items:
        # it could be that the entity is just not present for the tweet
        # e.g. tweet hashtag is always present, even as [], however
        # tweet media may not be present
        return
    if isinstance(items, dict):
        # this happens e.g. for "place" of a tweet
        row = object_fields
        # there might be composed keys in de list of required fields
        entity_field_values = [x for x in entity_fields if not _is_composed_key(x)]
        entity_field_composed = [x for x in entity_fields if _is_composed_key(x)]
        for field in entity_field_values:
            value = items[field]
            if isinstance(value, list):
                row += value
            else:
                row += [value]
        # now check required dictionaries
        for d in entity_field_composed:
            kd, vd = _get_key_value_composed(d)
            json_dict = items[kd]
            if not isinstance(json_dict, dict):
                raise RuntimeError("""Key {0} does not contain a dictionary
                in the json file""".format(kd))
            row += [json_dict[vd]]
        writer.writerow(row)
        return
    # in general it is a list
    for item in items:
        row = object_fields + extract_fields(item, entity_fields)
        writer.writerow(row) 
Example 31
Project: viewX-vscode   Author: textX   File: cytoscape_helper.py    MIT License 5 votes vote down vote up
def deserialize_json(json):
    """
    A helper method for deserializing json into Cytoscape.js elements.
    :param json: json representation of Cytoscape.js element
    :return: Cytoscape.js element object if json is valid, else json
    """
    class_name = json.pop('__classname__', None)
    if class_name == 'Element':  # type(self).__name__:
        obj = Element.__new__(Element)   # Make instance without calling __init__
        for key, value in json.items():
            setattr(obj, key, value)
            return obj
    else:
        return json 
Example 32
Project: razzy-spinner   Author: rafasashi   File: common.py    GNU General Public License v3.0 4 votes vote down vote up
def json2csv_entities(tweets_file, outfile, main_fields, entity_type, entity_fields,
                      encoding='utf8', errors='replace', gzip_compress=False):
    """
    Extract selected fields from a file of line-separated JSON tweets and
    write to a file in CSV format.

    This utility function allows a file of full Tweets to be easily converted
    to a CSV file for easier processing of Twitter entities. For example, the
    hashtags or media elements of a tweet can be extracted.

    It returns one line per entity of a Tweet, e.g. if a tweet has two hashtags
    there will be two lines in the output file, one per hashtag

    :param tweets_file: the file-like object containing full Tweets

    :param str outfile: The path of the text file where results should be\
    written

    :param list main_fields: The list of fields to be extracted from the main\
    object, usually the tweet. Useful examples: 'id_str' for the tweetID. See\
    <https://dev.twitter.com/overview/api/tweets> for a full list of fields.
    e. g.: ['id_str'], ['id', 'text', 'favorite_count', 'retweet_count']
    If `entity_type` is expressed with hierarchy, then it is the list of\
    fields of the object that corresponds to the key of the entity_type,\
    (e.g., for entity_type='user.urls', the fields in the main_fields list\
    belong to the user object; for entity_type='place.bounding_box', the\
    files in the main_field list belong to the place object of the tweet).

    :param list entity_type: The name of the entity: 'hashtags', 'media',\
    'urls' and 'user_mentions' for the tweet object. For a user object,\
    this needs to be expressed with a hierarchy: `'user.urls'`. For the\
    bounding box of the Tweet location, use `'place.bounding_box'`.

    :param list entity_fields: The list of fields to be extracted from the\
    entity. E.g. `['text']` (of the Tweet)

    :param error: Behaviour for encoding errors, see\
    https://docs.python.org/3/library/codecs.html#codec-base-classes

    :param gzip_compress: if `True`, ouput files are compressed with gzip
    """

    (writer, outf) = outf_writer_compat(outfile, encoding, errors, gzip_compress)
    header = get_header_field_list(main_fields, entity_type, entity_fields)
    writer.writerow(header)
    for line in tweets_file:
        tweet = json.loads(line)
        if _is_composed_key(entity_type):
            key, value = _get_key_value_composed(entity_type)
            object_json = _get_entity_recursive(tweet, key)
            if not object_json:
                # this can happen in the case of "place"
                continue
            object_fields = extract_fields(object_json, main_fields)
            items = _get_entity_recursive(object_json, value)
            _write_to_file(object_fields, items, entity_fields, writer)
        else:
            tweet_fields = extract_fields(tweet, main_fields)
            items = _get_entity_recursive(tweet, entity_type)
            _write_to_file(tweet_fields, items, entity_fields, writer)
    outf.close() 
Example 33
Project: OpenBottle   Author: xiaozhuchacha   File: common.py    MIT License 4 votes vote down vote up
def json2csv_entities(tweets_file, outfile, main_fields, entity_type, entity_fields,
                      encoding='utf8', errors='replace', gzip_compress=False):
    """
    Extract selected fields from a file of line-separated JSON tweets and
    write to a file in CSV format.

    This utility function allows a file of full Tweets to be easily converted
    to a CSV file for easier processing of Twitter entities. For example, the
    hashtags or media elements of a tweet can be extracted.

    It returns one line per entity of a Tweet, e.g. if a tweet has two hashtags
    there will be two lines in the output file, one per hashtag

    :param tweets_file: the file-like object containing full Tweets

    :param str outfile: The path of the text file where results should be\
    written

    :param list main_fields: The list of fields to be extracted from the main\
    object, usually the tweet. Useful examples: 'id_str' for the tweetID. See\
    <https://dev.twitter.com/overview/api/tweets> for a full list of fields.
    e. g.: ['id_str'], ['id', 'text', 'favorite_count', 'retweet_count']
    If `entity_type` is expressed with hierarchy, then it is the list of\
    fields of the object that corresponds to the key of the entity_type,\
    (e.g., for entity_type='user.urls', the fields in the main_fields list\
    belong to the user object; for entity_type='place.bounding_box', the\
    files in the main_field list belong to the place object of the tweet).

    :param list entity_type: The name of the entity: 'hashtags', 'media',\
    'urls' and 'user_mentions' for the tweet object. For a user object,\
    this needs to be expressed with a hierarchy: `'user.urls'`. For the\
    bounding box of the Tweet location, use `'place.bounding_box'`.

    :param list entity_fields: The list of fields to be extracted from the\
    entity. E.g. `['text']` (of the Tweet)

    :param error: Behaviour for encoding errors, see\
    https://docs.python.org/3/library/codecs.html#codec-base-classes

    :param gzip_compress: if `True`, ouput files are compressed with gzip
    """

    (writer, outf) = outf_writer_compat(outfile, encoding, errors, gzip_compress)
    header = get_header_field_list(main_fields, entity_type, entity_fields)
    writer.writerow(header)
    for line in tweets_file:
        tweet = json.loads(line)
        if _is_composed_key(entity_type):
            key, value = _get_key_value_composed(entity_type)
            object_json = _get_entity_recursive(tweet, key)
            if not object_json:
                # this can happen in the case of "place"
                continue
            object_fields = extract_fields(object_json, main_fields)
            items = _get_entity_recursive(object_json, value)
            _write_to_file(object_fields, items, entity_fields, writer)
        else:
            tweet_fields = extract_fields(tweet, main_fields)
            items = _get_entity_recursive(tweet, entity_type)
            _write_to_file(tweet_fields, items, entity_fields, writer)
    outf.close() 
Example 34
Project: OpenBottle   Author: xiaozhuchacha   File: common.py    MIT License 4 votes vote down vote up
def json2csv_entities(tweets_file, outfile, main_fields, entity_type, entity_fields,
                      encoding='utf8', errors='replace', gzip_compress=False):
    """
    Extract selected fields from a file of line-separated JSON tweets and
    write to a file in CSV format.

    This utility function allows a file of full Tweets to be easily converted
    to a CSV file for easier processing of Twitter entities. For example, the
    hashtags or media elements of a tweet can be extracted.

    It returns one line per entity of a Tweet, e.g. if a tweet has two hashtags
    there will be two lines in the output file, one per hashtag

    :param tweets_file: the file-like object containing full Tweets

    :param str outfile: The path of the text file where results should be\
    written

    :param list main_fields: The list of fields to be extracted from the main\
    object, usually the tweet. Useful examples: 'id_str' for the tweetID. See\
    <https://dev.twitter.com/overview/api/tweets> for a full list of fields.
    e. g.: ['id_str'], ['id', 'text', 'favorite_count', 'retweet_count']
    If `entity_type` is expressed with hierarchy, then it is the list of\
    fields of the object that corresponds to the key of the entity_type,\
    (e.g., for entity_type='user.urls', the fields in the main_fields list\
    belong to the user object; for entity_type='place.bounding_box', the\
    files in the main_field list belong to the place object of the tweet).

    :param list entity_type: The name of the entity: 'hashtags', 'media',\
    'urls' and 'user_mentions' for the tweet object. For a user object,\
    this needs to be expressed with a hierarchy: `'user.urls'`. For the\
    bounding box of the Tweet location, use `'place.bounding_box'`.

    :param list entity_fields: The list of fields to be extracted from the\
    entity. E.g. `['text']` (of the Tweet)

    :param error: Behaviour for encoding errors, see\
    https://docs.python.org/3/library/codecs.html#codec-base-classes

    :param gzip_compress: if `True`, ouput files are compressed with gzip
    """

    (writer, outf) = outf_writer_compat(outfile, encoding, errors, gzip_compress)
    header = get_header_field_list(main_fields, entity_type, entity_fields)
    writer.writerow(header)
    for line in tweets_file:
        tweet = json.loads(line)
        if _is_composed_key(entity_type):
            key, value = _get_key_value_composed(entity_type)
            object_json = _get_entity_recursive(tweet, key)
            if not object_json:
                # this can happen in the case of "place"
                continue
            object_fields = extract_fields(object_json, main_fields)
            items = _get_entity_recursive(object_json, value)
            _write_to_file(object_fields, items, entity_fields, writer)
        else:
            tweet_fields = extract_fields(tweet, main_fields)
            items = _get_entity_recursive(tweet, entity_type)
            _write_to_file(tweet_fields, items, entity_fields, writer)
    outf.close() 
Example 35
Project: Health-Checker   Author: KriAga   File: common.py    MIT License 4 votes vote down vote up
def json2csv_entities(tweets_file, outfile, main_fields, entity_type, entity_fields,
                      encoding='utf8', errors='replace', gzip_compress=False):
    """
    Extract selected fields from a file of line-separated JSON tweets and
    write to a file in CSV format.

    This utility function allows a file of full Tweets to be easily converted
    to a CSV file for easier processing of Twitter entities. For example, the
    hashtags or media elements of a tweet can be extracted.

    It returns one line per entity of a Tweet, e.g. if a tweet has two hashtags
    there will be two lines in the output file, one per hashtag

    :param tweets_file: the file-like object containing full Tweets

    :param str outfile: The path of the text file where results should be\
    written

    :param list main_fields: The list of fields to be extracted from the main\
    object, usually the tweet. Useful examples: 'id_str' for the tweetID. See\
    <https://dev.twitter.com/overview/api/tweets> for a full list of fields.
    e. g.: ['id_str'], ['id', 'text', 'favorite_count', 'retweet_count']
    If `entity_type` is expressed with hierarchy, then it is the list of\
    fields of the object that corresponds to the key of the entity_type,\
    (e.g., for entity_type='user.urls', the fields in the main_fields list\
    belong to the user object; for entity_type='place.bounding_box', the\
    files in the main_field list belong to the place object of the tweet).

    :param list entity_type: The name of the entity: 'hashtags', 'media',\
    'urls' and 'user_mentions' for the tweet object. For a user object,\
    this needs to be expressed with a hierarchy: `'user.urls'`. For the\
    bounding box of the Tweet location, use `'place.bounding_box'`.

    :param list entity_fields: The list of fields to be extracted from the\
    entity. E.g. `['text']` (of the Tweet)

    :param error: Behaviour for encoding errors, see\
    https://docs.python.org/3/library/codecs.html#codec-base-classes

    :param gzip_compress: if `True`, ouput files are compressed with gzip
    """

    (writer, outf) = outf_writer_compat(outfile, encoding, errors, gzip_compress)
    header = get_header_field_list(main_fields, entity_type, entity_fields)
    writer.writerow(header)
    for line in tweets_file:
        tweet = json.loads(line)
        if _is_composed_key(entity_type):
            key, value = _get_key_value_composed(entity_type)
            object_json = _get_entity_recursive(tweet, key)
            if not object_json:
                # this can happen in the case of "place"
                continue
            object_fields = extract_fields(object_json, main_fields)
            items = _get_entity_recursive(object_json, value)
            _write_to_file(object_fields, items, entity_fields, writer)
        else:
            tweet_fields = extract_fields(tweet, main_fields)
            items = _get_entity_recursive(tweet, entity_type)
            _write_to_file(tweet_fields, items, entity_fields, writer)
    outf.close() 
Example 36
Project: honours_project   Author: JFriel   File: common.py    GNU General Public License v3.0 4 votes vote down vote up
def json2csv_entities(tweets_file, outfile, main_fields, entity_type, entity_fields,
                      encoding='utf8', errors='replace', gzip_compress=False):
    """
    Extract selected fields from a file of line-separated JSON tweets and
    write to a file in CSV format.

    This utility function allows a file of full Tweets to be easily converted
    to a CSV file for easier processing of Twitter entities. For example, the
    hashtags or media elements of a tweet can be extracted.

    It returns one line per entity of a Tweet, e.g. if a tweet has two hashtags
    there will be two lines in the output file, one per hashtag

    :param tweets_file: the file-like object containing full Tweets

    :param str outfile: The path of the text file where results should be\
    written

    :param list main_fields: The list of fields to be extracted from the main\
    object, usually the tweet. Useful examples: 'id_str' for the tweetID. See\
    <https://dev.twitter.com/overview/api/tweets> for a full list of fields.
    e. g.: ['id_str'], ['id', 'text', 'favorite_count', 'retweet_count']
    If `entity_type` is expressed with hierarchy, then it is the list of\
    fields of the object that corresponds to the key of the entity_type,\
    (e.g., for entity_type='user.urls', the fields in the main_fields list\
    belong to the user object; for entity_type='place.bounding_box', the\
    files in the main_field list belong to the place object of the tweet).

    :param list entity_type: The name of the entity: 'hashtags', 'media',\
    'urls' and 'user_mentions' for the tweet object. For a user object,\
    this needs to be expressed with a hierarchy: `'user.urls'`. For the\
    bounding box of the Tweet location, use `'place.bounding_box'`.

    :param list entity_fields: The list of fields to be extracted from the\
    entity. E.g. `['text']` (of the Tweet)

    :param error: Behaviour for encoding errors, see\
    https://docs.python.org/3/library/codecs.html#codec-base-classes

    :param gzip_compress: if `True`, ouput files are compressed with gzip
    """

    (writer, outf) = outf_writer_compat(outfile, encoding, errors, gzip_compress)
    header = get_header_field_list(main_fields, entity_type, entity_fields)
    writer.writerow(header)
    for line in tweets_file:
        tweet = json.loads(line)
        if _is_composed_key(entity_type):
            key, value = _get_key_value_composed(entity_type)
            object_json = _get_entity_recursive(tweet, key)
            if not object_json:
                # this can happen in the case of "place"
                continue
            object_fields = extract_fields(object_json, main_fields)
            items = _get_entity_recursive(object_json, value)
            _write_to_file(object_fields, items, entity_fields, writer)
        else:
            tweet_fields = extract_fields(tweet, main_fields)
            items = _get_entity_recursive(tweet, entity_type)
            _write_to_file(tweet_fields, items, entity_fields, writer)
    outf.close() 
Example 37
Project: honours_project   Author: JFriel   File: common.py    GNU General Public License v3.0 4 votes vote down vote up
def json2csv_entities(tweets_file, outfile, main_fields, entity_type, entity_fields,
                      encoding='utf8', errors='replace', gzip_compress=False):
    """
    Extract selected fields from a file of line-separated JSON tweets and
    write to a file in CSV format.

    This utility function allows a file of full Tweets to be easily converted
    to a CSV file for easier processing of Twitter entities. For example, the
    hashtags or media elements of a tweet can be extracted.

    It returns one line per entity of a Tweet, e.g. if a tweet has two hashtags
    there will be two lines in the output file, one per hashtag

    :param tweets_file: the file-like object containing full Tweets

    :param str outfile: The path of the text file where results should be\
    written

    :param list main_fields: The list of fields to be extracted from the main\
    object, usually the tweet. Useful examples: 'id_str' for the tweetID. See\
    <https://dev.twitter.com/overview/api/tweets> for a full list of fields.
    e. g.: ['id_str'], ['id', 'text', 'favorite_count', 'retweet_count']
    If `entity_type` is expressed with hierarchy, then it is the list of\
    fields of the object that corresponds to the key of the entity_type,\
    (e.g., for entity_type='user.urls', the fields in the main_fields list\
    belong to the user object; for entity_type='place.bounding_box', the\
    files in the main_field list belong to the place object of the tweet).

    :param list entity_type: The name of the entity: 'hashtags', 'media',\
    'urls' and 'user_mentions' for the tweet object. For a user object,\
    this needs to be expressed with a hierarchy: `'user.urls'`. For the\
    bounding box of the Tweet location, use `'place.bounding_box'`.

    :param list entity_fields: The list of fields to be extracted from the\
    entity. E.g. `['text']` (of the Tweet)

    :param error: Behaviour for encoding errors, see\
    https://docs.python.org/3/library/codecs.html#codec-base-classes

    :param gzip_compress: if `True`, ouput files are compressed with gzip
    """

    (writer, outf) = outf_writer_compat(outfile, encoding, errors, gzip_compress)
    header = get_header_field_list(main_fields, entity_type, entity_fields)
    writer.writerow(header)
    for line in tweets_file:
        tweet = json.loads(line)
        if _is_composed_key(entity_type):
            key, value = _get_key_value_composed(entity_type)
            object_json = _get_entity_recursive(tweet, key)
            if not object_json:
                # this can happen in the case of "place"
                continue
            object_fields = extract_fields(object_json, main_fields)
            items = _get_entity_recursive(object_json, value)
            _write_to_file(object_fields, items, entity_fields, writer)
        else:
            tweet_fields = extract_fields(tweet, main_fields)
            items = _get_entity_recursive(tweet, entity_type)
            _write_to_file(tweet_fields, items, entity_fields, writer)
    outf.close() 
Example 38
Project: icdc-docker   Author: CBIIT   File: client.py    Apache License 2.0 4 votes vote down vote up
def list_with_params(self, limit=float("inf"), start=None, page_size=100, params=None, negate_params=None):
        """
        Return a generator of document object corresponding to the supplied parameters, such
        as ``{'hashes': {'md5': '...'},
              'size': '...',
              'metadata': {'file_state': '...'},
              'urls_metadata': {'s3://url': {'state': '...'}
             }``.
        """
        params_copy = copy.deepcopy(params) or {}
        if 'hashes' in params_copy:
            params_copy['hash'] = params_copy.pop('hashes')
        if 'urls_metadata' in params_copy:
            params_copy['urls_metadata'] = json.dumps(params_copy.pop('urls_metadata'))
        reformatted_params = dict()
        for param in ['hash', 'metadata']:
            if param in params_copy:
                reformatted_params[param] = []
                for k, v in params_copy[param].items():
                    reformatted_params[param].append(str(k) + ':' + str(v))
                del params_copy[param]
        reformatted_params.update(params_copy)
        reformatted_params.update({"limit": page_size, "start": start})
        if negate_params:
            reformatted_params.update({"negate_params": json.dumps(negate_params)})
        yielded = 0
        while True:
            resp = self._get("index", params=reformatted_params)
            handle_error(resp)
            json_str = resp.json()
            if not json_str["records"]:
                return
            for doc in json_str["records"]:
                if yielded < limit:
                    yield Document(self, None, json=doc)
                    yielded += 1
                else:
                    return
            if len(json_str['records']) == page_size:
                reformatted_params['start'] = json_str['records'][-1]['did']
            else:
                # There's no more results
                return 
Example 39
Project: aop-helpFinder   Author: jecarvaill   File: common.py    GNU General Public License v3.0 4 votes vote down vote up
def json2csv_entities(tweets_file, outfile, main_fields, entity_type, entity_fields,
                      encoding='utf8', errors='replace', gzip_compress=False):
    """
    Extract selected fields from a file of line-separated JSON tweets and
    write to a file in CSV format.

    This utility function allows a file of full Tweets to be easily converted
    to a CSV file for easier processing of Twitter entities. For example, the
    hashtags or media elements of a tweet can be extracted.

    It returns one line per entity of a Tweet, e.g. if a tweet has two hashtags
    there will be two lines in the output file, one per hashtag

    :param tweets_file: the file-like object containing full Tweets

    :param str outfile: The path of the text file where results should be\
    written

    :param list main_fields: The list of fields to be extracted from the main\
    object, usually the tweet. Useful examples: 'id_str' for the tweetID. See\
    <https://dev.twitter.com/overview/api/tweets> for a full list of fields.
    e. g.: ['id_str'], ['id', 'text', 'favorite_count', 'retweet_count']
    If `entity_type` is expressed with hierarchy, then it is the list of\
    fields of the object that corresponds to the key of the entity_type,\
    (e.g., for entity_type='user.urls', the fields in the main_fields list\
    belong to the user object; for entity_type='place.bounding_box', the\
    files in the main_field list belong to the place object of the tweet).

    :param list entity_type: The name of the entity: 'hashtags', 'media',\
    'urls' and 'user_mentions' for the tweet object. For a user object,\
    this needs to be expressed with a hierarchy: `'user.urls'`. For the\
    bounding box of the Tweet location, use `'place.bounding_box'`.

    :param list entity_fields: The list of fields to be extracted from the\
    entity. E.g. `['text']` (of the Tweet)

    :param error: Behaviour for encoding errors, see\
    https://docs.python.org/3/library/codecs.html#codec-base-classes

    :param gzip_compress: if `True`, ouput files are compressed with gzip
    """

    (writer, outf) = outf_writer_compat(outfile, encoding, errors, gzip_compress)
    header = get_header_field_list(main_fields, entity_type, entity_fields)
    writer.writerow(header)
    for line in tweets_file:
        tweet = json.loads(line)
        if _is_composed_key(entity_type):
            key, value = _get_key_value_composed(entity_type)
            object_json = _get_entity_recursive(tweet, key)
            if not object_json:
                # this can happen in the case of "place"
                continue
            object_fields = extract_fields(object_json, main_fields)
            items = _get_entity_recursive(object_json, value)
            _write_to_file(object_fields, items, entity_fields, writer)
        else:
            tweet_fields = extract_fields(tweet, main_fields)
            items = _get_entity_recursive(tweet, entity_type)
            _write_to_file(tweet_fields, items, entity_fields, writer)
    outf.close() 
Example 40
Project: serverless-chatbots-workshop   Author: datteswararao   File: common.py    Apache License 2.0 4 votes vote down vote up
def json2csv_entities(tweets_file, outfile, main_fields, entity_type, entity_fields,
                      encoding='utf8', errors='replace', gzip_compress=False):
    """
    Extract selected fields from a file of line-separated JSON tweets and
    write to a file in CSV format.

    This utility function allows a file of full Tweets to be easily converted
    to a CSV file for easier processing of Twitter entities. For example, the
    hashtags or media elements of a tweet can be extracted.

    It returns one line per entity of a Tweet, e.g. if a tweet has two hashtags
    there will be two lines in the output file, one per hashtag

    :param tweets_file: the file-like object containing full Tweets

    :param str outfile: The path of the text file where results should be\
    written

    :param list main_fields: The list of fields to be extracted from the main\
    object, usually the tweet. Useful examples: 'id_str' for the tweetID. See\
    <https://dev.twitter.com/overview/api/tweets> for a full list of fields.
    e. g.: ['id_str'], ['id', 'text', 'favorite_count', 'retweet_count']
    If `entity_type` is expressed with hierarchy, then it is the list of\
    fields of the object that corresponds to the key of the entity_type,\
    (e.g., for entity_type='user.urls', the fields in the main_fields list\
    belong to the user object; for entity_type='place.bounding_box', the\
    files in the main_field list belong to the place object of the tweet).

    :param list entity_type: The name of the entity: 'hashtags', 'media',\
    'urls' and 'user_mentions' for the tweet object. For a user object,\
    this needs to be expressed with a hierarchy: `'user.urls'`. For the\
    bounding box of the Tweet location, use `'place.bounding_box'`.

    :param list entity_fields: The list of fields to be extracted from the\
    entity. E.g. `['text']` (of the Tweet)

    :param error: Behaviour for encoding errors, see\
    https://docs.python.org/3/library/codecs.html#codec-base-classes

    :param gzip_compress: if `True`, ouput files are compressed with gzip
    """

    (writer, outf) = outf_writer_compat(outfile, encoding, errors, gzip_compress)
    header = get_header_field_list(main_fields, entity_type, entity_fields)
    writer.writerow(header)
    for line in tweets_file:
        tweet = json.loads(line)
        if _is_composed_key(entity_type):
            key, value = _get_key_value_composed(entity_type)
            object_json = _get_entity_recursive(tweet, key)
            if not object_json:
                # this can happen in the case of "place"
                continue
            object_fields = extract_fields(object_json, main_fields)
            items = _get_entity_recursive(object_json, value)
            _write_to_file(object_fields, items, entity_fields, writer)
        else:
            tweet_fields = extract_fields(tweet, main_fields)
            items = _get_entity_recursive(tweet, entity_type)
            _write_to_file(tweet_fields, items, entity_fields, writer)
    outf.close() 
Example 41
Project: serverless-chatbots-workshop   Author: datteswararao   File: common.py    Apache License 2.0 4 votes vote down vote up
def json2csv_entities(tweets_file, outfile, main_fields, entity_type, entity_fields,
                      encoding='utf8', errors='replace', gzip_compress=False):
    """
    Extract selected fields from a file of line-separated JSON tweets and
    write to a file in CSV format.

    This utility function allows a file of full Tweets to be easily converted
    to a CSV file for easier processing of Twitter entities. For example, the
    hashtags or media elements of a tweet can be extracted.

    It returns one line per entity of a Tweet, e.g. if a tweet has two hashtags
    there will be two lines in the output file, one per hashtag

    :param tweets_file: the file-like object containing full Tweets

    :param str outfile: The path of the text file where results should be\
    written

    :param list main_fields: The list of fields to be extracted from the main\
    object, usually the tweet. Useful examples: 'id_str' for the tweetID. See\
    <https://dev.twitter.com/overview/api/tweets> for a full list of fields.
    e. g.: ['id_str'], ['id', 'text', 'favorite_count', 'retweet_count']
    If `entity_type` is expressed with hierarchy, then it is the list of\
    fields of the object that corresponds to the key of the entity_type,\
    (e.g., for entity_type='user.urls', the fields in the main_fields list\
    belong to the user object; for entity_type='place.bounding_box', the\
    files in the main_field list belong to the place object of the tweet).

    :param list entity_type: The name of the entity: 'hashtags', 'media',\
    'urls' and 'user_mentions' for the tweet object. For a user object,\
    this needs to be expressed with a hierarchy: `'user.urls'`. For the\
    bounding box of the Tweet location, use `'place.bounding_box'`.

    :param list entity_fields: The list of fields to be extracted from the\
    entity. E.g. `['text']` (of the Tweet)

    :param error: Behaviour for encoding errors, see\
    https://docs.python.org/3/library/codecs.html#codec-base-classes

    :param gzip_compress: if `True`, ouput files are compressed with gzip
    """

    (writer, outf) = outf_writer_compat(outfile, encoding, errors, gzip_compress)
    header = get_header_field_list(main_fields, entity_type, entity_fields)
    writer.writerow(header)
    for line in tweets_file:
        tweet = json.loads(line)
        if _is_composed_key(entity_type):
            key, value = _get_key_value_composed(entity_type)
            object_json = _get_entity_recursive(tweet, key)
            if not object_json:
                # this can happen in the case of "place"
                continue
            object_fields = extract_fields(object_json, main_fields)
            items = _get_entity_recursive(object_json, value)
            _write_to_file(object_fields, items, entity_fields, writer)
        else:
            tweet_fields = extract_fields(tweet, main_fields)
            items = _get_entity_recursive(tweet, entity_type)
            _write_to_file(tweet_fields, items, entity_fields, writer)
    outf.close()