Python google.cloud.datastore.Entity() Examples

The following are code examples for showing how to use google.cloud.datastore.Entity(). They are from open source Python projects. You can vote up the examples you like or vote down the ones you don't like.

Example 1
Project: python-docs-samples   Author: GoogleCloudPlatform   File: quickstart.py    Apache License 2.0 8 votes vote down vote up
def run_quickstart():
    # [START datastore_quickstart]
    # Imports the Google Cloud client library
    from google.cloud import datastore

    # Instantiates a client
    datastore_client = datastore.Client()

    # The kind for the new entity
    kind = 'Task'
    # The name/ID for the new entity
    name = 'sampletask1'
    # The Cloud Datastore key for the new entity
    task_key = datastore_client.key(kind, name)

    # Prepares the new entity
    task = datastore.Entity(key=task_key)
    task['description'] = 'Buy milk'

    # Saves the entity
    datastore_client.put(task)

    print('Saved {}: {}'.format(task.key.name, task['description']))
    # [END datastore_quickstart] 
Example 2
Project: som   Author: vsoch   File: models.py    MIT License 6 votes vote down vote up
def __init__(self, **kwargs):
        '''A ModelBase is a controller for a general DataStore
        Entity. Most of the variables in init are for initial
        validation, and further fields etc are stored with
        the Entity itself (_Entity)
        :param client: the datastore client, should be passed on init
        :param _fields: intiial fields to validate on model creation
        :param _key: the initial key used for the entity
        :param _Entity: the final generated (validated) entity
        '''
        client = None
        if "client" in kwargs:
            client = kwargs['client']

        super(ModelBase, self).__init__(**kwargs)
        if client is None:
            client = datastore.Client()
        self.client = client
        self._fields = validate_model(self.model['fields'])
        self._key = self.client.key(*self.model['key'])
        self._Entity = None
        if "exclude_from_indexes" in self.model:
            self._exclude_from_indexes = self.model['exclude_from_indexes'] # Can be None
        else:
            self._exclude_from_indexes = None 
Example 3
Project: hangouts-chat-samples   Author: gsuitedevs   File: auth.py    Apache License 2.0 6 votes vote down vote up
def put_user_credentials(
            self, user_name: str, creds: Credentials) -> None:
        """Stores OAuth2 credentials for a user.

        Args:
            user_name (str): The identifier for the associated user.
            creds (Credentials): The OAuth2 credentials obtained for the user.
        """
        key = self.datastore_client.key('RefreshToken', user_name)
        entity = datastore.Entity(key)
        entity.update({
            'credentials': {
                'token': creds.token,
                'refresh_token': creds.refresh_token,
                'token_uri': creds.token_uri,
                'client_id': creds.client_id,
                'client_secret': creds.client_secret,
                'scopes': creds.scopes,
            },
            'timestamp': time.time()
        })
        self.datastore_client.put(entity) 
Example 4
Project: python-ndb   Author: googleapis   File: test_model.py    Apache License 2.0 6 votes vote down vote up
def test__from_datastore_compressed_to_uncompressed():
        class ThisKind(model.Model):
            foo = model.BlobProperty(compressed=False)

        key = datastore.Key("ThisKind", 123, project="testing")
        datastore_entity = datastore.Entity(key=key)
        uncompressed_value = b"abc" * 1000
        compressed_value = zlib.compress(uncompressed_value)
        datastore_entity.update({"foo": compressed_value})
        meanings = {"foo": (model._MEANING_COMPRESSED, compressed_value)}
        datastore_entity._meanings = meanings
        protobuf = helpers.entity_to_protobuf(datastore_entity)
        entity = model._entity_from_protobuf(protobuf)
        assert entity.foo == uncompressed_value
        ds_entity = model._entity_to_ds_entity(entity)
        assert ds_entity["foo"] == uncompressed_value 
Example 5
Project: python-ndb   Author: googleapis   File: test_model.py    Apache License 2.0 6 votes vote down vote up
def test_legacy_repeated_structured_property_projection():
        class OtherKind(model.Model):
            foo = model.IntegerProperty()
            bar = model.StringProperty()

        class ThisKind(model.Model):
            baz = model.StructuredProperty(OtherKind, repeated=True)
            copacetic = model.BooleanProperty()

        key = datastore.Key("ThisKind", 123, project="testing")
        datastore_entity = datastore.Entity(key=key)
        datastore_entity.update(
            {"baz.foo": 42, "baz.bar": "himom", "copacetic": True}
        )
        protobuf = helpers.entity_to_protobuf(datastore_entity)
        entity = model._entity_from_protobuf(protobuf)
        assert isinstance(entity, ThisKind)
        assert entity.baz[0].foo == 42
        assert entity.baz[0].bar == "himom"
        assert entity.copacetic is True 
Example 6
Project: python-ndb   Author: googleapis   File: test_model.py    Apache License 2.0 6 votes vote down vote up
def test_polymodel():
        class Animal(polymodel.PolyModel):
            foo = model.IntegerProperty()

        class Cat(Animal):
            bar = model.StringProperty()

        key = datastore.Key("Animal", 123, project="testing")
        datastore_entity = datastore.Entity(key=key)
        datastore_entity.update(
            {"foo": 42, "bar": "himom!", "class": ["Animal", "Cat"]}
        )

        entity = model._entity_from_ds_entity(datastore_entity)
        assert isinstance(entity, Cat)
        assert entity.foo == 42
        assert entity.bar == "himom!"
        assert entity.class_ == ["Animal", "Cat"] 
Example 7
Project: python-ndb   Author: googleapis   File: test_model.py    Apache License 2.0 6 votes vote down vote up
def test_polymodel_projection():
        class Animal(polymodel.PolyModel):
            foo = model.IntegerProperty()

        class Cat(Animal):
            bar = model.StringProperty()

        key = datastore.Key("Animal", 123, project="testing")
        datastore_entity = datastore.Entity(key=key)
        datastore_entity.update({"foo": 42, "bar": "himom!", "class": "Cat"})

        entity = model._entity_from_ds_entity(datastore_entity)
        assert isinstance(entity, Cat)
        assert entity.foo == 42
        assert entity.bar == "himom!"
        assert entity.class_ == ["Cat"] 
Example 8
Project: pulse-data   Author: Recidiviz   File: datastore_ingest_info.py    GNU General Public License v3.0 6 votes vote down vote up
def _get_ingest_info_entities_for_region(region: str, session_start_time=None) \
        -> List[datastore.Entity]:
    logging.info("Getting ingest info entities for region: [%s] and "
                 "session_start_time: [%s]", region, session_start_time)
    session_query = ds().query(kind=INGEST_INFO_KIND)
    session_query.add_filter('region', '=', region)
    if session_start_time:
        session_query.add_filter('session_start_time', '=', session_start_time)

    results = None
    try:
        results = retry_grpc(
            NUM_GRPC_RETRIES, session_query.fetch, limit=None)

    except Exception:
        raise DatastoreBatchGetError(region)

    return list(results) 
Example 9
Project: myplace   Author: vinihcampos   File: main.py    MIT License 6 votes vote down vote up
def add_event(client, title, description, date, time, address):
	key = client.key('Event')

	event = datastore.Entity(key=key)
	event['title'] = title
	event['description'] = description
	event['date'] = date
	event['time'] = time

	geocode_result = gmaps.geocode(address)
	if len(geocode_result) > 0:
		location = geocode_result[0]['geometry']['location']
		event['latitude'] = location['lat']
		event['longitude'] = location['lng']

	client.put(event)

	return event.key 
Example 10
Project: python-docs-samples   Author: GoogleCloudPlatform   File: snippets.py    Apache License 2.0 6 votes vote down vote up
def entity_with_parent(client):
    # [START datastore_entity_with_parent]
    key_with_parent = client.key(
        'TaskList', 'default', 'Task', 'sample_task')

    task = datastore.Entity(key=key_with_parent)

    task.update({
        'category': 'Personal',
        'done': False,
        'priority': 4,
        'description': 'Learn Cloud Datastore'
    })
    # [END datastore_entity_with_parent]

    return task 
Example 11
Project: python-docs-samples   Author: GoogleCloudPlatform   File: snippets.py    Apache License 2.0 6 votes vote down vote up
def properties(client):
    key = client.key('Task')
    # [START datastore_properties]
    task = datastore.Entity(
        key,
        exclude_from_indexes=['description'])
    task.update({
        'category': 'Personal',
        'description': 'Learn Cloud Datastore',
        'created': datetime.datetime.utcnow(),
        'done': False,
        'priority': 4,
        'percent_complete': 10.5,
    })
    # [END datastore_properties]

    return task 
Example 12
Project: python-docs-samples   Author: GoogleCloudPlatform   File: snippets.py    Apache License 2.0 6 votes vote down vote up
def array_value(client):
    key = client.key('Task')
    # [START datastore_array_value]
    task = datastore.Entity(key)
    task.update({
        'tags': [
            'fun',
            'programming'
        ],
        'collaborators': [
            'alice',
            'bob'
        ]
    })
    # [END datastore_array_value]

    return task 
Example 13
Project: python-docs-samples   Author: GoogleCloudPlatform   File: snippets.py    Apache License 2.0 6 votes vote down vote up
def insert(client):
    # [START datastore_insert]
    with client.transaction():
        incomplete_key = client.key('Task')

        task = datastore.Entity(key=incomplete_key)

        task.update({
            'category': 'Personal',
            'done': False,
            'priority': 4,
            'description': 'Learn Cloud Datastore'
        })

        client.put(task)
    # [END datastore_insert]

    return task 
Example 14
Project: python-docs-samples   Author: GoogleCloudPlatform   File: snippets.py    Apache License 2.0 6 votes vote down vote up
def batch_upsert(client):
    # [START datastore_batch_upsert]
    task1 = datastore.Entity(client.key('Task', 1))

    task1.update({
        'category': 'Personal',
        'done': False,
        'priority': 4,
        'description': 'Learn Cloud Datastore'
    })

    task2 = datastore.Entity(client.key('Task', 2))

    task2.update({
        'category': 'Work',
        'done': False,
        'priority': 8,
        'description': 'Integrate Cloud Datastore'
    })

    client.put_multi([task1, task2])
    # [END datastore_batch_upsert]

    return task1, task2 
Example 15
Project: python-docs-samples   Author: GoogleCloudPlatform   File: snippets.py    Apache License 2.0 6 votes vote down vote up
def ancestor_query(client):
    task = datastore.Entity(
        client.key('TaskList', 'default', 'Task'))
    task.update({
        'category': 'Personal',
        'description': 'Learn Cloud Datastore',
    })
    client.put(task)

    # [START datastore_ancestor_query]
    # Query filters are omitted in this example as any ancestor queries with a
    # non-key filter require a composite index.
    ancestor = client.key('TaskList', 'default')
    query = client.query(kind='Task', ancestor=ancestor)
    # [END datastore_ancestor_query]

    return list(query.fetch()) 
Example 16
Project: python-docs-samples   Author: GoogleCloudPlatform   File: snippets.py    Apache License 2.0 6 votes vote down vote up
def exploding_properties(client):
    # [START datastore_exploding_properties]
    task = datastore.Entity(client.key('Task'))
    task.update({
        'tags': [
            'fun',
            'programming',
            'learn'
        ],
        'collaborators': [
            'alice',
            'bob',
            'charlie'
        ],
        'created': datetime.datetime.utcnow()
    })
    # [END datastore_exploding_properties]

    return task 
Example 17
Project: python-docs-samples   Author: GoogleCloudPlatform   File: snippets.py    Apache License 2.0 6 votes vote down vote up
def transactional_get_or_create(client):
    # [START datastore_transactional_get_or_create]
    with client.transaction():
        key = client.key('Task', datetime.datetime.utcnow().isoformat())

        task = client.get(key)

        if not task:
            task = datastore.Entity(key)
            task.update({
                'description': 'Example task'
            })
            client.put(task)

        return task
    # [END datastore_transactional_get_or_create] 
Example 18
Project: python-docs-samples   Author: GoogleCloudPlatform   File: snippets.py    Apache License 2.0 6 votes vote down vote up
def transactional_single_entity_group_read_only(client):
    client.put_multi([
        datastore.Entity(key=client.key('TaskList', 'default')),
        datastore.Entity(key=client.key('TaskList', 'default', 'Task', 1))
    ])

    # [START datastore_transactional_single_entity_group_read_only]
    with client.transaction(read_only=True):
        task_list_key = client.key('TaskList', 'default')

        task_list = client.get(task_list_key)

        query = client.query(kind='Task', ancestor=task_list_key)
        tasks_in_list = list(query.fetch())

        return task_list, tasks_in_list
    # [END datastore_transactional_single_entity_group_read_only] 
Example 19
Project: python-docs-samples   Author: GoogleCloudPlatform   File: snippets.py    Apache License 2.0 6 votes vote down vote up
def namespace_run_query(client):
    # Create an entity in another namespace.
    task = datastore.Entity(
        client.key('Task', 'sample-task', namespace='google'))
    client.put(task)

    # [START datastore_namespace_run_query]
    # All namespaces
    query = client.query(kind='__namespace__')
    query.keys_only()

    all_namespaces = [entity.key.id_or_name for entity in query.fetch()]

    # Filtered namespaces
    start_namespace = client.key('__namespace__', 'g')
    end_namespace = client.key('__namespace__', 'h')
    query = client.query(kind='__namespace__')
    query.key_filter(start_namespace, '>=')
    query.key_filter(end_namespace, '<')

    filtered_namespaces = [entity.key.id_or_name for entity in query.fetch()]
    # [END datastore_namespace_run_query]

    return all_namespaces, filtered_namespaces 
Example 20
Project: python-docs-samples   Author: GoogleCloudPlatform   File: tasks.py    Apache License 2.0 6 votes vote down vote up
def add_task(client, description):
    key = client.key('Task')

    task = datastore.Entity(
        key, exclude_from_indexes=['description'])

    task.update({
        'created': datetime.datetime.utcnow(),
        'description': description,
        'done': False
    })

    client.put(task)

    return task.key
# [END datastore_add_entity]


# [START datastore_update_entity] 
Example 21
Project: python-docs-samples   Author: GoogleCloudPlatform   File: wiki.py    Apache License 2.0 6 votes vote down vote up
def save_page(ds, page, content):
    with ds.transaction():
        now = datetime.datetime.utcnow()
        current_key = path_to_key(ds, '{}.page/current.revision'.format(page))
        revision_key = path_to_key(ds, '{}.page/{}.revision'.format(page, now))

        if ds.get(revision_key):
            raise AssertionError("Revision %s already exists" % revision_key)

        current = ds.get(current_key)

        if current:
            revision = datastore.Entity(key=revision_key)
            revision.update(current)
            ds.put(revision)
        else:
            current = datastore.Entity(key=current_key)

        current['content'] = content

        ds.put(current) 
Example 22
Project: eclipse2017   Author: google   File: image_sorter_test.py    Apache License 2.0 6 votes vote down vote up
def test_sort_newest(self):
        datastore_client = datastore.Client(config.PROJECT_ID)
        key = datastore_client.key('Photo', '1')
        entity = datastore.Entity(key=key)
        entity["image_datetime"] = datetime.datetime.utcnow()
        time.sleep(1)
        key = datastore_client.key('Photo', '2')
        entity2 = datastore.Entity(key=key)
        entity2["image_datetime"] = datetime.datetime.utcnow()
        time.sleep(1)
        key = datastore_client.key('Photo', '3')
        entity3 = datastore.Entity(key=key)
        entity3["image_datetime"] = datetime.datetime.utcnow()

        entities = [entity, entity2, entity3]
        self.assertEqual(image_sorter.pick_image(entities), entity3)
        entities = [entity3, entity2, entity]
        self.assertEqual(image_sorter.pick_image(entities), entity3)
        entities = [entity2, entity3, entity]
        self.assertEqual(image_sorter.pick_image(entities), entity3) 
Example 23
Project: eclipse2017   Author: google   File: create_test_users.py    Apache License 2.0 6 votes vote down vote up
def create_user(client, user_id, eclipse_poly):
  user_key = client.key("User", user_id)
  print user_key.name
  user = datastore.Entity(key = user_key)
  user['name'] = u"Test User " + user_id
  user['email'] = u"test" + user_id + u"@example.com"
  # Get random location.
  point = eclipse_poly[random.randint(0, len(eclipse_poly) - 1)]
  u = float(random.uniform(-1.0, 1.0))
  v = float(random.uniform(-1.0, 1.0))
  user['geocoded_location'] = [point['lat'] + u, point['lng'] + v]
  print point
  print u
  print v
  print user['geocoded_location']
  client.put(user)
  user_role_key = client.key("UserRole", user_id)
  user_role = datastore.Entity(key = user_role_key)
  user_role['roles'] = [u"user"]
  # make some of them volunteers.
  if float(random.uniform(0.0, 1.0)) > 0.8:
    user_role['roles'].append(u"volunteer")

  client.put(user_role)
  return user 
Example 24
Project: neural-fingerprinting   Author: StephanZheng   File: cloud_client.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def entity(self, key):
    """Creates datastore entity."""
    return datastore.Entity(key) 
Example 25
Project: PyChunkedGraph   Author: seung-lab   File: flask_log_db.py    Mozilla Public License 2.0 5 votes vote down vote up
def _add_log(self, log_type, user_id, user_ip, request_time, response_time,
                 url, request_type=None, request_arg=None, request_data=None,
                 msg=None):
        # Extract relevant information and build entity

        key = self.client.key(self.kind, namespace=self.namespace)
        entity = datastore.Entity(key)

        url_split = url.split("/")

        if request_type:
            if "?" in url_split[-1]:
                request_type = url_split[-1].split("?")[0]
                request_opt_arg = url_split[-1].split("?")[1]
            else:
                request_type = url_split[-1]
                request_opt_arg = None

        if len(request_data) == 0:
            request_data = None
        else:
            request_data = json.loads(request_data)

        entity['type'] = log_type
        entity['user_id'] = user_id
        entity['user_ip'] = user_ip
        entity['date'] = request_time
        entity['response_time(ms)'] = response_time
        entity['request_type'] = request_type
        entity['request_arg'] = request_arg
        entity['request_data'] = str(request_data)
        entity['request_opt_arg'] = request_opt_arg
        entity['url'] = url
        entity['msg'] = msg

        self.client.put(entity)

        return entity.key.id 
Example 26
Project: som   Author: vsoch   File: models.py    MIT License 5 votes vote down vote up
def add(self,task):
        '''return all tasks in the set
        '''
        if isinstance(task,datastore.query.Query):
            self.queries.append(task)
        else:
            if not isinstance(task,datastore.Entity):
                task = task._Entity
            self.tasks.append(task) 
Example 27
Project: som   Author: vsoch   File: models.py    MIT License 5 votes vote down vote up
def __repr__(self):
        if self._Entity is None:
            return "Entity:None"
        return str(self._Entity.key) 
Example 28
Project: som   Author: vsoch   File: models.py    MIT License 5 votes vote down vote up
def __str__(self):
        if self._Entity is None:
            return "Entity:None"
        return str(self._Entity.key) 
Example 29
Project: som   Author: vsoch   File: models.py    MIT License 5 votes vote down vote up
def update_fields(self,new_fields,add_new=True):
        '''update fields will update the model's fields with an input dictionary
        new fields are added if add_new=True (default). This does not by default save
        the entity.
        '''
        for key,value in new_fields.items():
            if key not in self._Entity.keys():
                bot.debug("adding %s to Entity" %(key))
                self._Entity[key] = value
            else:
                if add_new == True:
                    bot.debug("%s found existing in Entity, overwriting" %(key))
                    self._Entity[key] = value 
Example 30
Project: som   Author: vsoch   File: models.py    MIT License 5 votes vote down vote up
def __init__(self, collection, entity, uid, create=False,fields=None):
        self.collection = collection
        self.model = entity(uid,collection)
        super(Entity, self).__init__()
        if create:
            self.update_or_create(fields=fields)
        else:
            self.update_or_create(fields=fields, save=False) 
Example 31
Project: clusterfuzz   Author: google   File: ndb_patcher.py    Apache License 2.0 5 votes vote down vote up
def _cloud_entity_to_ndb_entity(model_class, cloud_entity, projection=None):
  """Convert cloud entity to ndb.Model."""
  if cloud_entity is None:
    return None

  props = {}
  for key, value in six.iteritems(cloud_entity):
    ndb_property = getattr(model_class, key, None)
    if not isinstance(ndb_property, ndb.Property):
      # Deleted attribute from an old entity.
      continue

    if isinstance(ndb_property, ndb.ComputedProperty):
      continue

    if isinstance(value, (datastore.Entity, datastore.helpers.GeoPoint)):
      raise NdbPatcherException('Unsupported type for value: ' + repr(value))

    if isinstance(value, datastore.Key):
      value = _cloud_key_to_ndb_key(value)

    if type(getattr(model_class, key)) is ndb.DateProperty:
      value = value.date()

    if type(getattr(model_class, key)) is ndb.TimeProperty:
      value = value.time()

    if isinstance(value, (datetime.datetime, datetime.time)):
      # NDB datetimes are not timezone aware.
      value = value.replace(tzinfo=None)

    props[key] = value

  ndb_entity = model_class(
      id=cloud_entity.key.id_or_name, projection=projection, **props)
  return ndb_entity 
Example 32
Project: Flask-Blogging   Author: gouthambs   File: gcdatastore.py    MIT License 5 votes vote down vote up
def _get_new_post_id(self):
        key = self._client.key('PostIDCounter', 'Counter')
        query = self._client.get(key)

        if query:
            counter = dict(query)
        else:
            counter = None

        if counter:
            counter = counter["value"]+1
            key = self._client.key('PostIDCounter', 'Counter')
            task = self._client.get(key)
            task['value'] = counter
            self._client.put(task)

            return int(counter)
        else:
            # Create a new counter
            key = self._client.key('PostIDCounter', 'Counter')
            counter = datastore.Entity(key=key)
            counter.update({
                    'value': 1,
            })
            self._client.put(counter)
            return 1 
Example 33
Project: python-ndb   Author: googleapis   File: test_model.py    Apache License 2.0 5 votes vote down vote up
def test__from_datastore_compressed_to_compressed():
        class ThisKind(model.Model):
            foo = model.BlobProperty(compressed=True)

        key = datastore.Key("ThisKind", 123, project="testing")
        datastore_entity = datastore.Entity(key=key)
        uncompressed_value = b"abc" * 1000
        compressed_value = zlib.compress(uncompressed_value)
        datastore_entity.update({"foo": compressed_value})
        meanings = {"foo": (model._MEANING_COMPRESSED, compressed_value)}
        datastore_entity._meanings = meanings
        protobuf = helpers.entity_to_protobuf(datastore_entity)
        entity = model._entity_from_protobuf(protobuf)
        ds_entity = model._entity_to_ds_entity(entity)
        assert ds_entity["foo"] == compressed_value 
Example 34
Project: python-ndb   Author: googleapis   File: test_model.py    Apache License 2.0 5 votes vote down vote up
def test__from_datastore_uncompressed_to_uncompressed():
        class ThisKind(model.Model):
            foo = model.BlobProperty(compressed=False)

        key = datastore.Key("ThisKind", 123, project="testing")
        datastore_entity = datastore.Entity(key=key)
        uncompressed_value = b"abc" * 1000
        datastore_entity.update({"foo": uncompressed_value})
        protobuf = helpers.entity_to_protobuf(datastore_entity)
        entity = model._entity_from_protobuf(protobuf)
        assert entity.foo == uncompressed_value
        ds_entity = model._entity_to_ds_entity(entity)
        assert ds_entity["foo"] == uncompressed_value 
Example 35
Project: python-ndb   Author: googleapis   File: test_model.py    Apache License 2.0 5 votes vote down vote up
def test__from_datastore_uncompressed_to_compressed():
        class ThisKind(model.Model):
            foo = model.BlobProperty(compressed=True)

        key = datastore.Key("ThisKind", 123, project="testing")
        datastore_entity = datastore.Entity(key=key)
        uncompressed_value = b"abc" * 1000
        compressed_value = zlib.compress(uncompressed_value)
        datastore_entity.update({"foo": uncompressed_value})
        protobuf = helpers.entity_to_protobuf(datastore_entity)
        entity = model._entity_from_protobuf(protobuf)
        ds_entity = model._entity_to_ds_entity(entity)
        assert ds_entity["foo"] == compressed_value 
Example 36
Project: python-ndb   Author: googleapis   File: test_model.py    Apache License 2.0 5 votes vote down vote up
def _add_to_entity_helper(user_value):
        entity = entity_module.Entity()
        name = "u"

        user_value.add_to_entity(entity, name)
        assert list(entity.keys()) == [name]
        user_entity = entity[name]
        assert entity._meanings == {
            name: (model._MEANING_PREDEFINED_ENTITY_USER, user_entity)
        }
        assert user_entity["email"] == user_value._email
        assert user_entity["auth_domain"] == user_value._auth_domain
        return user_entity 
Example 37
Project: python-ndb   Author: googleapis   File: test_model.py    Apache License 2.0 5 votes vote down vote up
def test__to_base_type():
        class Mine(model.Model):
            foo = model.StringProperty()

        class MineToo(model.Model):
            bar = model.StructuredProperty(Mine)

        minetoo = MineToo(bar=Mine(foo="bar"))
        ds_bar = MineToo.bar._to_base_type(minetoo.bar)
        assert isinstance(ds_bar, entity_module.Entity)
        assert ds_bar["foo"] == "bar"
        assert ds_bar.kind == "Mine" 
Example 38
Project: python-ndb   Author: googleapis   File: test_model.py    Apache License 2.0 5 votes vote down vote up
def test__from_base_type(self):
        class Simple(model.Model):
            pass

        prop = model.StructuredProperty(Simple, name="ent")
        entity = entity_module.Entity()
        expected = Simple()
        assert prop._from_base_type(entity) == expected 
Example 39
Project: python-ndb   Author: googleapis   File: test_model.py    Apache License 2.0 5 votes vote down vote up
def test__to_base_type(self):
        class Simple(model.Model):
            pass

        prop = model.LocalStructuredProperty(Simple, name="ent")
        value = Simple()
        entity = entity_module.Entity()
        pb = helpers.entity_to_protobuf(entity)
        expected = pb.SerializePartialToString()
        assert prop._to_base_type(value) == expected 
Example 40
Project: python-ndb   Author: googleapis   File: test_model.py    Apache License 2.0 5 votes vote down vote up
def test__from_base_type(self):
        class Simple(model.Model):
            pass

        prop = model.LocalStructuredProperty(Simple, name="ent")
        entity = entity_module.Entity()
        expected = Simple()
        assert prop._from_base_type(entity) == expected 
Example 41
Project: python-ndb   Author: googleapis   File: test_model.py    Apache License 2.0 5 votes vote down vote up
def test__from_base_type_bytes(self):
        class Simple(model.Model):
            pass

        prop = model.LocalStructuredProperty(Simple, name="ent")
        pb = helpers.entity_to_protobuf(entity_module.Entity())
        value = pb.SerializePartialToString()
        expected = Simple()
        assert prop._from_base_type(value) == expected 
Example 42
Project: python-ndb   Author: googleapis   File: test_model.py    Apache License 2.0 5 votes vote down vote up
def test_standard_case():
        class ThisKind(model.Model):
            a = model.IntegerProperty()
            b = model.BooleanProperty()
            c = model.PickleProperty()
            d = model.StringProperty(repeated=True)
            e = model.PickleProperty(repeated=True)
            notaproperty = True

        dill = {"sandwiches": ["turkey", "reuben"], "not_sandwiches": "tacos"}
        gherkin = [{"a": {"b": "c"}, "d": 0}, [1, 2, 3], "himom"]
        key = datastore.Key("ThisKind", 123, project="testing")
        datastore_entity = datastore.Entity(key=key)
        datastore_entity.update(
            {
                "a": 42,
                "b": None,
                "c": pickle.dumps(gherkin, pickle.HIGHEST_PROTOCOL),
                "d": ["foo", "bar", "baz"],
                "e": [
                    pickle.dumps(gherkin, pickle.HIGHEST_PROTOCOL),
                    pickle.dumps(dill, pickle.HIGHEST_PROTOCOL),
                ],
                "notused": 32,
                "notaproperty": None,
            }
        )
        protobuf = helpers.entity_to_protobuf(datastore_entity)
        entity = model._entity_from_protobuf(protobuf)
        assert isinstance(entity, ThisKind)
        assert entity.a == 42
        assert entity.b is None
        assert entity.c == gherkin
        assert entity.d == ["foo", "bar", "baz"]
        assert entity.e == [gherkin, dill]
        assert entity._key == key_module.Key("ThisKind", 123, app="testing")
        assert entity.notaproperty is True 
Example 43
Project: python-ndb   Author: googleapis   File: test_model.py    Apache License 2.0 5 votes vote down vote up
def test_property_named_key():
        class ThisKind(model.Model):
            key = model.StringProperty()

        key = datastore.Key("ThisKind", 123, project="testing")
        datastore_entity = datastore.Entity(key=key)
        datastore_entity.update({"key": "luck"})
        protobuf = helpers.entity_to_protobuf(datastore_entity)
        entity = model._entity_from_protobuf(protobuf)
        assert isinstance(entity, ThisKind)
        assert entity.key == "luck"
        assert entity._key.kind() == "ThisKind"
        assert entity._key.id() == 123 
Example 44
Project: python-ndb   Author: googleapis   File: test_model.py    Apache License 2.0 5 votes vote down vote up
def test_expando_property():
        class ThisKind(model.Expando):
            key = model.StringProperty()

        key = datastore.Key("ThisKind", 123, project="testing")
        datastore_entity = datastore.Entity(key=key)
        datastore_entity.update({"key": "luck", "expando_prop": "good"})
        protobuf = helpers.entity_to_protobuf(datastore_entity)
        entity = model._entity_from_protobuf(protobuf)
        assert isinstance(entity, ThisKind)
        assert entity.key == "luck"
        assert entity._key.kind() == "ThisKind"
        assert entity._key.id() == 123
        assert entity.expando_prop == "good" 
Example 45
Project: python-ndb   Author: googleapis   File: test_model.py    Apache License 2.0 5 votes vote down vote up
def test_expando_property_list_value():
        class ThisKind(model.Expando):
            key = model.StringProperty()

        key = datastore.Key("ThisKind", 123, project="testing")
        datastore_entity = datastore.Entity(key=key)
        datastore_entity.update({"key": "luck", "expando_prop": ["good"]})
        protobuf = helpers.entity_to_protobuf(datastore_entity)
        entity = model._entity_from_protobuf(protobuf)
        assert isinstance(entity, ThisKind)
        assert entity.key == "luck"
        assert entity._key.kind() == "ThisKind"
        assert entity._key.id() == 123
        assert entity.expando_prop == ["good"] 
Example 46
Project: python-ndb   Author: googleapis   File: test_model.py    Apache License 2.0 5 votes vote down vote up
def test_value_but_non_expando_property():
        class ThisKind(model.Model):
            key = model.StringProperty()

        key = datastore.Key("ThisKind", 123, project="testing")
        datastore_entity = datastore.Entity(key=key)
        datastore_entity.update({"key": "luck", "expando_prop": None})
        protobuf = helpers.entity_to_protobuf(datastore_entity)
        entity = model._entity_from_protobuf(protobuf)
        assert isinstance(entity, ThisKind)
        assert entity.key == "luck"
        assert entity._key.kind() == "ThisKind"
        assert entity._key.id() == 123 
Example 47
Project: python-ndb   Author: googleapis   File: test_model.py    Apache License 2.0 5 votes vote down vote up
def test_repeated_structured_property():
        class OtherKind(model.Model):
            foo = model.IntegerProperty()
            bar = model.StringProperty()

        class ThisKind(model.Model):
            baz = model.StructuredProperty(OtherKind, repeated=True)
            copacetic = model.BooleanProperty()

        key = datastore.Key("ThisKind", 123, project="testing")
        datastore_entity = datastore.Entity(key=key)
        datastore_entity.update(
            {
                "baz.foo": [42, 144],
                "baz.bar": ["himom", "hellodad"],
                "copacetic": True,
            }
        )
        protobuf = helpers.entity_to_protobuf(datastore_entity)
        entity = model._entity_from_protobuf(protobuf)
        assert isinstance(entity, ThisKind)
        assert entity.baz[0].foo == 42
        assert entity.baz[0].bar == "himom"
        assert entity.baz[1].foo == 144
        assert entity.baz[1].bar == "hellodad"
        assert entity.copacetic is True 
Example 48
Project: python-ndb   Author: googleapis   File: test_model.py    Apache License 2.0 5 votes vote down vote up
def test_legacy_repeated_structured_property_uneven():
        class OtherKind(model.Model):
            foo = model.IntegerProperty()
            bar = model.StringProperty()

        class ThisKind(model.Model):
            baz = model.StructuredProperty(OtherKind, repeated=True)
            copacetic = model.BooleanProperty()

        key = datastore.Key("ThisKind", 123, project="testing")
        datastore_entity = datastore.Entity(key=key)
        datastore_entity.items = mock.Mock(
            return_value=(
                # Order counts for coverage
                ("baz.foo", [42, 144]),
                ("baz.bar", ["himom", "hellodad", "iminjail"]),
                ("copacetic", True),
            )
        )

        entity = model._entity_from_ds_entity(datastore_entity)
        assert isinstance(entity, ThisKind)
        assert entity.baz[0].foo == 42
        assert entity.baz[0].bar == "himom"
        assert entity.baz[1].foo == 144
        assert entity.baz[1].bar == "hellodad"
        assert entity.baz[2].foo is None
        assert entity.baz[2].bar == "iminjail"
        assert entity.copacetic is True 
Example 49
Project: python-ndb   Author: googleapis   File: test_model.py    Apache License 2.0 5 votes vote down vote up
def test_override_property():
        class ThatKind(model.Model):
            a = model.StringProperty()

        class ThisKind(ThatKind):
            a = model.IntegerProperty()
            b = model.BooleanProperty()
            c = model.PickleProperty()
            d = model.StringProperty(repeated=True)
            e = model.PickleProperty(repeated=True)
            notaproperty = True

        dill = {"sandwiches": ["turkey", "reuben"], "not_sandwiches": "tacos"}
        gherkin = [{"a": {"b": "c"}, "d": 0}, [1, 2, 3], "himom"]
        key = key_module.Key("ThisKind", 123, app="testing")

        entity = ThisKind(
            key=key,
            a=42,
            c=gherkin,
            d=["foo", "bar", "baz"],
            e=[gherkin, dill],
        )

        entity_pb = model._entity_to_protobuf(entity)
        assert isinstance(entity_pb, ds_types.Entity)
        assert entity_pb.properties["a"].integer_value == 42
        assert entity_pb.properties["b"].null_value == 0
        assert pickle.loads(entity_pb.properties["c"].blob_value) == gherkin
        d_values = entity_pb.properties["d"].array_value.values
        assert d_values[0].string_value == "foo"
        assert d_values[1].string_value == "bar"
        assert d_values[2].string_value == "baz"
        e_values = entity_pb.properties["e"].array_value.values
        assert pickle.loads(e_values[0].blob_value) == gherkin
        assert pickle.loads(e_values[1].blob_value) == dill
        assert "__key__" not in entity_pb.properties 
Example 50
Project: python-ndb   Author: googleapis   File: test_polymodel.py    Apache License 2.0 5 votes vote down vote up
def test_entity_from_protobuf():
        class Animal(polymodel.PolyModel):
            pass

        class Cat(Animal):
            pass

        key = datastore.Key("Animal", 123, project="testing")
        datastore_entity = datastore.Entity(key=key)
        datastore_entity["class"] = ["Animal", "Cat"]
        protobuf = helpers.entity_to_protobuf(datastore_entity)
        entity = model._entity_from_protobuf(protobuf)
        assert isinstance(entity, Cat) 
Example 51
Project: python-ndb   Author: googleapis   File: conftest.py    Apache License 2.0 5 votes vote down vote up
def ds_entity(with_ds_client, dispose_of):
    def make_entity(*key_args, **entity_kwargs):
        key = with_ds_client.key(*key_args)
        assert with_ds_client.get(key) is None
        entity = datastore.Entity(key=key)
        entity.update(entity_kwargs)
        with_ds_client.put(entity)
        dispose_of(key)

        return entity

    yield make_entity 
Example 52
Project: python-ndb   Author: googleapis   File: conftest.py    Apache License 2.0 5 votes vote down vote up
def ds_entity_with_meanings(with_ds_client, dispose_of):
    def make_entity(*key_args, **entity_kwargs):
        meanings = key_args[0]
        key = with_ds_client.key(*key_args[1:])
        assert with_ds_client.get(key) is None
        entity = datastore.Entity(key=key, exclude_from_indexes=("blob",))
        entity._meanings = meanings
        entity.update(entity_kwargs)
        with_ds_client.put(entity)
        dispose_of(key)

        return entity

    yield make_entity 
Example 53
Project: openmic-annotator   Author: cosmir   File: database.py    MIT License 5 votes vote down vote up
def put(self, uri, record, exclude_from_indexes=None):
        """Put a record into the database."""
        exclude_from_indexes = exclude_from_indexes or []

        # Create Entity from record + key
        kind, gid = urilib.split(uri)
        key = self._client.key(kind, gid)

        entity = datastore.Entity(
            key, exclude_from_indexes=exclude_from_indexes)
        entity.update(record)
        self._client.put(entity) 
Example 54
Project: green-spider   Author: netzbegruenung   File: spider.py    Apache License 2.0 5 votes vote down vote up
def execute_single_job(datastore_client, job, entity_kind):
    """
    Executes spider for one single job
    """
    validate_job(job)

    logging.info("Starting job %s", job["url"])
    result = check_and_rate_site(entry=job)

    logging.debug("Full JSON representation of returned result: %s", json.dumps(result, default=str))

    logging.info("Job %s finished checks", job["url"])
    logging.info("Job %s writing to DB", job["url"])

    key = datastore_client.key(entity_kind, job["url"])
    entity = datastore.Entity(key=key)
    record = {
        'created': datetime.utcnow(),
        'meta': result['meta'],
        'checks': result['checks'],
        'rating': result['rating'],
        'score': result['score'],
    }
    entity.update(record)
    try:
        datastore_client.put(entity)
    except InvalidArgument as ex:
        logging.error("Could not write result: %s", ex)
    except Exception as ex:
        logging.error("Could not write result: %s", ex) 
Example 55
Project: pulse-data   Author: Recidiviz   File: secrets.py    GNU General Public License v3.0 5 votes vote down vote up
def set_secret(name, value):
    secret = datastore.Entity(key=ds().key(SECRET_KIND, name))
    secret.update({'name': name, 'value': value})
    ds().put(secret) 
Example 56
Project: pulse-data   Author: Recidiviz   File: datastore_ingest_info.py    GNU General Public License v3.0 5 votes vote down vote up
def new(cls, key, session_start_time=None,
            region=None, ingest_info=None, task_hash=None, error=None,
            trace_id=None):
        new_ingest_info = cls(datastore.Entity(key))
        batch_ingest_info_data = BatchIngestInfoData(task_hash=task_hash,
                                                     ingest_info=ingest_info,
                                                     error=error,
                                                     trace_id=trace_id)
        # pylint: disable=protected-access
        new_ingest_info._entity['region']: str = region
        new_ingest_info._entity['batch_ingest_info_data']: str = \
            batch_ingest_info_data.to_serializable()
        new_ingest_info._entity['session_start_time']: datetime = \
            session_start_time
        return new_ingest_info 
Example 57
Project: pulse-data   Author: Recidiviz   File: datastore_ingest_info.py    GNU General Public License v3.0 5 votes vote down vote up
def _divide_into_chunks(results: List[datastore.Entity], chunk_size: int) -> \
        List[List[datastore.Entity]]:
    list_of_chunks = []
    for i in range(0, len(results), chunk_size):
        chunk = results[i: i + chunk_size]
        list_of_chunks.append(chunk)
    return list_of_chunks 
Example 58
Project: pulse-data   Author: Recidiviz   File: datastore_ingest_info.py    GNU General Public License v3.0 5 votes vote down vote up
def _batch_ingest_info_data_from_entities(
        entity_list: List[datastore.Entity]) -> List[BatchIngestInfoData]:
    return [_DatastoreIngestInfo.get_batch_ingest_info_data(entity) for entity
            in entity_list] 
Example 59
Project: pulse-data   Author: Recidiviz   File: sessions.py    GNU General Public License v3.0 5 votes vote down vote up
def _sessions_from_entities(entity_generator: Iterator[datastore.Entity]) \
        -> Iterator[ScrapeSession]:
    return (ScrapeSession.from_entity(entity) for entity in entity_generator) 
Example 60
Project: pulse-data   Author: Recidiviz   File: sessions.py    GNU General Public License v3.0 5 votes vote down vote up
def new(cls, key, record_id=None, region=None, created_on=None):
        entity = datastore.Entity(key)
        entity['record_id'] = record_id  # type: string
        entity['region'] = region  # type: string
        entity['created_on']: datetime = \
            created_on if created_on else datetime.now()

        return cls(entity) 
Example 61
Project: turbinia   Author: google   File: state_manager.py    Apache License 2.0 5 votes vote down vote up
def write_new_task(self, task):
    key = self.client.key('TurbiniaTask', task.id)
    try:
      entity = datastore.Entity(key)
      entity.update(self.get_task_dict(task))
      log.info('Writing new task {0:s} into Datastore'.format(task.name))
      self.client.put(entity)
      task.state_key = key
    except exceptions.GoogleCloudError as e:
      log.error(
          'Failed to update task {0:s} in datastore: {1!s}'.format(
              task.name, e))
    return key 
Example 62
Project: arXie-Bot   Author: thundergolfer   File: accounts.py    MIT License 5 votes vote down vote up
def update_with_user(self, team, slack_user, username, pw):
        client = datastore.Client(self.project_id, namespace=team)

        complete_key = client.key('Login', slack_user)

        task = datastore.Entity(key=complete_key)

        task.update({
            'username': username,
            'password': pw,
        })

        client.put(task) 
Example 63
Project: python-docs-samples   Author: GoogleCloudPlatform   File: snippets.py    Apache License 2.0 5 votes vote down vote up
def basic_entity(client):
    # [START datastore_basic_entity]
    task = datastore.Entity(client.key('Task'))
    task.update({
        'category': 'Personal',
        'done': False,
        'priority': 4,
        'description': 'Learn Cloud Datastore'
    })
    # [END datastore_basic_entity]

    return task 
Example 64
Project: python-docs-samples   Author: GoogleCloudPlatform   File: snippets.py    Apache License 2.0 5 votes vote down vote up
def projection_query(client):
    # Create the entity that we're going to query.
    task = datastore.Entity(client.key('Task'))
    task.update({
        'category': 'Personal',
        'done': False,
        'priority': 4,
        'description': 'Learn Cloud Datastore',
        'percent_complete': 0.5
    })
    client.put(task)

    # [START datastore_projection_query]
    query = client.query(kind='Task')
    query.projection = ['priority', 'percent_complete']
    # [END datastore_projection_query]

    # [START datastore_run_query_projection]
    priorities = []
    percent_completes = []

    for task in query.fetch():
        priorities.append(task['priority'])
        percent_completes.append(task['percent_complete'])
    # [END datastore_run_query_projection]

    return priorities, percent_completes 
Example 65
Project: python-docs-samples   Author: GoogleCloudPlatform   File: main.py    Apache License 2.0 5 votes vote down vote up
def index():
    ds = datastore.Client()

    user_ip = request.remote_addr

    # Keep only the first two octets of the IP address.
    if is_ipv6(user_ip):
        user_ip = ':'.join(user_ip.split(':')[:2])
    else:
        user_ip = '.'.join(user_ip.split('.')[:2])

    entity = datastore.Entity(key=ds.key('visit'))
    entity.update({
        'user_ip': user_ip,
        'timestamp': datetime.datetime.utcnow()
    })

    ds.put(entity)

    query = ds.query(kind='visit', order=('-timestamp',))

    results = [
        'Time: {timestamp} Addr: {user_ip}'.format(**x)
        for x in query.fetch(limit=10)]

    output = 'Last 10 visits:\n{}'.format('\n'.join(results))

    return output, 200, {'Content-Type': 'text/plain; charset=utf-8'}
# [END gae_flex_datastore_app] 
Example 66
Project: python-docs-samples   Author: GoogleCloudPlatform   File: main.py    Apache License 2.0 5 votes vote down vote up
def store_time(dt):
    entity = datastore.Entity(key=datastore_client.key('visit'))
    entity.update({
        'timestamp': dt
    })

    datastore_client.put(entity) 
Example 67
Project: python-docs-samples   Author: GoogleCloudPlatform   File: main.py    Apache License 2.0 5 votes vote down vote up
def store_time(dt):
    entity = datastore.Entity(key=datastore_client.key('visit'))
    entity.update({
        'timestamp': dt
    })

    datastore_client.put(entity) 
Example 68
Project: python-docs-samples   Author: GoogleCloudPlatform   File: blog.py    Apache License 2.0 5 votes vote down vote up
def create_user(ds, username, profile):
    key = path_to_key(ds, '{0}.user'.format(username))
    entity = datastore.Entity(key)
    entity.update(profile)
    ds.put(entity) 
Example 69
Project: python-docs-samples   Author: GoogleCloudPlatform   File: blog.py    Apache License 2.0 5 votes vote down vote up
def create_post(ds, username, post_content):
    now = datetime.datetime.utcnow()
    key = path_to_key(ds, '{0}.user/{1}.post'.format(username, now))
    entity = datastore.Entity(key)

    entity.update({
        'created': now,
        'created_by': username,
        'content': post_content
    })

    ds.put(entity) 
Example 70
Project: psq   Author: GoogleCloudPlatform   File: datastore_storage.py    Apache License 2.0 5 votes vote down vote up
def put_task(self, task):
        if task.status not in self.store_on_status:
            return

        entity = datastore.Entity(
            key=self._get_task_key(task.id),
            exclude_from_indexes=('data',))

        entity['data'] = dumps(task)
        entity['timestamp'] = datetime.utcnow()

        self.datastore.put(entity) 
Example 71
Project: eclipse2017   Author: google   File: location_test.py    Apache License 2.0 5 votes vote down vote up
def _setup_user(self, userid_hash):
        json = {"email": "[email protected]"}

        key = self.datastore_client.key("User", userid_hash)
        entity = datastore.Entity(key=key)
        util._update_entity(json, {'email'}, entity)
        self.datastore_client.put(entity) 
Example 72
Project: eclipse2017   Author: google   File: count_test.py    Apache License 2.0 5 votes vote down vote up
def _setup_user(self, userid_hash):
        json = {"email": "[email protected]"}

        key = self.datastore_client.key("User", userid_hash)
        entity = datastore.Entity(key=key)
        util._update_entity(json, {'email'}, entity)
        self.datastore_client.put(entity) 
Example 73
Project: eclipse2017   Author: google   File: site_test.py    Apache License 2.0 5 votes vote down vote up
def _create_user_via_datastore(self, userid_hash):
        user = datastore.Entity(key = self.client.key("User", userid_hash))
        user['name'] = u"Test User " + userid_hash
        user['email'] = u"test" + userid_hash + u"@example.com"
        users.create_or_update_user(self.client, user)
        roles.create_user_role(self.client, userid_hash) 
Example 74
Project: eclipse2017   Author: google   File: users.py    Apache License 2.0 5 votes vote down vote up
def get_empty_user_entity(client, user_id):
    key = client.key("User", user_id)
    entity = datastore.Entity(key=key)
    return entity 
Example 75
Project: eclipse2017   Author: google   File: add_movie.py    Apache License 2.0 5 votes vote down vote up
def main():
    args  = get_arguments()
    client = datastore.Client(project=args.project_id)
    key = client.key("Movie")
    entity = datastore.Entity(key = key)
    entity.key = key
    entity.update({'id': args.id,
                   'time': datetime.datetime.utcnow()})
    client.put(entity)
    f = open(args.outfile, "w")
    f.write("var movie_id = '%s';\n" % args.id)
    f.close() 
Example 76
Project: clusterfuzz   Author: google   File: ndb_patcher.py    Apache License 2.0 4 votes vote down vote up
def _ndb_entity_to_cloud_entity(ndb_entity):
  """Convert ndb.Model to cloud entity to prepare for put()."""
  if ndb_entity is None:
    return None

  project_id = utils.get_application_id()
  unindexed_properties = _unindexed_properties(ndb_entity.__class__)

  ndb_entity._prepare_for_put()
  ndb_entity._pre_put_hook()

  if ndb_entity.key:
    # Existing key.
    cloud_entity = datastore.Entity(
        key=_ndb_key_to_cloud_key(ndb_entity.key),
        exclude_from_indexes=unindexed_properties)
  else:
    # Auto-generate key.
    base_key = datastore.Key(ndb_entity._get_kind(), project=project_id)
    generated_key = _retry_wrap(_client().allocate_ids)(base_key, 1)[0]
    cloud_entity = datastore.Entity(
        key=generated_key, exclude_from_indexes=unindexed_properties)

    ndb_entity.key = _cloud_key_to_ndb_key(generated_key)

  for key, value in six.iteritems(ndb_entity.to_dict()):
    ndb_property = getattr(ndb_entity.__class__, key)
    if type(ndb_property) in UNSUPPORTED_PROPERTY_TYPES:
      raise NdbPatcherException('Unsupported property type: ' +
                                ndb_property.__name__)

    if (isinstance(value, str) and type(ndb_property) is not ndb.BlobProperty):
      # All 'str' values are written as byte strings by Cloud Datastore, but ndb
      # ndb_entitys can have 'str' values for StringProperty or TextProperty, so
      # check the type of the property.
      value = unicode(value)
    elif type(value) is datetime.date:
      value = datetime.datetime.combine(value, datetime.datetime.min.time())
    elif type(value) is datetime.time:
      value = datetime.datetime.combine(datetime.date(1970, 1, 1), value)
    elif isinstance(value, ndb.Key):
      value = _ndb_key_to_cloud_key(value)

    cloud_entity[key] = value

  return cloud_entity 
Example 77
Project: twstockfilter   Author: matthklo   File: update_dividend_data.py    BSD 2-Clause "Simplified" License 4 votes vote down vote up
def commit_raw_data(entities, data):
    stock_dict = data[g_year]
    
    b = g_gdclient.batch()
    b.begin()
    bcnt = 0
    years = range(g_year, g_year - 6, -1)

    for s in stock_dict.iteritems():
        stock_id = s[0]

        # Build dd/rd content
        dd = {}
        rd = {}
        for y in years:
            if stock_id in data[y]:
                d = data[y][stock_id]
                dd[str(y % 100)] = [ d['cash_dividend'], d['stock_dividend'] ]
                rd[str(y % 100)] = [ d['profit'], d['eps'] ]

        if stock_id not in entities:
            # Need to create a new entity for this item
            e = datastore.Entity(key=g_gdclient.key('tw_stock_data',stock_id), exclude_from_indexes=('dd','rd'))
            e.update({
                'id': stock_id,
                'name': s[1]['name'],
                'dd': json.dumps(dd),
                'rd': json.dumps(rd)
            })
            entities[stock_id] = e
            b.put(e)
        else:
            e = entities[stock_id]
            e.update({'dd': json.dumps(dd), 'rd': json.dumps(rd)})
            b.put(e)

        bcnt += 1
        if bcnt >= 500:
            b.commit()
            b = g_gdclient.batch()
            b.begin()
            bcnt = 0
    
    if bcnt > 0:
        b.commit() 
Example 78
Project: Flask-Blogging   Author: gouthambs   File: gcdatastore.py    MIT License 4 votes vote down vote up
def save_post(self, title, text, user_id, tags, draft=False,
                  post_date=None, last_modified_date=None, meta_data=None,
                  post_id=None):
        if post_id is not None:
            update_op = True
        else:
            update_op = False

        post_id = post_id or self._get_new_post_id()
        current_datetime = datetime.datetime.utcnow()
        post_date = post_date or current_datetime
        last_modified_date = last_modified_date or current_datetime
        tags = self.normalize_tags(tags)
        draft = True if draft else False

        if not update_op:
            key = self._client.key('Post', int(post_id))
            post = datastore.Entity(key=key, exclude_from_indexes=['text'])
            post.update({
                    'title': title,
                    'text': text,
                    'user_id': user_id,
                    'tags': tags or [],
                    'draft': draft,
                    'post_date': post_date,
                    'last_modified_date': last_modified_date,
                    'meta_data': meta_data,
                    'post_id': int(post_id)
            })
            self._client.put(post)
            return post_id
        else:
            key = self._client.key('Post', int(post_id))
            post = self._client.get(key)
            if not post:
                post_id = self._get_new_post_id()
                key = self._client.key('Post', int(post_id))
                post = datastore.Entity(key=key, exclude_from_indexes=['text'])
            post.update({
                    'title': title,
                    'text': text,
                    'user_id': user_id,
                    'tags': tags or [],
                    'draft': draft,
                    'post_date': post_date,
                    'last_modified_date': last_modified_date,
                    'meta_data': meta_data,
                    'post_id': int(post_id)
            })
            self._client.put(post)
            return int(post_id) 
Example 79
Project: eclipse2017   Author: google   File: pipeline.py    Apache License 2.0 4 votes vote down vote up
def upload(self, fnames):
        uploaded_files = []

        bucket = self.storage.get_bucket(config.GCS_PROCESSED_PHOTOS_BUCKET)
        batch = self.datastore.batch()
        batch.begin()

        for fname in fnames:
            name, ext = os.path.splitext(fname)
            fpath = '{0}/{1}{2}'.format(constants.IMAGE_PROCESSOR_DATA_DIR, name, ext)
            objname = '{0}{1}'.format(name, ext)
            blob = storage.Blob(objname, bucket)
            try:
                blob.upload_from_file(open(fpath, "rb"))
                uploaded_files.append(fname)
                msg = 'Successfully uploaded {0} to Cloud Storage'
                logging.info(msg.format(fname))
            except Exception, e:
                msg = 'Failed to upload {0} to Cloud Storage: {1}'
                logging.error(msg.format(fname, e))
            else:
                # Update original photo entity
                photo_key = self.datastore.key(ds.DATASTORE_PHOTO, fname)
                photo_entity = self.datastore.get(photo_key)
                photo_entity.update({'processed': True})
                batch.put(photo_entity)

                # Create datastore entry for oriented image
                name, ext = os.path.splitext(fname)
                resized_fname = '{0}{1}'.format(name, ext)
                oriented_key = self.datastore.key(ds.DATASTORE_ORIENTED_IMAGE, resized_fname)
                oriented_entity = datastore.Entity(oriented_key)
                oriented_entity['original_photo'] = photo_key
                oriented_entity['image_type'] = unicode(ds.TOTALITY_IMAGE_TYPE)
                lat = photo_entity['lat']
                lon = photo_entity['lon']
                # TODO(dek): properly repsect LatRef and LonRef here
                lon = -lon
                p = Point(lat, lon)
                np = self.eclipse_gis.interpolate_nearest_point_on_line(p)
                # TODO(dek):
                # map each location into its associated center point
                # (based on the golden data in eclipse_gis)
                # and sort by location/time bins
                oriented_entity[ds.TOTALITY_ORDERING_PROPERTY] = np
                batch.put(oriented_entity)

        # Cloud Datastore API request 
Example 80
Project: eclipse2017   Author: google   File: rename_photo_table.py    Apache License 2.0 4 votes vote down vote up
def main():
    args  = get_arguments()

    client = datastore.Client(project=args.project_id)

    # Fetch all keys of the new Photo table
    query = client.query(kind="PhotoVolunteerTest")
    query.keys_only()

    cursor = None
    results = []
    while True:
        entities = query.fetch(start_cursor=cursor, limit=1000)
        l = list(entities)
        results.extend(l)
        if len(l) < 1000:
            break
        cursor = entities.next_page_token

    # Delete all keys in the PhotoVolunteerTest table
    result_chunks = chunks(results, 500)
    for result_chunk in result_chunks:
        batch = client.batch()
        batch.begin()
        for result in result_chunk:
            batch.delete(result.key)
        batch.commit()

    # Fetch all keys of the old Photo table
    query = client.query(kind="Photo")

    cursor = None
    results = []
    while True:
        entities = query.fetch(start_cursor=cursor, limit=1000)
        l = list(entities)
        results.extend(l)
        if len(l) < 1000:
            break
        cursor = entities.next_page_token

    # Copy all the keys from the Photo table to the PhotoVolunteerTest table
    result_chunks = chunks(results, 500)
    for result_chunk in result_chunks:
        batch = client.batch()
        batch.begin()
        for result in result_chunk:
            key = client.key("PhotoVolunteerTest", result.key.name)
            entity = datastore.Entity(key=key)
            # Fix datetimes not roundtripping properly (they are missing tz info)
            entity.update(result)
            batch.put(entity)
        batch.commit()