Python google.appengine.ext.ndb.LocalStructuredProperty() Examples

def get_property_converter(self, prop):
    """Returns callable that can convert values corresponding to ndb property.

    Args:
      prop: instance of ndb.Property subclass that defines typing information.

    Returns:
      Callable (property instance, incoming value) -> converter values.
    """
    # For structured properties, recursively call convert_dict.
    if isinstance(prop, (ndb.StructuredProperty, ndb.LocalStructuredProperty)):
      return lambda prop, x: self.convert_dict(prop._modelclass, x)
    # For other properties consult the registry of converters.
    for prop_cls, include_subclasses, conv in self.property_converters:
      # pylint: disable=unidiomatic-typecheck
      if (include_subclasses and isinstance(prop, prop_cls) or
          not include_subclasses and type(prop) == prop_cls):
        return conv
    # Give up.
    raise TypeError('Don\'t know how to work with %s' % type(prop).__name__)


### Public API. 

def _test_repeated_structured_properties_class(self, structured_cls):
    """Common testing for StructuredProperty and LocalStructuredProperty."""
    class Inner(ndb.Model):
      a = ndb.IntegerProperty()

    class Outter(ndb.Model, serializable.SerializableModelMixin):
      inner = structured_cls(Inner, repeated=True)

    # Repeated structured property -> list of dicts.
    entity = Outter()
    entity.inner.extend([Inner(a=1), Inner(a=2)])
    self.assertEqual(
        {'inner': [{'a': 1}, {'a': 2}]},
        entity.to_serializable_dict())

    # Reverse also works.
    self.assertEqual(
        entity,
        Outter.from_serializable_dict({'inner': [{'a': 1}, {'a': 2}]})) 

def get_property_converter(self, prop):
    """Returns callable that can convert values corresponding to ndb property.

    Args:
      prop: instance of ndb.Property subclass that defines typing information.

    Returns:
      Callable (property instance, incoming value) -> converter values.
    """
    # For structured properties, recursively call convert_dict.
    if isinstance(prop, (ndb.StructuredProperty, ndb.LocalStructuredProperty)):
      return lambda prop, x: self.convert_dict(prop._modelclass, x)
    # For other properties consult the registry of converters.
    for prop_cls, include_subclasses, conv in self.property_converters:
      # pylint: disable=unidiomatic-typecheck
      if (include_subclasses and isinstance(prop, prop_cls) or
          not include_subclasses and type(prop) == prop_cls):
        return conv
    # Give up.
    raise TypeError('Don\'t know how to work with %s' % type(prop).__name__)


### Public API. 

def _test_repeated_structured_properties_class(self, structured_cls):
    """Common testing for StructuredProperty and LocalStructuredProperty."""
    class Inner(ndb.Model):
      a = ndb.IntegerProperty()

    class Outter(ndb.Model, serializable.SerializableModelMixin):
      inner = structured_cls(Inner, repeated=True)

    # Repeated structured property -> list of dicts.
    entity = Outter()
    entity.inner.extend([Inner(a=1), Inner(a=2)])
    self.assertEqual(
        {'inner': [{'a': 1}, {'a': 2}]},
        entity.to_serializable_dict())

    # Reverse also works.
    self.assertEqual(
        entity,
        Outter.from_serializable_dict({'inner': [{'a': 1}, {'a': 2}]})) 

def _test_repeated_structured_properties_class(self, structured_cls):
    """Common testing for StructuredProperty and LocalStructuredProperty."""
    class Inner(ndb.Model):
      a = ndb.IntegerProperty()

    class Outter(ndb.Model, serializable.SerializableModelMixin):
      inner = structured_cls(Inner, repeated=True)

    # Repeated structured property -> list of dicts.
    entity = Outter()
    entity.inner.extend([Inner(a=1), Inner(a=2)])
    self.assertEqual(
        {'inner': [{'a': 1}, {'a': 2}]},
        entity.to_serializable_dict())

    # Reverse also works.
    self.assertEqual(
        entity,
        Outter.from_serializable_dict({'inner': [{'a': 1}, {'a': 2}]})) 

def get_property_converter(self, prop):
    """Returns callable that can convert values corresponding to ndb property.

    Args:
      prop: instance of ndb.Property subclass that defines typing information.

    Returns:
      Callable (property instance, incoming value) -> converter values.
    """
    # For structured properties, recursively call convert_dict.
    if isinstance(prop, (ndb.StructuredProperty, ndb.LocalStructuredProperty)):
      return lambda prop, x: self.convert_dict(prop._modelclass, x)
    # For other properties consult the registry of converters.
    for prop_cls, include_subclasses, conv in self.property_converters:
      # pylint: disable=unidiomatic-typecheck
      if (include_subclasses and isinstance(prop, prop_cls) or
          not include_subclasses and type(prop) == prop_cls):
        return conv
    # Give up.
    raise TypeError('Don\'t know how to work with %s' % type(prop).__name__)


### Public API. 

def get_property_converter(self, prop):
    """Returns callable that can convert values corresponding to ndb property.

    Args:
      prop: instance of ndb.Property subclass that defines typing information.

    Returns:
      Callable (property instance, incoming value) -> converter values.
    """
    # For structured properties, recursively call convert_dict.
    if isinstance(prop, (ndb.StructuredProperty, ndb.LocalStructuredProperty)):
      return lambda prop, x: self.convert_dict(prop._modelclass, x)
    # For other properties consult the registry of converters.
    for prop_cls, include_subclasses, conv in self.property_converters:
      # pylint: disable=unidiomatic-typecheck
      if (include_subclasses and isinstance(prop, prop_cls) or
          not include_subclasses and type(prop) == prop_cls):
        return conv
    # Give up.
    raise TypeError('Don\'t know how to work with %s' % type(prop).__name__)


### Public API. 

def _test_repeated_structured_properties_class(self, structured_cls):
    """Common testing for StructuredProperty and LocalStructuredProperty."""
    class Inner(ndb.Model):
      a = ndb.IntegerProperty()

    class Outter(ndb.Model, serializable.SerializableModelMixin):
      inner = structured_cls(Inner, repeated=True)

    # Repeated structured property -> list of dicts.
    entity = Outter()
    entity.inner.extend([Inner(a=1), Inner(a=2)])
    self.assertEqual(
        {'inner': [{'a': 1}, {'a': 2}]},
        entity.to_serializable_dict())

    # Reverse also works.
    self.assertEqual(
        entity,
        Outter.from_serializable_dict({'inner': [{'a': 1}, {'a': 2}]})) 

def _test_repeated_structured_properties_class(self, structured_cls):
    """Common testing for StructuredProperty and LocalStructuredProperty."""
    class Inner(ndb.Model):
      a = ndb.IntegerProperty()

    class Outter(ndb.Model, serializable.SerializableModelMixin):
      inner = structured_cls(Inner, repeated=True)

    # Repeated structured property -> list of dicts.
    entity = Outter()
    entity.inner.extend([Inner(a=1), Inner(a=2)])
    self.assertEqual(
        {'inner': [{'a': 1}, {'a': 2}]},
        entity.to_serializable_dict())

    # Reverse also works.
    self.assertEqual(
        entity,
        Outter.from_serializable_dict({'inner': [{'a': 1}, {'a': 2}]})) 

def test_local_structured_properties(self):
    """Test handling of LocalStructuredProperty."""
    self._test_structured_properties_class(ndb.LocalStructuredProperty) 

def test_local_structured_properties(self):
    """Test handling of LocalStructuredProperty."""
    self._test_structured_properties_class(ndb.LocalStructuredProperty) 

def _test_structured_properties_class(self, structured_cls):
    """Common testing for StructuredProperty and LocalStructuredProperty."""
    # Plain ndb.Model.
    class InnerSimple(ndb.Model):
      a = ndb.IntegerProperty()

    # With SerializableModelMixin.
    class InnerSmart(ndb.Model, serializable.SerializableModelMixin):
      serializable_properties = {
        'a': serializable.READABLE | serializable.WRITABLE,
      }
      a = ndb.IntegerProperty()
      b = ndb.IntegerProperty()

    class Outter(ndb.Model, serializable.SerializableModelMixin):
      simple = structured_cls(InnerSimple)
      smart = structured_cls(InnerSmart)

    # InnerSimple gets serialized entirely, while only readable fields
    # on InnerSmart are serialized.
    entity = Outter()
    entity.simple = InnerSimple(a=1)
    entity.smart = InnerSmart(a=2, b=3)
    self.assertEqual(
        {'simple': {'a': 1}, 'smart': {'a': 2}},
        entity.to_serializable_dict())

    # Works backwards as well. Note that 'convert_serializable_dict' returns
    # a dictionary that can be fed to entity's 'populate' or constructor. Entity
    # by itself is smart enough to transform subdicts into structured
    # properties.
    self.assertEqual(
        Outter(simple=InnerSimple(a=1), smart=InnerSmart(a=2)),
        Outter.from_serializable_dict({'simple': {'a': 1}, 'smart': {'a': 2}})) 

def __init__(self, **kwargs):
    # This is the recommended way to use ndb.LocalStructuredProperty inside a
    # snapshot.
    #
    # Warning: The only reason it works is because Snapshot is itself inside a
    # ndb.LocalStructuredProperty.
    kwargs.setdefault('inner', InnerSnapshot())
    super(Snapshot, self).__init__(**kwargs) 

def test_repeated_local_structured_properties(self):
    """Test handling of LocalStructuredProperty(repeated=True)."""
    self._test_repeated_structured_properties_class(ndb.LocalStructuredProperty) 

def _test_structured_properties_class(self, structured_cls):
    """Common testing for StructuredProperty and LocalStructuredProperty."""
    # Plain ndb.Model.
    class InnerSimple(ndb.Model):
      a = ndb.IntegerProperty()

    # With SerializableModelMixin.
    class InnerSmart(ndb.Model, serializable.SerializableModelMixin):
      serializable_properties = {
        'a': serializable.READABLE | serializable.WRITABLE,
      }
      a = ndb.IntegerProperty()
      b = ndb.IntegerProperty()

    class Outter(ndb.Model, serializable.SerializableModelMixin):
      simple = structured_cls(InnerSimple)
      smart = structured_cls(InnerSmart)

    # InnerSimple gets serialized entirely, while only readable fields
    # on InnerSmart are serialized.
    entity = Outter()
    entity.simple = InnerSimple(a=1)
    entity.smart = InnerSmart(a=2, b=3)
    self.assertEqual(
        {'simple': {'a': 1}, 'smart': {'a': 2}},
        entity.to_serializable_dict())

    # Works backwards as well. Note that 'convert_serializable_dict' returns
    # a dictionary that can be fed to entity's 'populate' or constructor. Entity
    # by itself is smart enough to transform subdicts into structured
    # properties.
    self.assertEqual(
        Outter(simple=InnerSimple(a=1), smart=InnerSmart(a=2)),
        Outter.from_serializable_dict({'simple': {'a': 1}, 'smart': {'a': 2}})) 

def test_repeated_local_structured_properties(self):
    """Test handling of LocalStructuredProperty(repeated=True)."""
    self._test_repeated_structured_properties_class(ndb.LocalStructuredProperty) 

def __init__(self, **kwargs):
    # This is the recommended way to use ndb.LocalStructuredProperty inside a
    # snapshot.
    #
    # Warning: The only reason it works is because Snapshot is itself inside a
    # ndb.LocalStructuredProperty.
    kwargs.setdefault('inner', InnerSnapshot())
    super(Snapshot, self).__init__(**kwargs) 

def _test_structured_properties_class(self, structured_cls):
    """Common testing for StructuredProperty and LocalStructuredProperty."""
    # Plain ndb.Model.
    class InnerSimple(ndb.Model):
      a = ndb.IntegerProperty()

    # With SerializableModelMixin.
    class InnerSmart(ndb.Model, serializable.SerializableModelMixin):
      serializable_properties = {
        'a': serializable.READABLE | serializable.WRITABLE,
      }
      a = ndb.IntegerProperty()
      b = ndb.IntegerProperty()

    class Outter(ndb.Model, serializable.SerializableModelMixin):
      simple = structured_cls(InnerSimple)
      smart = structured_cls(InnerSmart)

    # InnerSimple gets serialized entirely, while only readable fields
    # on InnerSmart are serialized.
    entity = Outter()
    entity.simple = InnerSimple(a=1)
    entity.smart = InnerSmart(a=2, b=3)
    self.assertEqual(
        {'simple': {'a': 1}, 'smart': {'a': 2}},
        entity.to_serializable_dict())

    # Works backwards as well. Note that 'convert_serializable_dict' returns
    # a dictionary that can be fed to entity's 'populate' or constructor. Entity
    # by itself is smart enough to transform subdicts into structured
    # properties.
    self.assertEqual(
        Outter(simple=InnerSimple(a=1), smart=InnerSmart(a=2)),
        Outter.from_serializable_dict({'simple': {'a': 1}, 'smart': {'a': 2}})) 

def __init__(self, **kwargs):
    # This is the recommended way to use ndb.LocalStructuredProperty inside a
    # snapshot.
    #
    # Warning: The only reason it works is because Snapshot is itself inside a
    # ndb.LocalStructuredProperty.
    kwargs.setdefault('inner', InnerSnapshot())
    super(Snapshot, self).__init__(**kwargs) 

def test_repeated_local_structured_properties(self):
    """Test handling of LocalStructuredProperty(repeated=True)."""
    self._test_repeated_structured_properties_class(ndb.LocalStructuredProperty) 

def _test_structured_properties_class(self, structured_cls):
    """Common testing for StructuredProperty and LocalStructuredProperty."""
    # Plain ndb.Model.
    class InnerSimple(ndb.Model):
      a = ndb.IntegerProperty()

    # With SerializableModelMixin.
    class InnerSmart(ndb.Model, serializable.SerializableModelMixin):
      serializable_properties = {
        'a': serializable.READABLE | serializable.WRITABLE,
      }
      a = ndb.IntegerProperty()
      b = ndb.IntegerProperty()

    class Outter(ndb.Model, serializable.SerializableModelMixin):
      simple = structured_cls(InnerSimple)
      smart = structured_cls(InnerSmart)

    # InnerSimple gets serialized entirely, while only readable fields
    # on InnerSmart are serialized.
    entity = Outter()
    entity.simple = InnerSimple(a=1)
    entity.smart = InnerSmart(a=2, b=3)
    self.assertEqual(
        {'simple': {'a': 1}, 'smart': {'a': 2}},
        entity.to_serializable_dict())

    # Works backwards as well. Note that 'convert_serializable_dict' returns
    # a dictionary that can be fed to entity's 'populate' or constructor. Entity
    # by itself is smart enough to transform subdicts into structured
    # properties.
    self.assertEqual(
        Outter(simple=InnerSimple(a=1), smart=InnerSmart(a=2)),
        Outter.from_serializable_dict({'simple': {'a': 1}, 'smart': {'a': 2}})) 

def test_local_structured_properties(self):
    """Test handling of LocalStructuredProperty."""
    self._test_structured_properties_class(ndb.LocalStructuredProperty) 

def __init__(self, **kwargs):
    # This is the recommended way to use ndb.LocalStructuredProperty inside a
    # snapshot.
    #
    # Warning: The only reason it works is because Snapshot is itself inside a
    # ndb.LocalStructuredProperty.
    kwargs.setdefault('inner', InnerSnapshot())
    super(Snapshot, self).__init__(**kwargs) 

def test_local_structured_properties(self):
    """Test handling of LocalStructuredProperty."""
    self._test_structured_properties_class(ndb.LocalStructuredProperty) 

def to_dict(self):
    """Supports both old and new format."""
    # to_dict() doesn't recurse correctly into ndb.LocalStructuredProperty! It
    # will call the default method and not the overridden one. :(
    out = super(TaskRequest, self).to_dict(exclude=[
        'manual_tags', 'properties_old', 'pubsub_auth_token',
        'resultdb_update_token', 'service_account_token', 'task_slice'
    ])
    if self.properties_old:
      out['properties'] = self.properties_old.to_dict()
    if self.task_slices:
      out['task_slices'] = [t.to_dict() for t in self.task_slices]
    return out 

def _pre_put_hook(self):
    # _pre_put_hook() doesn't recurse correctly into
    # ndb.LocalStructuredProperty. Call the function manually.
    super(TaskSlice, self)._pre_put_hook()
    self.properties._pre_put_hook()
    if self.wait_for_capacity is None:
      raise datastore_errors.BadValueError('wait_for_capacity is required') 

def to_dict(self):
    # to_dict() doesn't recurse correctly into ndb.LocalStructuredProperty! It
    # will call the default method and not the overridden one. :(
    out = super(TaskSlice, self).to_dict(exclude=['properties'])
    out['properties'] = self.properties.to_dict()
    return out 

def _pre_put_hook(self):
    if not self.server:
      raise datastore_errors.BadValueError('cipd server is required')
    if not self.client_package:
      raise datastore_errors.BadValueError('client_package is required')
    if self.client_package.path:
      raise datastore_errors.BadValueError('client_package.path must be unset')
    # _pre_put_hook() doesn't recurse correctly into
    # ndb.LocalStructuredProperty. Call the function manually.
    self.client_package._pre_put_hook()

    if not self.packages:
      raise datastore_errors.BadValueError(
          'cipd_input cannot have an empty package list')
    if len(self.packages) > 64:
      raise datastore_errors.BadValueError(
          'Up to 64 CIPD packages can be listed for a task')

    # Make sure we don't install multiple versions of the same package at the
    # same path.
    package_path_names = set()
    for p in self.packages:
      # _pre_put_hook() doesn't recurse correctly into
      # ndb.LocalStructuredProperty. Call the function manually.
      p._pre_put_hook()
      if not p.path:
        raise datastore_errors.BadValueError(
            'package %s:%s: path is required' % (p.package_name, p.version))
      path_name = (p.path, p.package_name)
      if path_name in package_path_names:
        raise datastore_errors.BadValueError(
           'package %r is specified more than once in path %r'
          % (p.package_name, p.path))
      package_path_names.add(path_name)
    self.packages.sort(key=lambda p: (p.path, p.package_name)) 

def test_repeated_local_structured_properties(self):
    """Test handling of LocalStructuredProperty(repeated=True)."""
    self._test_repeated_structured_properties_class(ndb.LocalStructuredProperty) 

def accumulate(lhs, rhs, skip):
  """Adds the values from rhs into lhs.

  Arguments:
  - lhs: in/out ndb.Model that the sum is accumulated into.
  - rhs: in ndb.Model that is accumulated to lhs.
  - skip: list or set of property names to skip.

  Both must be an ndb.Model. lhs is modified. rhs is not.

  rhs._properties not in lhs._properties are lost.
  lhs._properties not in rhs._properties are untouched.

  This function has specific handling for ndb.LocalStructuredProperty, it
  refuses any instance with a default value. THIS IS A TRAP. The default object
  instance will be aliased one way or another. It's just not worth the risk.
  """
  assert isinstance(lhs, ndb.Model), lhs
  assert isinstance(rhs, ndb.Model), rhs

  # Access to a protected member NNN of a client class
  # pylint: disable=W0212
  for key in set(lhs._properties).intersection(rhs._properties):
    if key in skip:
      continue
    if hasattr(lhs, key) and hasattr(rhs, key):
      # Repeated properties have to be handled manually.
      assert not lhs._properties[key]._repeated, key
      default = lhs._properties[key]._default
      lhs_value = getattr(lhs, key, default)
      rhs_value = getattr(rhs, key, default)
      if hasattr(lhs_value, 'accumulate'):
        # Do not use ndb.LocalStructuredProperty(MyClass, default=MyClass())
        # since any object created without specifying a object for this property
        # will get the exact instance provided as the default argument, it's
        # dangerous aliasing. See the unit test for a way to set a default value
        # that is safe.
        assert default is None, key
        assert callable(lhs_value.accumulate), key
        lhs_value.accumulate(rhs_value)
      else:
        # Assume X + None == X for any type.
        if lhs_value is None:
          value = rhs_value
        elif rhs_value is None:
          value = lhs_value
        else:
          value = lhs_value + rhs_value
        try:
          setattr(lhs, key, value)
        except AttributeError:
          # This happens when a structure changes and the old entity is being
          # summed to.
          logging.error('Couldn\'t set %s to %s', key, value)