Python google.appengine.api.memcache.set() Examples

The following are 30 code examples for showing how to use google.appengine.api.memcache.set(). These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example.

You may check out the related API usage on the sidebar.

You may also want to check out all available functions/classes of the module google.appengine.api.memcache , or try the search function .

Example 1
Project: browserscope   Author: elsigh   File: util.py    License: Apache License 2.0 6 votes vote down vote up
def GvizTableData(request):
    """Returns a string formatted for consumption by a Google Viz table."""
    #def throw_deadline():
    #    logging.info('MANUAL THROW!! DeadlineExceededError DeadlineExceededError')
    #    raise DeadlineExceededError
    #t = Timer(15.0, throw_deadline)

    test_set = None
    category = request.GET.get('category')
    if not category:
        return http.HttpResponseBadRequest('Must pass category=something')

    test_set = all_test_sets.GetTestSet(category)
    if not test_set:
        return http.HttpResponseBadRequest(
            'No test set was found for category=%s' % category)

    formatted_gviz_table_data = GetStats(request, test_set, 'gviz_table_data')
    return http.HttpResponse(formatted_gviz_table_data) 
Example 2
Project: browserscope   Author: elsigh   File: memcache_viewer.py    License: Apache License 2.0 6 votes vote down vote up
def _get_memcache_value_and_type(self, key):
    """Fetch value from memcache and detect its type.

    Args:
      key: String

    Returns:
      (value, type), value is a Python object or None if the key was not set in
      the cache, type is a string describing the type of the value.
    """
    try:
      value = memcache.get(key)
    except (pickle.UnpicklingError, AttributeError, EOFError, ImportError,
            IndexError), e:
      # Pickled data could be broken or the user might have stored custom class
      # instances in the cache (which can't be unpickled from here).
      msg = 'Failed to retrieve value from cache: %s' % e
      return msg, 'error' 
Example 3
Project: browserscope   Author: elsigh   File: memcache_viewer.py    License: Apache License 2.0 6 votes vote down vote up
def _set_memcache_value(self, key, type_, value):
    """Convert a string value and store the result in memcache.

    Args:
      key: String
      type_: String, describing what type the value should have in the cache.
      value: String, will be converted according to type_.

    Returns:
      Result of memcache.set(key, converted_value).  True if value was set.

    Raises:
      ValueError: Value can't be converted according to type_.
    """
    for _, converter, typestr in self.TYPES:
      if typestr == type_:
        value = converter(value)
        break
    else:
      raise ValueError('Type %s not supported.' % type_)
    return memcache.set(key, value) 
Example 4
Project: pledgeservice   Author: MayOneUS   File: handlers.py    License: Apache License 2.0 6 votes vote down vote up
def get(self):
    util.EnableCors(self)

    WP_PLEDGES = 4099
    VERSION_12_AND_UNDER = 59009 

    count = memcache.get('TOTAL-PLEDGES')
    if not count:
      query = model.Pledge.all(keys_only=True).filter('model_version >', 12)
      i = 0
      while True:
          result = query.fetch(1000)
          i = i + len(result)
          if len(result) < 1000:
              break
          cursor = query.cursor()
          query.with_cursor(cursor)
      count = i + WP_PLEDGES + VERSION_12_AND_UNDER
      memcache.set('TOTAL-PLEDGES', count, 120)

    self.response.headers['Content-Type'] = 'application/json'
    json.dump({'count':count}, self.response) 
Example 5
Project: isthislegit   Author: duo-labs   File: email.py    License: BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
def update(cls, domain, time=0):
        """
        Updates the memcached stats for a given domain

        This is used when a report is updated so that memcached
        has the current stats.

        Args:
            domain - str - The domain to use for the namespace
            time - int - The timeout for stored keys (default: 5 seconds)
        """
        namespace = "{}|{}".format('stats', domain)

        for status in VALID_STATUSES:
            count = EmailReport.query(EmailReport.reported_domain == domain,
                                      EmailReport.status == status).count()
            memcache.set(
                key=status, namespace=namespace, value=count, time=time) 
Example 6
Project: isthislegit   Author: duo-labs   File: email.py    License: BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
def update(cls, domain, time=0):
        """
        Updates the memcached stats for a given domain

        This is used when a report is updated so that memcached
        has the current stats.

        Args:
            domain - str - The domain to use for the namespace
            time - int - The timeout for stored keys (default: 5 seconds)
        """
        namespace = "{}|".format(domain)
        records = cls._get_from_datastore(domain, cls._memcache_result_count)
        memcache.set(
            key=cls._memcache_key,
            namespace=namespace,
            value=json.dumps(records),
            time=time) 
Example 7
Project: isthislegit   Author: duo-labs   File: timeline.py    License: BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
def _update_memcached(cls, domain, time=3600 * 24, records=None):
        """
        Updates memcached with the latest data from the datastore
        and returns that data. By default stores entries to expire after
        24 hours.
        """
        namespace = "{}|".format(domain)
        if not records:
            records = cls._get_from_datastore(domain,
                                              cls._memcache_date_offset)
        memcache.set(
            key=cls._memcache_key,
            namespace=namespace,
            value=json.dumps(records),
            time=time)
        return records 
Example 8
Project: googleapps-message-recall   Author: google   File: sessions_ndb.py    License: Apache License 2.0 6 votes vote down vote up
def get_by_sid(cls, sid):
        """Returns a ``Session`` instance by session id.

        :param sid:
            A session id.
        :returns:
            An existing ``Session`` entity.
        """
        data = memcache.get(sid)
        if not data:
            session = model.Key(cls, sid).get()
            if session:
                data = session.data
                memcache.set(sid, data)

        return data 
Example 9
Project: googleapps-message-recall   Author: google   File: credentials_utils.py    License: Apache License 2.0 6 votes vote down vote up
def GetUserAccessToken(user_email, force_refresh=False):
  """Helper to get a refreshed access_token for a user via service account.

  Args:
    user_email: User email for which access_token will be retrieved.
    force_refresh: Boolean, if True force a token refresh.

  Returns:
    Cached access_token or a new one.
  """
  access_token = memcache.get(user_email, namespace=_CACHE_NAMESPACE)
  if access_token and not force_refresh:
    return access_token

  credentials = _GetSignedJwtAssertionCredentials(user_email)
  # Have observed the following error from refresh():
  # 'Unable to fetch URL: https://accounts.google.com/o/oauth2/token'
  _LOG.debug('Refreshing access token for %s.', user_email)
  credentials.refresh(http_utils.GetHttpObject())
  access_token = credentials.access_token
  if memcache.set(user_email, access_token, time=_ACCESS_TOKEN_CACHE_S,
                  namespace=_CACHE_NAMESPACE):
    return access_token
  raise recall_errors.MessageRecallCounterError(
      'Exceeded retry limit in GetUserAccessToken: %s.' % user_email) 
Example 10
Project: python-docs-samples   Author: GoogleCloudPlatform   File: snippets.py    License: Apache License 2.0 6 votes vote down vote up
def add_values():
    # [START add_values]
    # Add a value if it doesn't exist in the cache
    # with a cache expiration of 1 hour.
    memcache.add(key="weather_USA_98105", value="raining", time=3600)

    # Set several values, overwriting any existing values for these keys.
    memcache.set_multi(
        {"USA_98115": "cloudy", "USA_94105": "foggy", "USA_94043": "sunny"},
        key_prefix="weather_",
        time=3600
    )

    # Atomically increment an integer value.
    memcache.set(key="counter", value=0)
    memcache.incr("counter")
    memcache.incr("counter")
    memcache.incr("counter")
    # [END add_values] 
Example 11
Project: luci-py   Author: luci   File: model.py    License: Apache License 2.0 6 votes vote down vote up
def get_content(namespace, hash_key):
  """Returns the content from either memcache or datastore, when stored inline.

  This does NOT return data from GCS, it is up to the client to do that.

  Returns:
    tuple(content, ContentEntry)
    At most only one of the two is set.

  Raises LookupError if the content cannot be found.
  Raises ValueError if the hash_key is invalid.
  """
  memcache_entry = memcache.get(hash_key, namespace='table_%s' % namespace)
  if memcache_entry is not None:
    return (memcache_entry, None)
  else:
    # Raises ValueError
    key = get_entry_key(namespace, hash_key)
    entity = key.get()
    if entity is None:
      raise LookupError("namespace %s, key %s does not refer to anything" %
        (namespace, hash_key))
    return (entity.content, entity) 
Example 12
Project: luci-py   Author: luci   File: utils.py    License: Apache License 2.0 6 votes vote down vote up
def get_task_queue_host():
  """Returns domain name of app engine instance to run a task queue task on.

  By default will use 'backend' module. Can be changed by calling
  set_task_queue_module during application startup.

  This domain name points to a matching version of appropriate app engine
  module - <version>.<module>.<app-id>.appspot.com where:
    version: version of the module that is calling this function.
    module: app engine module to execute task on.

  That way a task enqueued from version 'A' of default module would be executed
  on same version 'A' of backend module.
  """
  # modules.get_hostname sometimes fails with unknown internal error.
  # Cache its result in a memcache to avoid calling it too often.
  cache_key = 'task_queue_host:%s:%s' % (_task_queue_module, get_app_version())
  value = gae_memcache.get(cache_key)
  if not value:
    value = modules.get_hostname(module=_task_queue_module)
    gae_memcache.set(cache_key, value)
  return value 
Example 13
Project: luci-py   Author: luci   File: bot_code.py    License: Apache License 2.0 6 votes vote down vote up
def get_bot_version(host):
  """Retrieves the current bot version (SHA256) loaded on this server.

  The memcache is first checked for the version, otherwise the value
  is generated and then stored in the memcache.

  Returns:
    tuple(hash of the current bot version, dict of additional files).
  """
  signature = _get_signature(host)
  version = memcache.get('version-' + signature, namespace='bot_code')
  if version:
    return version, None

  # Need to calculate it.
  additionals = {'config/bot_config.py': get_bot_config().content}
  bot_dir = os.path.join(ROOT_DIR, 'swarming_bot')
  version = bot_archive.get_swarming_bot_version(
      bot_dir, host, utils.get_app_version(), additionals,
      local_config.settings())
  memcache.set('version-' + signature, version, namespace='bot_code', time=60)
  return version, additionals 
Example 14
Project: luci-py   Author: luci   File: task_queues.py    License: Apache License 2.0 6 votes vote down vote up
def _pre_put_hook(self):
    super(TaskDimensions, self)._pre_put_hook()
    sets = set()
    for s in self.sets:
      s._pre_put_hook()
      sets.add('\000'.join(s.dimensions_flat))
    if len(sets) != len(self.sets):
      # Make sure there's no duplicate TaskDimensionsSet.
      raise datastore_errors.BadValueError(
          '%s.sets must all be unique' % self.__class__.__name__)


### Private APIs.


# Limit in rebuild_task_cache_async. Meant to be overridden in unit test. 
Example 15
Project: luci-py   Author: luci   File: task_queues.py    License: Apache License 2.0 6 votes vote down vote up
def assert_task_async(request):
  """Makes sure the TaskRequest dimensions, for each TaskProperties, are listed
  as a known queue.

  This function must be called before storing the TaskRequest in the DB.

  When a cache miss occurs, a task queue is triggered.

  Warning: the task will not be run until the task queue ran, which causes a
  user visible delay. There is no SLA but expected range is normally seconds at
  worst. This only occurs on new kind of requests, which is not that often in
  practice.
  """
  # It's important that the TaskRequest to not be stored in the DB yet, still
  # its key could be set.
  exp_ts = request.created_ts
  futures = []
  for i in range(request.num_task_slices):
    t = request.task_slice(i)
    exp_ts += datetime.timedelta(seconds=t.expiration_secs)
    futures.append(_assert_task_props_async(t.properties, exp_ts))
  for f in futures:
    yield f 
Example 16
Project: luci-py   Author: luci   File: utils.py    License: Apache License 2.0 6 votes vote down vote up
def get_task_queue_host():
  """Returns domain name of app engine instance to run a task queue task on.

  By default will use 'backend' module. Can be changed by calling
  set_task_queue_module during application startup.

  This domain name points to a matching version of appropriate app engine
  module - <version>.<module>.<app-id>.appspot.com where:
    version: version of the module that is calling this function.
    module: app engine module to execute task on.

  That way a task enqueued from version 'A' of default module would be executed
  on same version 'A' of backend module.
  """
  # modules.get_hostname sometimes fails with unknown internal error.
  # Cache its result in a memcache to avoid calling it too often.
  cache_key = 'task_queue_host:%s:%s' % (_task_queue_module, get_app_version())
  value = gae_memcache.get(cache_key)
  if not value:
    value = modules.get_hostname(module=_task_queue_module)
    gae_memcache.set(cache_key, value)
  return value 
Example 17
Project: luci-py   Author: luci   File: utils.py    License: Apache License 2.0 6 votes vote down vote up
def get_task_queue_host():
  """Returns domain name of app engine instance to run a task queue task on.

  By default will use 'backend' module. Can be changed by calling
  set_task_queue_module during application startup.

  This domain name points to a matching version of appropriate app engine
  module - <version>.<module>.<app-id>.appspot.com where:
    version: version of the module that is calling this function.
    module: app engine module to execute task on.

  That way a task enqueued from version 'A' of default module would be executed
  on same version 'A' of backend module.
  """
  # modules.get_hostname sometimes fails with unknown internal error.
  # Cache its result in a memcache to avoid calling it too often.
  cache_key = 'task_queue_host:%s:%s' % (_task_queue_module, get_app_version())
  value = gae_memcache.get(cache_key)
  if not value:
    value = modules.get_hostname(module=_task_queue_module)
    gae_memcache.set(cache_key, value)
  return value 
Example 18
Project: luci-py   Author: luci   File: gitiles_import.py    License: Apache License 2.0 6 votes vote down vote up
def import_config_set(config_set):
  """Imports a config set."""
  import_attempt_metric.increment(fields={'config_set': config_set})
  service_match = config.SERVICE_CONFIG_SET_RGX.match(config_set)
  if service_match:
    service_id = service_match.group(1)
    return import_service(service_id)

  project_match = config.PROJECT_CONFIG_SET_RGX.match(config_set)
  if project_match:
    project_id = project_match.group(1)
    return import_project(project_id)

  ref_match = config.REF_CONFIG_SET_RGX.match(config_set)
  if ref_match:
    project_id = ref_match.group(1)
    ref_name = ref_match.group(2)
    return import_ref(project_id, ref_name)

  raise ValueError('Invalid config set "%s' % config_set)


## A cron job that schedules an import push task for each config set 
Example 19
Project: hackernewsbot   Author: phil-r   File: database.py    License: MIT License 5 votes vote down vote up
def add_memcache(self):
    memcache.set(self.key.id(), self.url) 
Example 20
Project: loaner   Author: google   File: template_model.py    License: Apache License 2.0 5 votes vote down vote up
def _cache_template(template):
    """Caches the title and body of a Template separately in memcache."""
    memcache.set(_CACHED_TITLE_NAME % template.name, template.title)
    memcache.set(_CACHED_BODY_NAME % template.name, template.body) 
Example 21
Project: loaner   Author: google   File: template_model_test.py    License: Apache License 2.0 5 votes vote down vote up
def test_get_all_from_memcache(self):
    template_list = ['template1', 'template2', 'template3']
    mem_name = 'template_list'
    memcache.set(mem_name, template_list)
    template_list_memcache = template_model.Template.get_all()
    self.assertLen(template_list_memcache, 2)
    memcache.flush_all()
    reference_datastore_template_list = template_model.Template.get_all()
    self.assertLen(reference_datastore_template_list, 2) 
Example 22
Project: loaner   Author: google   File: config_model_test.py    License: Apache License 2.0 5 votes vote down vote up
def test_get_from_memcache(self):
    config = 'string_config'
    config_value = 'this should be read.'
    memcache.set(config, config_value)
    reference_datastore_config = config_model.Config.get_by_id(config)
    config_memcache = config_model.Config.get(config)

    self.assertEqual(config_memcache, config_value)
    self.assertEqual(reference_datastore_config.string_value, 'config value 1') 
Example 23
Project: loaner   Author: google   File: config_model_test.py    License: Apache License 2.0 5 votes vote down vote up
def test_get_identifier_with_use_asset(self):
    config_model.Config.set('use_asset_tags', True)
    config_datastore = config_model.Config.get('device_identifier_mode')
    self.assertEqual(config_datastore,
                     config_model.DeviceIdentifierMode.BOTH_REQUIRED) 
Example 24
Project: loaner   Author: google   File: config_model_test.py    License: Apache License 2.0 5 votes vote down vote up
def test_set(self, test_config):
    config_model.Config.set(test_config[0], test_config[1])
    memcache_config = memcache.get(test_config[0])
    config = config_model.Config.get(test_config[0])

    self.assertEqual(memcache_config, test_config[1])
    self.assertEqual(config, test_config[1]) 
Example 25
Project: loaner   Author: google   File: config_model_test.py    License: Apache License 2.0 5 votes vote down vote up
def test_set_no_validation(self):
    fake_key = 'fake_int'
    fake_value = 23
    config_model.Config.set(fake_key, fake_value, False)
    result = config_model.Config.get(fake_key)
    self.assertEqual(result, fake_value) 
Example 26
Project: loaner   Author: google   File: config_model.py    License: Apache License 2.0 5 votes vote down vote up
def set(cls, name, value, validate=True):
    """Stores values for a config name in memcache and datastore.

    Args:
      name: str, name of the config setting.
      value: str, int, bool, list value to set or change config setting.
      validate: bool, checks keys against config_defaults if enabled.
    Raises:
      KeyError: Error raised when name does not exist in config.py file.
    """
    if validate:
      config_defaults = utils.load_config_from_yaml()
      if name not in config_defaults:
        raise KeyError(_CONFIG_NOT_FOUND_MSG % name)

    if isinstance(value, six.string_types):
      stored_config = cls.get_or_insert(name)
      stored_config.string_value = value
      stored_config.put()
    if isinstance(value, bool) and isinstance(value, int):
      stored_config = cls.get_or_insert(name)
      stored_config.bool_value = value
      stored_config.put()
    if isinstance(value, int) and not isinstance(value, bool):
      stored_config = cls.get_or_insert(name)
      stored_config.integer_value = value
      stored_config.put()
    if isinstance(value, list):
      stored_config = cls.get_or_insert(name)
      stored_config.list_value = value
      stored_config.put()

    memcache.set(name, value) 
Example 27
Project: browserscope   Author: elsigh   File: cron.py    License: Apache License 2.0 5 votes vote down vote up
def UpdateRecentTests(request):
  max_recent_tests = 10
  visible_categories = all_test_sets.GetVisibleTestSets()
  #logging.info('visible_categories %s' % visible_categories)

  prev_recent_tests = memcache.get(util.RECENT_TESTS_MEMCACHE_KEY)
  prev_result_parent_key = None
  if prev_recent_tests:
    prev_result_parent_key = prev_recent_tests[0]['result_parent_key']

  recent_tests = []
  recent_query = db.Query(ResultParent).order('-created').filter('category IN',
      [vis.category for vis in visible_categories])
  for result_parent in recent_query.fetch(max_recent_tests):
    if str(result_parent.key()) == prev_result_parent_key:
      num_needed = max_recent_tests - len(recent_tests)
      if num_needed == max_recent_tests:
        return http.HttpResponse('No update needed.')
      else:
        recent_tests.extend(prev_recent_tests[:num_needed])
        break
    recent_scores = result_parent.GetResults()
    test_set = all_test_sets.GetTestSet(result_parent.category)
    visible_test_keys = [t.key for t in test_set.VisibleTests()]
    recent_stats = test_set.GetStats(visible_test_keys, recent_scores)
    recent_tests.append({
        'result_parent_key': str(result_parent.key()),
        'category': result_parent.category,
        'created': result_parent.created,
        'user_agent_pretty': result_parent.user_agent.pretty(),
        'score': recent_stats['summary_score'],
        'display': recent_stats['summary_display'],
        })
  #logging.info('Setting recent tests: %s' % recent_tests)
  memcache.set(util.RECENT_TESTS_MEMCACHE_KEY, recent_tests,
               time=settings.STATS_MEMCACHE_TIMEOUT)
  #logging.info('Read recent tests: %s' %
  #    memcache.get(key=util.RECENT_TESTS_MEMCACHE_KEY))
  return http.HttpResponse('Done') 
Example 28
Project: browserscope   Author: elsigh   File: util.py    License: Apache License 2.0 5 votes vote down vote up
def BrowserEvolution(request):
    category = request.GET.get('category', 'summary')
    output = request.GET.get('o')
    if output == 'embed':
        extends_tpl = 'blank.html'
    else:
        extends_tpl = 'base.html'
    user_agents = request.GET.get('ua')
    version_level = request.GET.get('v')

    test_set = None
    test_set = all_test_sets.GetTestSet(category)
    if not test_set:
        return http.HttpResponseBadRequest(
            'No test set was found for category=%s' % category)

    (user_agents, user_agent_dict) = GetTimelineUserAgentDict(
        category, version_level, user_agents)

    json = GetStats(request, test_set, 'json', user_agents)
    params = {
        'category': category,
        'category_name': request.GET.get('label', test_set.category_name),
        'json': json,
        'extends_tpl': extends_tpl
    }
    return Render(request, 'evolution.html', params) 
Example 29
Project: browserscope   Author: elsigh   File: util.py    License: Apache License 2.0 5 votes vote down vote up
def BrowserTimeLineData(request):
    """Returns a string formatted for consumption by a Google Viz table."""
    test_set = None
    category = request.GET.get('category', 'summary')

    test_set = all_test_sets.GetTestSet(category)
    if not test_set:
        return http.HttpResponseBadRequest(
            'No test set was found for category=%s' % category)

    formatted_gviz_table_data = GetStats(request, test_set, 'gviz_timeline_data')
    return http.HttpResponse(formatted_gviz_table_data) 
Example 30
Project: browserscope   Author: elsigh   File: util.py    License: Apache License 2.0 5 votes vote down vote up
def CheckThrottleIpAddress(ip, user_agent_string, category):
    """Prevent beacon abuse and over-achievers ;)
    This throttle allows users to send a beacon 10 times per day per IP per
    major browser per category.
    i.e. The memcache key is something like 1.1.1.1_Opera 10_network

    We only use memcache here, so if we allow some extras in b/c of that, phooey.

    Args:
        ip: A masked IP address string.
        user_agent_string: A user agent string.
        category: A test category string.
    Returns:
        A Boolean, True if things are aok, False otherwise.
    """

    key = '%s_%s_%s' % (ip, user_agent_string, category)
    timeout = 86400  # 60 * 60 * 24
    runs_per_timeout = 10

    runs = memcache.get(key, IP_THROTTLE_NS)
    if runs is None:
        memcache.set(key=key, value=1, time=timeout, namespace=IP_THROTTLE_NS)
    elif runs <= runs_per_timeout:
        memcache.incr(key=key, delta=1, namespace=IP_THROTTLE_NS)
    else:
        return False
    return True