Python google.appengine.api.memcache.get_multi() Examples

def CacheGet(cls, key_names, ranker_classes):
    serialized_rankers = memcache.get_multi(
        key_names, namespace=cls.MEMCACHE_NAMESPACE)
    rankers = dict((k, ranker_classes[k].FromString(k, v))
                   for k, v in serialized_rankers.items())
    db_key_names = {}
    for key_name in key_names:
      if key_name not in rankers:
        db_key_names.setdefault(ranker_classes[key_name], []).append(key_name)
    if db_key_names:
      for ranker_class, key_names in db_key_names.items():
        for key_name, ranker in zip(
            key_names, ranker_class.get_by_key_name(key_names)):
          if ranker:
            rankers[key_name] = ranker
    return rankers 

def testBasic(self):
    category = self.test_set.category
    cls = result_stats.CategoryStatsManager
    browsers = ['Earth', 'Wind', 'Fire']
    test_keys = ['apple', 'banana', 'coconut']
    self.mox.StubOutWithMock(self.test_set, 'GetMediansAndNumScores')
    self.mox.StubOutWithMock(self.test_set, 'GetStats')
    self.mox.StubOutWithMock(result_stats.SummaryStatsManager, 'UpdateStats')
    self.test_set.GetMediansAndNumScores('Earth').AndReturn(('m1', 'n1'))
    self.test_set.GetStats(test_keys, 'm1', 'n1').AndReturn('s1')
    self.test_set.GetMediansAndNumScores('Wind').AndReturn(('m2', 'n2'))
    self.test_set.GetStats(test_keys, 'm2', 'n2').AndReturn('s2')
    self.test_set.GetMediansAndNumScores('Fire').AndReturn(('m3', 'n3'))
    self.test_set.GetStats(test_keys, 'm3', 'n3').AndReturn('s3')
    expected_ua_stats = {'Earth': 's1', 'Wind': 's2', 'Fire': 's3'}
    result_stats.SummaryStatsManager.UpdateStats(
        category, expected_ua_stats).AndReturn('notused')
    self.mox.ReplayAll()
    cls.UpdateStatsCache(category, browsers)
    self.mox.VerifyAll()
    ua_stats = memcache.get_multi(browsers, **cls.MemcacheParams(category))
    self.assertEqual(expected_ua_stats, ua_stats) 

def retrieve(key):
    result = memcache.get_multi(['%s.%s' % (key, i) for i in xrange(32)])
    serialized = ''.join(
        [v for k, v in sorted(result.items()) if v is not None])
    try:
        return pickle.loads(serialized)
    except EOFError as e:
        logging.info(e)
        return None 

def task(stories):
  def check_story(rpc):
    try:
      result = rpc.get_result()
      story = json.loads(result.content)
      if story and story.get('score') >= 100:
        StoryPost.add(story)
      elif story and story.get('score'):
        # api returned a comment once (issue with id 21447853)
        logging.info('STOP: {id} has low score ({score})'.format(**story))
      elif story:
        logging.info('STOP: {id} has no score'.format(**story))
      else:
        logging.info("STOP: story was probably deleted/flagged")
    except urlfetch.DownloadError as ex:
      logging.exception(ex)
    except ValueError as ex:
      logging.info(result.content)
      logging.exception(ex)


  # stringify ids for use in memcache and convert to set for later
  ids = set(str(story_id) for story_id in stories)
  logging.info('checking stories: {}'.format(ids))
  # get stories that we already posted to reduce the number of requests
  cached_stories = set(memcache.get_multi(ids).keys())
  logging.info('cached stories: {}'.format(cached_stories))
  # remove stories we know about from stories that we need to check
  stories_to_check = ids.difference(cached_stories)
  rpcs = map(lambda id: item_async(id, check_story), stories_to_check)
  for rpc in rpcs:
    rpc.wait() 

def UpdateStats(cls, category, stats):
        """Update the summary stats in memory and the datastore.

        This will only update part of a summary score row.

        Args:
            category: a category string like 'network'
            stats: a dict of browser stats (see CategoryStatsManager.GetStats)
        Returns:
            The summary stats that have been updated by the given stats.
            (Used by GetStats.)
        """
        browsers = [b for b in stats.keys() if b != 'total_runs']
        update_summary_stats = memcache.get_multi(
                browsers, namespace=cls.MEMCACHE_NAMESPACE)
        for browser in browsers:
            ua_summary_stats = update_summary_stats.setdefault(browser, {
                    'results': {}})
            ua_summary_stats['results'][category] = {
                    'score': stats[browser]['summary_score'],
                    'display': stats[browser]['summary_display'],
                    'total_runs': stats[browser]['total_runs'],
                    }
            if category == 'acid3':
                ua_summary_stats['results']['acid3']['display'] = (
                        stats[browser]['results']['score']['display'])
        memcache.set_multi(update_summary_stats, namespace=cls.MEMCACHE_NAMESPACE)
        return update_summary_stats 

def FindUncachedStats(cls, category, browsers):
        """Find which stats are not cached.

        Also, resets the summary stats for the stats that are found in memcache.

        Args:
            category: a category string like 'network'
            browsers: a list of browsers like ['Firefox 3.6', 'IE 8.0']
        Returns:
            a list of browsers without stats in memcache.
        """
        stats = memcache.get_multi(browsers, **cls.MemcacheParams(category))
        SummaryStatsManager.UpdateStats(category, stats)
        return [b for b in browsers if b not in stats] 

def testCacheGetOneItemInMemcache(self):
    ranker_class = self.mox.CreateMock(result_ranker.CountRanker)
    self.mox.StubOutWithMock(memcache, 'get_multi')
    memcache.get_multi(['k1'], **self.memcache_params).AndReturn({'k1': 's1'})
    ranker_class.FromString('k1', 's1').AndReturn('r1')
    self.mox.ReplayAll()
    rankers = self.cls.CacheGet(['k1'], {'k1': ranker_class})
    self.mox.VerifyAll()
    self.assertEqual({'k1': 'r1'}, rankers) 

def testCacheGetOneItemInDb(self):
    ranker_class = self.mox.CreateMock(result_ranker.CountRanker)
    self.mox.StubOutWithMock(memcache, 'get_multi')
    memcache.get_multi(['k1'], **self.memcache_params).AndReturn({})
    ranker_class.get_by_key_name(['k1']).AndReturn(['r1'])
    self.mox.ReplayAll()
    rankers = self.cls.CacheGet(['k1'], {'k1': ranker_class})
    self.mox.VerifyAll()
    self.assertEqual({'k1': 'r1'}, rankers) 

def testCacheGetOneItemNotFound(self):
    ranker_class = self.mox.CreateMock(result_ranker.CountRanker)
    self.mox.StubOutWithMock(memcache, 'get_multi')
    memcache.get_multi(['k1'], **self.memcache_params).AndReturn({})
    ranker_class.get_by_key_name(['k1']).AndReturn([None])
    self.mox.ReplayAll()
    rankers = self.cls.CacheGet(['k1'], {'k1': ranker_class})
    self.mox.VerifyAll()
    self.assertEqual({}, rankers) 

def get(self):
        # [START batch]
        values = {'comment': 'I did not ... ', 'comment_by': 'Bill Holiday'}
        if not memcache.set_multi(values):
            logging.error('Unable to set Memcache values')
        tvalues = memcache.get_multi(('comment', 'comment_by'))
        self.response.write(tvalues)
        # [END batch] 

def load_summary_protos(java_application=False):
  """Load all valid summary records from memcache.

  Args:
    java_application: Boolean. If true, this function is being invoked
      by the download_appstats tool on a java application.

  Returns:
    A list of StatsProto instances, in reverse chronological order
    (i.e. most recent first).

  NOTE: This is limited to returning at most config.KEY_MODULUS records,
  since there are only that many distinct keys.  See also make_key().
  """
  tmpl = config.KEY_PREFIX + config.KEY_TEMPLATE + config.PART_SUFFIX
  if java_application:

    tmpl = '"' + tmpl + '"'
  keys = [tmpl % i
          for i in
          range(0, config.KEY_DISTANCE * config.KEY_MODULUS,
                config.KEY_DISTANCE)]
  results = memcache.get_multi(keys, namespace=config.KEY_NAMESPACE)
  records = []
  for rec in results.itervalues():
    try:
      pb = StatsProto(rec)
    except Exception, err:
      logging.warn('Bad record: %s', err)
    else:
      records.append(pb) 

def load_summary_protos():
  """Load all valid summary records from memcache.

  Returns:
    A list of RequestStatProto instances, in reverse chronological order
    (i.e. most recent first).

  NOTE: This is limited to returning at most config.KEY_MODULUS records,
  since there are only that many distinct keys.  See also make_key().
  """
  tmpl = '%s%s%s' % (recording.config.KEY_PREFIX,
                     recording.config.KEY_TEMPLATE,
                     recording.config.PART_SUFFIX)
  keys = [tmpl % i
          for i in
          range(0, recording.config.KEY_DISTANCE * recording.config.KEY_MODULUS,
                recording.config.KEY_DISTANCE)]
  results = memcache.get_multi(keys, namespace=recording.config.KEY_NAMESPACE)
  records = []
  for rec in results.itervalues():
    try:
      pb = protobuf.decode_message(apphosting.RequestStatProto, rec)
    except Exception, err:
      logging.warn('Bad record: %s', err)
    else:
      records.append(pb) 

def AddUserAgent(cls, category, user_agent):
        """Adds a user agent's browser strings to version-level groups.

        AddUserAgent assumes that it does not receive overlapping calls.
        - It should only get called by the update-user-groups task queue.

        Adds a browser for every version level.
        If a level does not have a string, then use the one from the previous level.
        For example, "Safari 4.3" would increment the following:
                level  browser
                        0  Safari
                        1  Safari 4
                        2  Safari 4.3
                        3  Safari 4.3

        Args:
            category: a category string like 'network' or 'reflow'.
            user_agent: a UserAgent instance.
        """
        key_names = [cls.KeyName(category, v) for v in range(4)]
        version_levels = range(4)
        if category in [t.category for t in all_test_sets.GetVisibleTestSets()]:
            key_names.extend([cls.KeyName('summary', v) for v in range(4)])
            version_levels.extend(range(4))
        level_browsers = memcache.get_multi(key_names,
                                            namespace=cls.MEMCACHE_NAMESPACE)
        browser_key_names = []
        ua_browsers = user_agent.get_string_list()
        max_ua_browsers_index = len(ua_browsers) - 1
        for version_level, key_name in zip(version_levels, key_names):
            browser = ua_browsers[min(max_ua_browsers_index, version_level)]
            if browser not in level_browsers.get(key_name, []):
                browser_key_names.append((browser, key_name))
        managers = cls.get_by_key_name([x[1] for x in browser_key_names])

        updated_managers = []
        memcache_mapping = {}
        for (browser, key_name), manager in zip(browser_key_names, managers):
            if manager is None:
                manager = cls.get_or_insert(key_name)
            if browser not in manager.browsers:
                cls.InsortBrowser(manager.browsers, browser)
                updated_managers.append(manager)
                memcache_mapping[key_name] = manager.browsers
        if updated_managers:
            db.put(updated_managers)
            memcache.set_multi(memcache_mapping, namespace=cls.MEMCACHE_NAMESPACE) 

def GetStats(cls, browsers, categories=None):
        """Return the summary stats for a set of browsers and categories.

        Gets stats out of summary memcache. If needed, re-aggregate them for the
        categories. These data may come from memcache or all the way from the
        datastore.

        Args:
            browsers: a list of browsers to use instead of version level.
            categories: a list of categories like ['security', 'richtext'].
        Returns:
            {
                    browser_x: {
                            category_y: {
                                 'score': score_xy,
                                 'display': display_xy,
                                 'total_runs': total_runs_xy,
                                 }, ...
                            }, ...
            }
        """
        summary_stats = memcache.get_multi(
                browsers, namespace=cls.MEMCACHE_NAMESPACE)
        if not categories:
            categories = [t.category for t in all_test_sets.GetVisibleTestSets()]
        # Trim any unwanted stats and find any missing stats.
        missing_stats = {}
        for browser in browsers:
            ua_summary_stats = summary_stats.get(browser, {'results': {}})
            existing_categories = ua_summary_stats['results'].keys()
            for category in existing_categories:
                if category not in categories:
                    del ua_summary_stats['results'][category]
            for category in categories:
                if category not in existing_categories:
                    missing_stats.setdefault(category, []).append(browser)
        # Load any missing stats
        for category, browsers in missing_stats.items():
            updated_stats = cls._FindAndUpdateStats(category, browsers)
            summary_stats.update(updated_stats)

        cls._AddSummaryOfSummaries(summary_stats)
        return summary_stats 

def testUpdateStatsTwoCategories(self):
    for category in ('Aardvark', 'Badger'):
      updated_summary_stats = self.cls.UpdateStats(
          category, TEST_STATS[category])
    expected_updated_summary_stats = {
        'Firefox 3': {
            'results': {
                'Aardvark': {
                    'score': 2,
                    'display': '93/100',
                    'total_runs': 8,
                    },
                'Badger': {
                    'score': 7,
                    'display': '12/15',
                    'total_runs': 3,
                    },
                },
            },
        'IE 7': {
            'results': {
                'Badger': {
                    'score': 9,
                    'display': '14/15',
                    'total_runs': 1,
                    },
                },
            },
        }
    self.assertEqual(expected_updated_summary_stats, updated_summary_stats)
    expected_summary_stats = expected_updated_summary_stats.copy()
    expected_summary_stats.update({
        'Firefox 3.5': {
            'results': {
                'Aardvark': {
                    'score': 5,
                    'display': '83/100',
                    'total_runs': 5,
                    },
                },
            },
        })
    memcache_summary_stats = memcache.get_multi(
        ['Firefox 3', 'Firefox 3.5', 'IE 7'],
        namespace=self.cls.MEMCACHE_NAMESPACE)
    self.assertEqual(expected_summary_stats, memcache_summary_stats)