Python google.appengine.api.memcache.get_multi() Examples

The following are 15 code examples for showing how to use google.appengine.api.memcache.get_multi(). These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example.

You may check out the related API usage on the sidebar.

You may also want to check out all available functions/classes of the module google.appengine.api.memcache , or try the search function .

Example 1
Project: browserscope   Author: elsigh   File: result_ranker.py    License: Apache License 2.0 6 votes vote down vote up
def CacheGet(cls, key_names, ranker_classes):
    serialized_rankers = memcache.get_multi(
        key_names, namespace=cls.MEMCACHE_NAMESPACE)
    rankers = dict((k, ranker_classes[k].FromString(k, v))
                   for k, v in serialized_rankers.items())
    db_key_names = {}
    for key_name in key_names:
      if key_name not in rankers:
        db_key_names.setdefault(ranker_classes[key_name], []).append(key_name)
    if db_key_names:
      for ranker_class, key_names in db_key_names.items():
        for key_name, ranker in zip(
            key_names, ranker_class.get_by_key_name(key_names)):
          if ranker:
            rankers[key_name] = ranker
    return rankers 
Example 2
Project: browserscope   Author: elsigh   File: test_result_stats.py    License: Apache License 2.0 6 votes vote down vote up
def testBasic(self):
    category = self.test_set.category
    cls = result_stats.CategoryStatsManager
    browsers = ['Earth', 'Wind', 'Fire']
    test_keys = ['apple', 'banana', 'coconut']
    self.mox.StubOutWithMock(self.test_set, 'GetMediansAndNumScores')
    self.mox.StubOutWithMock(self.test_set, 'GetStats')
    self.mox.StubOutWithMock(result_stats.SummaryStatsManager, 'UpdateStats')
    self.test_set.GetMediansAndNumScores('Earth').AndReturn(('m1', 'n1'))
    self.test_set.GetStats(test_keys, 'm1', 'n1').AndReturn('s1')
    self.test_set.GetMediansAndNumScores('Wind').AndReturn(('m2', 'n2'))
    self.test_set.GetStats(test_keys, 'm2', 'n2').AndReturn('s2')
    self.test_set.GetMediansAndNumScores('Fire').AndReturn(('m3', 'n3'))
    self.test_set.GetStats(test_keys, 'm3', 'n3').AndReturn('s3')
    expected_ua_stats = {'Earth': 's1', 'Wind': 's2', 'Fire': 's3'}
    result_stats.SummaryStatsManager.UpdateStats(
        category, expected_ua_stats).AndReturn('notused')
    self.mox.ReplayAll()
    cls.UpdateStatsCache(category, browsers)
    self.mox.VerifyAll()
    ua_stats = memcache.get_multi(browsers, **cls.MemcacheParams(category))
    self.assertEqual(expected_ua_stats, ua_stats) 
Example 3
Project: iris   Author: doitintl   File: main.py    License: MIT License 5 votes vote down vote up
def retrieve(key):
    result = memcache.get_multi(['%s.%s' % (key, i) for i in xrange(32)])
    serialized = ''.join(
        [v for k, v in sorted(result.items()) if v is not None])
    try:
        return pickle.loads(serialized)
    except EOFError as e:
        logging.info(e)
        return None 
Example 4
Project: hackernewsbot   Author: phil-r   File: main.py    License: MIT License 5 votes vote down vote up
def task(stories):
  def check_story(rpc):
    try:
      result = rpc.get_result()
      story = json.loads(result.content)
      if story and story.get('score') >= 100:
        StoryPost.add(story)
      elif story and story.get('score'):
        # api returned a comment once (issue with id 21447853)
        logging.info('STOP: {id} has low score ({score})'.format(**story))
      elif story:
        logging.info('STOP: {id} has no score'.format(**story))
      else:
        logging.info("STOP: story was probably deleted/flagged")
    except urlfetch.DownloadError as ex:
      logging.exception(ex)
    except ValueError as ex:
      logging.info(result.content)
      logging.exception(ex)


  # stringify ids for use in memcache and convert to set for later
  ids = set(str(story_id) for story_id in stories)
  logging.info('checking stories: {}'.format(ids))
  # get stories that we already posted to reduce the number of requests
  cached_stories = set(memcache.get_multi(ids).keys())
  logging.info('cached stories: {}'.format(cached_stories))
  # remove stories we know about from stories that we need to check
  stories_to_check = ids.difference(cached_stories)
  rpcs = map(lambda id: item_async(id, check_story), stories_to_check)
  for rpc in rpcs:
    rpc.wait() 
Example 5
Project: browserscope   Author: elsigh   File: result_stats.py    License: Apache License 2.0 5 votes vote down vote up
def UpdateStats(cls, category, stats):
        """Update the summary stats in memory and the datastore.

        This will only update part of a summary score row.

        Args:
            category: a category string like 'network'
            stats: a dict of browser stats (see CategoryStatsManager.GetStats)
        Returns:
            The summary stats that have been updated by the given stats.
            (Used by GetStats.)
        """
        browsers = [b for b in stats.keys() if b != 'total_runs']
        update_summary_stats = memcache.get_multi(
                browsers, namespace=cls.MEMCACHE_NAMESPACE)
        for browser in browsers:
            ua_summary_stats = update_summary_stats.setdefault(browser, {
                    'results': {}})
            ua_summary_stats['results'][category] = {
                    'score': stats[browser]['summary_score'],
                    'display': stats[browser]['summary_display'],
                    'total_runs': stats[browser]['total_runs'],
                    }
            if category == 'acid3':
                ua_summary_stats['results']['acid3']['display'] = (
                        stats[browser]['results']['score']['display'])
        memcache.set_multi(update_summary_stats, namespace=cls.MEMCACHE_NAMESPACE)
        return update_summary_stats 
Example 6
Project: browserscope   Author: elsigh   File: result_stats.py    License: Apache License 2.0 5 votes vote down vote up
def FindUncachedStats(cls, category, browsers):
        """Find which stats are not cached.

        Also, resets the summary stats for the stats that are found in memcache.

        Args:
            category: a category string like 'network'
            browsers: a list of browsers like ['Firefox 3.6', 'IE 8.0']
        Returns:
            a list of browsers without stats in memcache.
        """
        stats = memcache.get_multi(browsers, **cls.MemcacheParams(category))
        SummaryStatsManager.UpdateStats(category, stats)
        return [b for b in browsers if b not in stats] 
Example 7
Project: browserscope   Author: elsigh   File: test_result_ranker.py    License: Apache License 2.0 5 votes vote down vote up
def testCacheGetOneItemInMemcache(self):
    ranker_class = self.mox.CreateMock(result_ranker.CountRanker)
    self.mox.StubOutWithMock(memcache, 'get_multi')
    memcache.get_multi(['k1'], **self.memcache_params).AndReturn({'k1': 's1'})
    ranker_class.FromString('k1', 's1').AndReturn('r1')
    self.mox.ReplayAll()
    rankers = self.cls.CacheGet(['k1'], {'k1': ranker_class})
    self.mox.VerifyAll()
    self.assertEqual({'k1': 'r1'}, rankers) 
Example 8
Project: browserscope   Author: elsigh   File: test_result_ranker.py    License: Apache License 2.0 5 votes vote down vote up
def testCacheGetOneItemInDb(self):
    ranker_class = self.mox.CreateMock(result_ranker.CountRanker)
    self.mox.StubOutWithMock(memcache, 'get_multi')
    memcache.get_multi(['k1'], **self.memcache_params).AndReturn({})
    ranker_class.get_by_key_name(['k1']).AndReturn(['r1'])
    self.mox.ReplayAll()
    rankers = self.cls.CacheGet(['k1'], {'k1': ranker_class})
    self.mox.VerifyAll()
    self.assertEqual({'k1': 'r1'}, rankers) 
Example 9
Project: browserscope   Author: elsigh   File: test_result_ranker.py    License: Apache License 2.0 5 votes vote down vote up
def testCacheGetOneItemNotFound(self):
    ranker_class = self.mox.CreateMock(result_ranker.CountRanker)
    self.mox.StubOutWithMock(memcache, 'get_multi')
    memcache.get_multi(['k1'], **self.memcache_params).AndReturn({})
    ranker_class.get_by_key_name(['k1']).AndReturn([None])
    self.mox.ReplayAll()
    rankers = self.cls.CacheGet(['k1'], {'k1': ranker_class})
    self.mox.VerifyAll()
    self.assertEqual({}, rankers) 
Example 10
Project: python-docs-samples   Author: GoogleCloudPlatform   File: batch.py    License: Apache License 2.0 5 votes vote down vote up
def get(self):
        # [START batch]
        values = {'comment': 'I did not ... ', 'comment_by': 'Bill Holiday'}
        if not memcache.set_multi(values):
            logging.error('Unable to set Memcache values')
        tvalues = memcache.get_multi(('comment', 'comment_by'))
        self.response.write(tvalues)
        # [END batch] 
Example 11
Project: python-compat-runtime   Author: GoogleCloudPlatform   File: recording.py    License: Apache License 2.0 5 votes vote down vote up
def load_summary_protos(java_application=False):
  """Load all valid summary records from memcache.

  Args:
    java_application: Boolean. If true, this function is being invoked
      by the download_appstats tool on a java application.

  Returns:
    A list of StatsProto instances, in reverse chronological order
    (i.e. most recent first).

  NOTE: This is limited to returning at most config.KEY_MODULUS records,
  since there are only that many distinct keys.  See also make_key().
  """
  tmpl = config.KEY_PREFIX + config.KEY_TEMPLATE + config.PART_SUFFIX
  if java_application:

    tmpl = '"' + tmpl + '"'
  keys = [tmpl % i
          for i in
          range(0, config.KEY_DISTANCE * config.KEY_MODULUS,
                config.KEY_DISTANCE)]
  results = memcache.get_multi(keys, namespace=config.KEY_NAMESPACE)
  records = []
  for rec in results.itervalues():
    try:
      pb = StatsProto(rec)
    except Exception, err:
      logging.warn('Bad record: %s', err)
    else:
      records.append(pb) 
Example 12
Project: protorpc   Author: google   File: __init__.py    License: Apache License 2.0 5 votes vote down vote up
def load_summary_protos():
  """Load all valid summary records from memcache.

  Returns:
    A list of RequestStatProto instances, in reverse chronological order
    (i.e. most recent first).

  NOTE: This is limited to returning at most config.KEY_MODULUS records,
  since there are only that many distinct keys.  See also make_key().
  """
  tmpl = '%s%s%s' % (recording.config.KEY_PREFIX,
                     recording.config.KEY_TEMPLATE,
                     recording.config.PART_SUFFIX)
  keys = [tmpl % i
          for i in
          range(0, recording.config.KEY_DISTANCE * recording.config.KEY_MODULUS,
                recording.config.KEY_DISTANCE)]
  results = memcache.get_multi(keys, namespace=recording.config.KEY_NAMESPACE)
  records = []
  for rec in results.itervalues():
    try:
      pb = protobuf.decode_message(apphosting.RequestStatProto, rec)
    except Exception, err:
      logging.warn('Bad record: %s', err)
    else:
      records.append(pb) 
Example 13
Project: browserscope   Author: elsigh   File: result_stats.py    License: Apache License 2.0 4 votes vote down vote up
def AddUserAgent(cls, category, user_agent):
        """Adds a user agent's browser strings to version-level groups.

        AddUserAgent assumes that it does not receive overlapping calls.
        - It should only get called by the update-user-groups task queue.

        Adds a browser for every version level.
        If a level does not have a string, then use the one from the previous level.
        For example, "Safari 4.3" would increment the following:
                level  browser
                        0  Safari
                        1  Safari 4
                        2  Safari 4.3
                        3  Safari 4.3

        Args:
            category: a category string like 'network' or 'reflow'.
            user_agent: a UserAgent instance.
        """
        key_names = [cls.KeyName(category, v) for v in range(4)]
        version_levels = range(4)
        if category in [t.category for t in all_test_sets.GetVisibleTestSets()]:
            key_names.extend([cls.KeyName('summary', v) for v in range(4)])
            version_levels.extend(range(4))
        level_browsers = memcache.get_multi(key_names,
                                            namespace=cls.MEMCACHE_NAMESPACE)
        browser_key_names = []
        ua_browsers = user_agent.get_string_list()
        max_ua_browsers_index = len(ua_browsers) - 1
        for version_level, key_name in zip(version_levels, key_names):
            browser = ua_browsers[min(max_ua_browsers_index, version_level)]
            if browser not in level_browsers.get(key_name, []):
                browser_key_names.append((browser, key_name))
        managers = cls.get_by_key_name([x[1] for x in browser_key_names])

        updated_managers = []
        memcache_mapping = {}
        for (browser, key_name), manager in zip(browser_key_names, managers):
            if manager is None:
                manager = cls.get_or_insert(key_name)
            if browser not in manager.browsers:
                cls.InsortBrowser(manager.browsers, browser)
                updated_managers.append(manager)
                memcache_mapping[key_name] = manager.browsers
        if updated_managers:
            db.put(updated_managers)
            memcache.set_multi(memcache_mapping, namespace=cls.MEMCACHE_NAMESPACE) 
Example 14
Project: browserscope   Author: elsigh   File: result_stats.py    License: Apache License 2.0 4 votes vote down vote up
def GetStats(cls, browsers, categories=None):
        """Return the summary stats for a set of browsers and categories.

        Gets stats out of summary memcache. If needed, re-aggregate them for the
        categories. These data may come from memcache or all the way from the
        datastore.

        Args:
            browsers: a list of browsers to use instead of version level.
            categories: a list of categories like ['security', 'richtext'].
        Returns:
            {
                    browser_x: {
                            category_y: {
                                 'score': score_xy,
                                 'display': display_xy,
                                 'total_runs': total_runs_xy,
                                 }, ...
                            }, ...
            }
        """
        summary_stats = memcache.get_multi(
                browsers, namespace=cls.MEMCACHE_NAMESPACE)
        if not categories:
            categories = [t.category for t in all_test_sets.GetVisibleTestSets()]
        # Trim any unwanted stats and find any missing stats.
        missing_stats = {}
        for browser in browsers:
            ua_summary_stats = summary_stats.get(browser, {'results': {}})
            existing_categories = ua_summary_stats['results'].keys()
            for category in existing_categories:
                if category not in categories:
                    del ua_summary_stats['results'][category]
            for category in categories:
                if category not in existing_categories:
                    missing_stats.setdefault(category, []).append(browser)
        # Load any missing stats
        for category, browsers in missing_stats.items():
            updated_stats = cls._FindAndUpdateStats(category, browsers)
            summary_stats.update(updated_stats)

        cls._AddSummaryOfSummaries(summary_stats)
        return summary_stats 
Example 15
Project: browserscope   Author: elsigh   File: test_result_stats.py    License: Apache License 2.0 4 votes vote down vote up
def testUpdateStatsTwoCategories(self):
    for category in ('Aardvark', 'Badger'):
      updated_summary_stats = self.cls.UpdateStats(
          category, TEST_STATS[category])
    expected_updated_summary_stats = {
        'Firefox 3': {
            'results': {
                'Aardvark': {
                    'score': 2,
                    'display': '93/100',
                    'total_runs': 8,
                    },
                'Badger': {
                    'score': 7,
                    'display': '12/15',
                    'total_runs': 3,
                    },
                },
            },
        'IE 7': {
            'results': {
                'Badger': {
                    'score': 9,
                    'display': '14/15',
                    'total_runs': 1,
                    },
                },
            },
        }
    self.assertEqual(expected_updated_summary_stats, updated_summary_stats)
    expected_summary_stats = expected_updated_summary_stats.copy()
    expected_summary_stats.update({
        'Firefox 3.5': {
            'results': {
                'Aardvark': {
                    'score': 5,
                    'display': '83/100',
                    'total_runs': 5,
                    },
                },
            },
        })
    memcache_summary_stats = memcache.get_multi(
        ['Firefox 3', 'Firefox 3.5', 'IE 7'],
        namespace=self.cls.MEMCACHE_NAMESPACE)
    self.assertEqual(expected_summary_stats, memcache_summary_stats)