ndb 范围内的唯一键

Question

我正在使用谷歌应用引擎，需要拥有 1000 到 2^31 之间的实体的密钥。我正在考虑两种方法：

1) 保留已创建密钥的计数器，详见此处https://cloud.google.com/appengine/articles/sharding_counters。但这需要对每个键进行多次数据存储读/写，我不确定它是否保证是一致的。

2) 在我的范围内生成一个随机 int 并检查该密钥是否已经在数据库中。为了使它便宜，我想要一个 keys_only 查询，但除了将密钥也保存为单独的字段之外，我找不到这样做的方法： MyEntity.query(MyEntity.key_field==new_random_number).fetch(keys_only=True)

有没有更好的方法来实现这一点？

Answer 1

您希望在生产环境中每秒写入多少次？您的两个建议都很好，但是对于我们的应用程序，我决定采用分片计数器方法。您还可以在放置实体之前设置实体的 id 以完全避免查询：

MyModel(id="foo")

那你可以查一下：

MyModel.get_by_id("foo")

Id 不一定是字符串，也可以是数字：

MyModel(id=123)

如果您决定使用分片计数器，这是我们的生产级代码，与您在那篇文章中读到的内容非常接近；o) Memcache 增加了我们获得正确计数所需的一致性级别。

class GeneralShardedCounterConfig(ndb.Model):
    SHARD_KEY_TEMPLATE = 'gen-count-{}-{:d}'
    num_shards = ndb.IntegerProperty(default=200)

    @classmethod
    def all_keys(cls, name):
        config = cls.get_or_insert(name)
        shard_key_strings = [GeneralShardedCounterConfig.SHARD_KEY_TEMPLATE.format(name, index)
                             for index in range(config.num_shards)]
        return [ndb.Key(GeneralShardedCounter, shard_key_string)
                for shard_key_string in shard_key_strings]


class GeneralShardedCounter(BaseModel):
    count = ndb.IntegerProperty(default=0)

    @classmethod
    def get_count(cls, name):
        total = memcache.get(name)
        if total is None:
            total = 0
            all_keys = GeneralShardedCounterConfig.all_keys(name)
            for counter in ndb.get_multi(all_keys):
                if counter is not None:
                    total += counter.count
            memcache.set(name, total, constants.SHORT_MEMCACHE_TTL)
        return total

    @classmethod
    @ndb.transactional(retries=5)
    def increase_shards(cls, name, num_shards):
        config = GeneralShardedCounterConfig.get_or_insert(name)
        if config.num_shards < num_shards:
            config.num_shards = num_shards
            config.put()

    @classmethod
    @ndb.transactional(xg=True)
    def _increment(cls, name, num_shards):
        index = random.randint(0, num_shards - 1)
        shard_key_string = GeneralShardedCounterConfig.SHARD_KEY_TEMPLATE.format(name, index)
        counter = cls.get_by_id(shard_key_string)
        if counter is None:
            counter = cls(id=shard_key_string)
        counter.count += 1
        counter.put()
        # Memcache increment does nothing if the name is not a key in memcache
        memcache.incr(name)

    @classmethod
    def increment(cls, name):
        config = GeneralShardedCounterConfig.get_or_insert(name)
        cls._increment(name, config.num_shards)

    @classmethod
    def _add(cls, name, value, num_shards):
        index = random.randint(0, num_shards - 1)
        shard_key_string = GeneralShardedCounterConfig.SHARD_KEY_TEMPLATE.format(name, index)
        counter = cls.get_by_id(shard_key_string)
        if counter is None:
            counter = cls(id=shard_key_string)
        counter.count += value
        counter.put()
        # Memcache increment does nothing if the name is not a key in memcache
        memcache.incr(name, value)

    @classmethod
    def add(cls, name, value):
        config = GeneralShardedCounterConfig.get_or_insert(name)
        cls._add(name, value, config.num_shards)

Answer 2

get_or_insert 示例。插入 7 个唯一键

import webapp2
from google.appengine.ext import ndb
from datetime import datetime
import random
import logging


class Examples(ndb.Model):
    data = ndb.StringProperty()
    modified = ndb.DateTimeProperty(auto_now=True)
    created = ndb.DateTimeProperty()  # NOT auto_now_add HERE !!


class MainHandler(webapp2.RequestHandler):

    def get(self):

        count = 0
        while count < 7:
            random_key = str(random.randrange(1, 9))
            dt_created = datetime.now()
            example = Examples.get_or_insert(random_key, created=dt_created, data='some data for ' + random_key)
            if example.created != dt_created:
                logging.warning('Random key %s not unique' % random_key)
                continue
            count += 1

        self.response.write('Keys inserted')

app = webapp2.WSGIApplication([
    ('/', MainHandler)
], debug=True)