如何使用tastepie API 从所有对象中获取数据？

Question

我的目标是从tastepie API 获取所有数据。这个 API 里面有超过 1.000.000 个对象！ 我创建了另一个 Django 项目和一个自定义的“product_import.py”命令，该命令应该从此 API 获取所有数据。

当我达到限制时，我应该怎么做才能继续从 API 获取数据？我想从 API ['meta']['next'] 字典中获取数据以继续在另一个项目中收集对象。

下面是我现在创建的一个示例，但被卡住了。

api.py

# coding: utf-8

from tastypie.resources import ModelResource
from tastypie import fields
from tastypie.authentication import ApiKeyAuthentication
from tastypie.authorization import DjangoAuthorization

from .models import Product


class UserguideResource(ModelResource):

    class Meta:
        resource_name = 'product'
        allowed_methods = ['get']
        fields = ['id', 'name', 'date_added', 'author']
        queryset = Product.objects.all()
        authentication = ApiKeyAuthentication()
        authorization = DjangoAuthorization()

这个文件是在不同的 Django 项目中创建的，它使用了上面项目中的 sweetpie API。我想从此 API 获取所有数据。

product_import.py

from django.core.management.base import BaseCommand
import requests


class Command(BaseCommand):
    """ Import Products from API """

    def __init__(self):
        self.API_KEY = 'secret-key'
        self.API_USERNAME = 'keNzi'
        self.product_api_url = 'http://localhost:8000/api/v1/product?username={0}&api_key={1}'.format(
            self.API_USERNAME, self.API_KEY)
        self.limit = 1
        self.offset = 0

    def get_latest_product_from_api(self):
        response = requests.get(
            "{0}&limit={1}&offset={2}".format(
                self.product_api_url,
                self.limit,
                self.offset,
            )
        ).json()
        return response

    def get_next_url_from_product_list(self):
        return self.get_latest_product_from_api()['meta']['next']

    def get_next_product_list(self):
        self.next = self.get_next_url_from_product_list()
        response = requests.get(
            "{0}{1}".format(
                self.product_api_url,
                self.next,
            )
        ).json()
        return response

    def handle(self, *args, **options):
        print(self.get_latest_product_from_api())
        print(self.get_next_product_list())

print() 显示来自 API 的产品，但我想继续并从中收集所有数据。

Answer 1

这是一个运行良好的代码 ;)

import requests
from django.core.management.base import BaseCommand
from product.models import Product


class Command(BaseCommand):
    """ Import Products from API """

    def __init__(self):
        self.API_URL = 'http://localhost:8000/api/v1'
        self.API_USERNAME = 'keNzi'
        self.API_KEY = 'keNzi-secret-2017'   # created in /admin/tastypie/apikey/
        self.API_AUTH = '?username={0}&api_key={1}'.format(
            self.API_USERNAME, self.API_KEY)
        self.product_api_url = '{0}/product/{1}'.format(
            self.API_URL, self.API_AUTH)
        self.limit = 1
        self.offset = 0

    def get_product_from_api(self):
        response = requests.get(
            "{0}&limit={1}&offset={2}".format(
                self.product_api_url,
                self.limit,
                self.offset,
            )
        ).json()
        return response

    def get_next_url_from_product_api(self):
        return self.get_product_from_api()['meta']['next']

    def handle(self, *args, **options):
        next_url = True
        while next_url:
            for obj in self.get_product_from_api()['objects']:
                print(obj['id'], obj['file'])
                product = Product.objects.create(
                    name=obj['name'],
                    description=obj['text'],
                    created=obj['date_added'],
                )
                product.save()
                print('Product saved! ID of the new objects: ', product.id)

            self.offset += 1
            next_url = self.get_next_url_from_product_api()