【发布时间】：2021-07-12 16:56:27
【问题描述】：
我有一个名为 Location 的模型，我正在使用产生 4000 个对象的过滤器查询该模型：
count = Location.objects.filter(**filters).count()
4000
有一个相关的Model叫做KPI，每个Location有很多KPI，有2,944,000条KPI记录。
我有一个非常复杂的 Location 查询，它注释了很多 KPI 数据。
注释：
def contribute_annotations(self):
user = self.request.user
self.kpis = user.user_selected_kpis.get_all_kpis_qs()
kpis_names = tuple(kpi.internal_name for kpi in self.kpis)
branch_date = Subquery(BranchKPIs.objects.
filter(branch__location__id=OuterRef(ID)).
order_by('-date').
values(DATE)[:1]
)
# summing the members amount
filters_for_branch = (
Q(location_branches__prem=True) &
~Q(location_branches__branch_scores__members_count=0) &
Q(location_branches__branch_scores__date=F(BRANCH_DATE))
)
sum_of_members_prem_count = Coalesce(Sum('location_branches__branch_scores__members_count',
output_field=IntegerField(),
filter=filters_for_branch),
0)
# location kpis prefetch object
location_kpis_qs = LocationKPIs.objects.filter(date__range=month_range).only(DATE, LOCATION, *kpis_names)
prefetch_location_kpis = Prefetch(lookup=RelatedNames.LOCATION_SCORES,
queryset=location_kpis_qs,
)
assigned_members_count_of_latest = Case(When(location_scores__date=F(LATEST_DATE),
then=f'location_scores__assigned_members_count'))
members_count_of_latest = Case(When(location_scores__date=F(LATEST_DATE),
then=f'location_scores__members_count'))
# kpis annotations for Avg, Trends, and Sizing
kpis_annotations, alias_for_trends, kpis_objects = {}, {}, {}
for kpi in self.kpis:
name = kpi.internal_name
# annotating the last kpi score
kpis_annotations[name] = Case(When(location_scores__date=F('latest_date'),
then=f'location_scores__{name}'), default=0)
# annotating the kpi's month avg
alias_for_trends[f'{name}_avg'] = Coalesce(
Avg(f'location_scores__{name}',
filter=Q(location_scores__date__range=month_range), output_field=IntegerField()
),
0
)
# comparing latest score to the monthly avg in order to determine the kpi's trend
when_equal = When(**{f'{name}_avg': F(name)}, then=0)
when_trend_is_down = When(**{f'{name}_avg__gt': F(name)}, then=-1)
when_trend_is_up = When(**{f'{name}_avg__lt': F(name)}, then=1)
kpi_trend = Case(when_equal, when_trend_is_up, when_trend_is_down,
default=0, output_field=IntegerField())
# annotating the score color
when_red = When(**{f'{name}__gte': kpi.location_level_red_threshold.lower,
f'{name}__lte': kpi.location_level_red_threshold.upper},
then=1
)
when_yellow = When(**{f'{name}__gte': kpi.location_level_yellow_threshold.lower,
f'{name}__lte': kpi.location_level_yellow_threshold.upper},
then=2
)
when_green = When(**{f'{name}__gte': kpi.location_level_green_threshold.lower,
f'{name}__lte': kpi.location_level_green_threshold.upper},
then=3
)
score_type = Case(when_red, when_yellow, when_green, default=2)
# outputs kpi : {score: int, trend: int, score_type: int}
kpis_objects[name] = JSONObject(
score=F(name),
trend=kpi_trend,
score_type=score_type
)
# cases for the pin size of the location, it depends on how many members are in it
when_in_s_size = When(
Q(member_count__gte=settings.S_LOCATION_SIZE[0]) & Q(member_count__lte=settings.S_LOCATION_SIZE[-1]),
then=1)
when_in_m_size = When(
Q(member_count__gte=settings.M_LOCATION_SIZE[0]) & Q(member_count__lte=settings.M_LOCATION_SIZE[-1]),
then=2)
when_in_l_size = When(
Q(member_count__gte=settings.L_LOCATION_SIZE[0]) & Q(member_count__lte=settings.L_LOCATION_SIZE[-1]),
then=3)
when_in_xl_size = When(
Q(member_count__gte=settings.XL_LOCATION_SIZE[0]) & Q(member_count__lte=settings.XL_LOCATION_SIZE[-1]),
then=4)
location_size = Case(when_in_s_size, when_in_m_size, when_in_l_size, when_in_xl_size,
default=2,
output_field=IntegerField())
# location's address string
location_str = Concat(LOCATION__STREET, LOCATION__CITY, LOCATION__COUNTRY,
output_field=CharField())
return (
sum_of_members_prem_count, prefetch_location_kpis, assigned_members_count_of_latest, members_count_of_latest,
kpis_annotations, location_size, alias_for_trends, location_str, kpis_names, kpis_objects, branch_date)
filters = {'user': self.request.user, ACTIVE: True}
(sum_of_members_prem_count, prefetch_location_kpis, assigned_members_count_of_latest, members_count_of_latest,
kpis_annotations, location_size, alias_for_trends, location_str, kpis_names, kpis_objects, branch_date) = self.contribute_annotations()
query_set = (Location.objects.
filter(**filters).
select_related(RelatedNames.LOCATION).
prefetch_related(prefetch_location_kpis).
alias(latest_date=Max('scores__date'),
branch_date=branch_date,
**alias_for_trends,
**kpis_annotations
).
annotate(members_prem_count=sum_of_members_prem,
members_count=members_count_of_latest,
assigned_members_count=assigned_count_of_latest,
farm_latitude=Min(LOCATION__LATITUDE),
farm_longitude=Min(LOCATION__LONGITUDE),
address=location_str,
farm_size=farm_size,
latest_date=Max('farm_scores__date'),
**kpis_objects
).
values(ID, NAME, ADMIN_EMAIL, ADMIN_PHONE, MEMBERS_PREM_COUNT,
MEMBERS_COUNT, ASSIGNED_MEMBERS_COUNT, SIZE, ADDRESS,
latitude=F(LOCATION_LATITUDE), longitude=F(LOCATION_LONGITUDE), *kpis_names
)
)
此查询产生 2,944,000 条记录，这意味着每个 KPI 记录而不是位置记录。 我尝试以多种方式添加不同的调用，但我最终得到：
NotImplementedError: annotate() + distinct(fields) is not implemented.
或者查询只是忽略它并且不添加不同的位置对象。
文档表明 values 和 distinct 不能很好地结合在一起，并且可能在某个地方存在破坏它的顺序。 我查看了所有涉及的模型、查询和子查询并删除了 order by，但它仍然不起作用。
我也尝试将其添加到查询中：
query_set.query.clear_ordering(True)
query_set = query_set.order_by(ID).distinct(ID)
但这会引发 NotImplementedError
【问题讨论】：
-
因为你的代码无法被理解，你有太多的变量突然出现。请查看如何写minimal reproducible example。
-
@AbdulAzizBarkat 我想避免添加所有这些来源，因为它很长，但会编辑
-
请注意，除了reproducible之外，它也需要minimal...
标签： python django orm django-orm