【发布时间】：2020-06-11 11:34:25
【问题描述】：
我从 API 获取 JSON。此 API 省略了 null 值（null 的属性不会通过线路发送），因此数据可能是稀疏的。这些属性包含字符串、数字、布尔值、unix-timestamps、ISO8601-timestamps 和 ISO8601-durations 的混合。
这是一个包含所有数据类型的示例 JSON（作为 Python 列表/字典）
data_full = [
{'name': 'alice', 'lastname': 'foo', 'value': 1.11, 'unix_ts': 1591848156000, 'iso_ts': '2020-05-17T12:33:44Z',
'iso_dur': 'PT1H11M', 'bool_val': True},
{'name': 'clair', 'lastname': 'bar', 'value': 3.33, 'unix_ts': 1591648156000, 'iso_ts': '2020-03-17T12:33:44Z',
'iso_dur': 'PT3H33M', 'bool_val': True},
]
稀疏数据可能缺少任何行或所有行的字段，或者 API 结果也可能完全为空。例子
some_fields_missing_in_some_rows = [
{'name': 'alice', 'lastname': 'foo', 'value': 1.23, 'unix_ts': 1591848156000,
'iso_ts': '2020-05-17T12:33:44Z',
'iso_dur': 'PT1H11M', 'bool_val': True},
{'name': 'clair', }
]
some_fields_missing_in_all_rows = [
{'name': 'alice'},
{'name': 'clair'}
]
no_data = []
我使用json_normalize 将其转换为 Pandas DataFrame。为了允许进行进一步的预测处理，我希望在所有稀疏情况下，输出 dtype 与数据已满相同，并且在缺失的地方插入了正确的 NA。我很难得到正确类型（np.nan 或其他）的缺失值。
下面完整包含的测试用例显示了问题（也就是说，如果您通过了 4 个测试，我相信它正在做我期望的事情）。
一个明显的问题是如何创建并使用 NaN 填充 str 类型的空列。
感谢您提供任何反馈。
import datetime
from typing import List, Tuple
from unittest import TestCase
import isodate
import numpy as np
import pandas as pd
class TestDFNormalization(TestCase):
def test_full_fields(self):
jsList = [
{'name': 'alice', 'lastname': 'foo', 'value': 1.11, 'unix_ts': 1591848156000,
'iso_ts': '2020-05-17T12:33:44Z',
'iso_dur': 'PT1H11M', 'bool_val': True},
{'name': 'clair', 'lastname': 'bar', 'value': 3.33, 'unix_ts': 1591648156000,
'iso_ts': '2020-03-17T12:33:44Z',
'iso_dur': 'PT3H33M', 'bool_val': True},
]
df = extract_df(js=jsList)
print(df.dtypes)
print(df)
self.assert_dtypes_conform(df)
self.assert_correct_NaNs(df, 2) # no NaN, so all rows (=2) kept
def test_sparse_fields(self):
some_fields_missing_in_some_rows = [
{'name': 'alice', 'lastname': 'foo', 'value': 1.23, 'unix_ts': 1591848156000,
'iso_ts': '2020-05-17T12:33:44Z',
'iso_dur': 'PT1H11M', 'bool_val': True},
{'name': 'clair', }
]
df = extract_df(js=some_fields_missing_in_some_rows )
print(df.dtypes)
print(df)
self.assert_dtypes_conform(df)
self.assert_correct_NaNs(df, 1) # some NaN, only 1 row kept
def test_lacking_fields(self):
some_fields_missing_in_all_rows = [
{'name': 'alice'},
{'name': 'clair'}
]
df = extract_df(js=some_fields_missing_in_all_rows )
print(df.dtypes)
print(df)
self.assert_dtypes_conform(df)
self.assert_correct_NaNs(df, 0) # all NaN, no rows
def test_no_data(self):
no_data = []
df = extract_df(js=no_data )
print(df.dtypes)
print(df)
self.assert_dtypes_conform(df)
self.assert_correct_NaNs(df, 0) # no rows
def assert_dtypes_conform(self, df: pd.DataFrame) -> None:
self.assertEqual("object", df['name'].dtype)
self.assertEqual("object", df['lastname'].dtype)
self.assertEqual("float", df['value'].dtype)
self.assertEqual("datetime64[ns, UTC]", df['unix_ts'].dtype)
self.assertEqual("datetime64[ns, UTC]", df['iso_ts'].dtype)
self.assertEqual("timedelta64[ns]", df['iso_dur'].dtype)
self.assertEqual("boolean", df['bool_val'].dtype)
def assert_correct_NaNs(self, df: pd.DataFrame, expectedNumRowsAfterDropNA: int) -> None:
self.assertEqual(expectedNumRowsAfterDropNA, len(df.dropna(subset=['lastname']).index))
self.assertEqual(expectedNumRowsAfterDropNA, len(df.dropna(subset=['value']).index))
self.assertEqual(expectedNumRowsAfterDropNA, len(df.dropna(subset=['unix_ts']).index))
self.assertEqual(expectedNumRowsAfterDropNA, len(df.dropna(subset=['iso_ts']).index))
self.assertEqual(expectedNumRowsAfterDropNA, len(df.dropna(subset=['iso_dur']).index))
self.assertEqual(expectedNumRowsAfterDropNA, len(df.dropna(subset=['bool_val']).index))
def extract_df(js: List) -> pd.DataFrame:
df = pd.json_normalize(js)
create_cols_if_absent(df=df,
expected_cols=('name', 'lastname', 'value', 'unix_ts', 'iso_ts', 'iso_dur', 'bool_val'))
# astype_per_column(df=df, column='name', dtype='str')
# astype_per_column(df=df, column='lastname', dtype='str')
# astype_per_column(df=df, column='value', dtype='float')
parse_unix_ms(df=df, column='unix_ts')
parse_iso(df=df, column='iso_ts')
parse_dur(df=df, column='iso_dur')
astype_per_column(df=df, column='bool_val', dtype='boolean')
return df
def create_cols_if_absent(df: pd.DataFrame, expected_cols: Tuple) -> None:
for col in expected_cols:
if col not in df.columns:
df[col] = np.nan # or None or pd.NA or np.nan ?
def parse_unix_ms(df, column):
df[column] = pd.to_datetime(df[column], unit='ms', origin='unix', utc=True)
def parse_iso(df, column):
df[column] = pd.to_datetime(df[column], utc=True)
def parse_iso_duration(durationstring: str) -> datetime.timedelta:
if not durationstring or pd.isna(durationstring):
return None
return isodate.parse_duration(durationstring)
def parse_dur(df, column) -> None:
df[column] = pd.to_timedelta(
df[column].apply(parse_iso_duration)) # why does to_timedelta() not support ISO8601 notation?
def astype_per_column(df: pd.DataFrame, column: str, dtype) -> None:
df[column] = df[column].astype(dtype)
【问题讨论】：
标签： python python-3.x pandas dataframe missing-data