【问题标题】:Correcting for the df's when using weights in lm在 lm 中使用权重时校正 df
【发布时间】:2021-04-04 10:50:14
【问题描述】:

基本上我的问题很简单。我想知道在lm 函数内部,数据被子集(NA 被删除),基于公式中使用的所有变量。我想知道这一点的原因是因为我想仅使用子集数据(其中删除了NA)而不是我的完整数据集来对变量求和。

背景:

我尝试将R 中的lm 函数改编为lm 中的account for the correct degrees of freedom when using weights。我认为最简单的解决方案是从函数内部计算子集数据的权重总和,具体取决于所选变量。因此,首先我会根据所选变量查看 lm 对数据集进行子集化的位置,并计算 sum(ind$weight_freq),我想将其作为函数的输出之一提供,以便我可以参考它。

示例数据:

library(dplyr)

set.seed(1024)

# individual (true) dataset
x <- round(rnorm(1e5))
y <- round(x + x^2 + rnorm(1e5))
ind <- data.frame(x, y)

# Create an NA value
ind[1,1] <- NA

ind <- ind %>%
  group_by(x, y) %>%
  summarize(weight_freq= n())

我一开始只是将 lm 代码复制到一个新函数中,并将所有 &lt;- 替换为 &lt;&lt;-,以查看数据子集的位置,使用 lm_plus(y ~ x, data = ind, weights = weight_freq),虽然我得到了错误,@987654334 @,代码足够远,可以进行子集化(因为mf 应该是99999,因为NA,它是):

lm_plus <- function (formula, data, subset, weights, na.action, method = "qr", 
    model = TRUE, x = FALSE, y = FALSE, qr = TRUE, singular.ok = TRUE, 
    contrasts = NULL, offset, ...) 
{
    ret.x <<- x
    ret.y <<- y
    cl <<- match.call()
    mf <<- match.call(expand.dots = FALSE)
    m <<- match(c("formula", "data", "subset", 
        "weights", "na.action", "offset"), 
        names(mf), 0L)
    mf <<- mf[c(1L, m)]
    mf$drop.unused.levels <<- TRUE
    mf[[1L]] <<- quote(stats::model.frame)
    mf <<- eval(mf, parent.frame())
    if (method == "model.frame") 
        return(mf)
    else if (method != "qr") 
        warning(gettextf("method = '%s' is not supported. Using 'qr'", 
            method), domain = NA)
    mt <<- attr(mf, "terms")
    y <<- model.response(mf, "numeric")
    w <<- as.vector(model.weights(mf))
    if (!is.null(w) && !is.numeric(w)) 
        stop("'weights' must be a numeric vector")
    offset <<- model.offset(mf)
    mlm <<- is.matrix(y)
    ny <<- if (mlm) 
        nrow(y)
    else length(y)
    if (!is.null(offset)) {
        if (!mlm) 
            offset <<- as.vector(offset)
        if (NROW(offset) != ny) 
            stop(gettextf("number of offsets is %d, should equal %d (number of observations)", 
                NROW(offset), ny), domain = NA)
    }
    if (is.empty.model(mt)) {
        x <<- NULL
        z <<- list(coefficients = if (mlm) matrix(NA_real_, 0, 
            ncol(y)) else numeric(), residuals = y, fitted.values = 0 * 
            y, weights = w, rank = 0L, df.residual = if (!is.null(w)) sum(w != 
            0) else ny)
        if (!is.null(offset)) {
            z$fitted.values <<- offset
            z$residuals <<- y - offset
        }
    }
    else {
        x <<- model.matrix(mt, mf, contrasts)
        z <<- if (is.null(w)) 
            lm.fit(x, y, offset = offset, singular.ok = singular.ok, 
                ...)
        else lm.wfit(x, y, w, offset = offset, singular.ok = singular.ok, 
            ...)
    }
    class(z) <<- c(if (mlm) "mlm", "lm")
    z$na.action <<- attr(mf, "na.action")
    z$offset <<- offset
    z$contrasts <<- attr(x, "contrasts")
    z$xlevels <<- .getXlevels(mt, mf)
    z$call <<- cl
    z$terms <<- mt
    if (model) 
        z$model <<- mf
    if (ret.x) 
        z$x <<- x
    if (ret.y) 
        z$y <<- y
    if (!qr) 
        z$qr <<- NULL
    z
}

然后我尝试将每个 mf 实例重命名为 mf1mf2mf3,以查看 mf 的实际子集位置,但我卡住了,因为我遇到了错误(即使我以为我确保我在mf 之间有正确的引用。

我也尝试在这里和那里输入test &lt;&lt;- sum(mf$weights, na.rm=TRUE),但没有成功。

有没有人可以帮助我在正确的地方对权重求和?

【问题讨论】:

  • 当您说“数据子集在哪里?”时,您的意思是“NA 值在哪里被删除?”
  • @AllanCameron 是的,完全正确。用你的措辞来改进解释。
  • 他们不是在model.frame 内部而不是lm 内部删除吗?
  • @AllanCameron,这可以解释为什么我无法解决它哈哈。老实说,代码对我来说有点太复杂了。关键是我不明白它发生在哪里。
  • 我不确定(目前无法检查),但似乎很有可能。 na.action 被传递给 model.frame,我相信如果您查看源代码,您会看到 NA 值被丢弃的位置

标签: r function syntax lm


【解决方案1】:

这行得通:

编辑:作为一个小提示。我认为最好完全覆盖该函数。我认为它没有任何缺点,我注意到当您不使用 lm 时,其他一些依赖于 lm 的函数开始抱怨。

lm_plus <- function (formula, data, subset, weights, na.action, method = "qr", 
    model = TRUE, x = FALSE, y = FALSE, qr = TRUE, singular.ok = TRUE, 
    contrasts = NULL, offset, ...) 
{
    ret.x <- x
    ret.y <- y
    cl <- match.call()
    mf <- match.call(expand.dots = FALSE)
    m <- match(c("formula", "data", "subset", 
        "weights", "na.action", "offset"), 
        names(mf), 0L)
    mf <- mf[c(1L, m)]
    mf$drop.unused.levels <- TRUE
    mf[[1L]] <- quote(stats::model.frame)
    mf <- eval(mf, parent.frame())
    # test <<- sum(mf$weights, na.rm=TRUE)
    if (method == "model.frame") 
        return(mf)
    else if (method != "qr") 
        warning(gettextf("method = '%s' is not supported. Using 'qr'", 
            method), domain = NA)
    mt <- attr(mf, "terms")
    y <- model.response(mf, "numeric")
    w <- as.vector(model.weights(mf))
    sum_of_weights <- sum(as.vector(model.weights(mf)))
    if (!is.null(w) && !is.numeric(w)) 
        stop("'weights' must be a numeric vector")
    offset <- model.offset(mf)
    mlm <- is.matrix(y)
    ny <- if (mlm) 
        nrow(y)
    else length(y)
    if (!is.null(offset)) {
        if (!mlm) 
            offset <- as.vector(offset)
        if (NROW(offset) != ny) 
            stop(gettextf("number of offsets is %d, should equal %d (number of observations)", 
                NROW(offset), ny), domain = NA)
    }
    if (is.empty.model(mt)) {
        x <- NULL
        z <- list(coefficients = if (mlm) matrix(NA_real_, 0, 
            ncol(y)) else numeric(), residuals = y, fitted.values = 0 * 
            y, weights = w, rank = 0L, df.residual = if (!is.null(w)) sum(w != 
            0) else ny)
        if (!is.null(offset)) {
            z$fitted.values <- offset
            z$residuals <- y - offset
        }
    }
    else {
        x <- model.matrix(mt, mf, contrasts)
        z <- if (is.null(w)) 
            lm.fit(x, y, offset = offset, singular.ok = singular.ok, 
                ...)
        else lm.wfit(x, y, w, offset = offset, singular.ok = singular.ok, 
            ...)
    }
    class(z) <- c(if (mlm) "mlm", "lm")
    z$na.action <- attr(mf, "na.action")
    z$offset <- offset
    z$contrasts <- attr(x, "contrasts")
    z$xlevels <- .getXlevels(mt, mf)
    z$call <- cl
    z$terms <- mt
    if (model) 
        z$model <- mf
    if (ret.x) 
        z$x <- x
    if (ret.y) 
        z$y <- y
    if (!qr) 
        z$qr <- NULL
    z$df.residual <- sum_of_weights - length(coef(z))
    z
}

示例数据:

library(dplyr)
library(modelsummary)

set.seed(1024)

# individual (true) dataset
x <- round(rnorm(1e5))
y <- round(x + x^2 + rnorm(1e5))
ind <- data.frame(x, y)

# aggregated dataset
agg <- ind %>%
  group_by(x, y) %>%
  summarize(freq = n())

# Note that the last entry uses lm_plus
models <- list( 
  "True"                = lm(y ~ x, data = ind),
  "Aggregated"          = lm(y ~ x, data = agg),
  "Aggregated & W"      = lm(y ~ x, data = agg, weights=freq),
  "Aggregated & W & DF" = lm_plus(y ~ x, data = agg, weights=freq)
)

modelsummary(models, fmt=5)

【讨论】:

    猜你喜欢
    • 2016-02-02
    • 2017-01-11
    • 2013-02-24
    • 2011-09-22
    • 1970-01-01
    • 2018-06-04
    • 1970-01-01
    • 2015-09-10
    • 2018-10-13
    相关资源
    最近更新 更多