下载文件时如何打印字节？ -golang

Question

我想知道在下载文件时是否可以计算并打印下载的字节数。

out, err := os.Create("file.txt")
defer out.Close()
if err != nil {
    fmt.Println(fmt.Sprint(err))
    panic(err)
}
resp, err := http.Get("http://example.com/zip")
defer resp.Body.Close()
if err != nil {
    fmt.Println(fmt.Sprint(err))
    panic(err)
}

n, er := io.Copy(out, resp.Body)
if er != nil {
    fmt.Println(fmt.Sprint(err))
}
fmt.Println(n, "bytes ")

Answer 1

其他答案已经解释了PassThru。只需根据Dave Jack 的回答提供一个完整的回调函数示例。

package main

import (
    "fmt"
    "io"
    "net/http"
    "os"
    "strconv"
)

// writeCounter counts the number of bytes written to it.
type writeCounter struct {
    total      int64 // total size
    downloaded int64 // downloaded # of bytes transferred
    onProgress func(downloaded int64, total int64)
}

// Write implements the io.Writer interface.
//
// Always completes and never returns an error.
func (wc *writeCounter) Write(p []byte) (n int, e error) {
    n = len(p)
    wc.downloaded += int64(n)
    wc.onProgress(wc.downloaded, wc.total)
    return
}

func newWriter(size int64, onProgress func(downloaded, total int64)) io.Writer {
    return &writeCounter{total: size, onProgress: onProgress}
}

func main() {
    client := http.DefaultClient
    url := "http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/ForBiggerFun.mp4"
    saveTo := "/Users/tin/Desktop/ForBiggerFun.mp4"

    download(client, url, saveTo, func(downloaded, total int64) {
        fmt.Printf("Downloaded %d bytes for a total of %d\n", downloaded, total)
    })
}

func download(client *http.Client, url, filePath string, onProgress func(downloaded, total int64)) (err error) {
    // Create file writer
    file, err := os.Create(filePath)
    if err != nil {
        return
    }
    defer file.Close()

    // Determinate the file size
    resp, err := client.Head(url)
    if err != nil {
        return
    }
    contentLength := resp.Header.Get("content-length")
    length, err := strconv.Atoi(contentLength)
    if err != nil {
        return
    }

    // Make request
    resp, err = client.Get(url)
    if err != nil {
        return
    }
    defer resp.Body.Close()

    // pipe stream
    body := io.TeeReader(resp.Body, newWriter(int64(length), onProgress))
    _, err = io.Copy(file, body)
    return err
}

Answer 2

stdlib 现在提供类似 jimt 的 PassThru: io.TeeReader。它有助于简化一些事情：

// WriteCounter counts the number of bytes written to it.
type WriteCounter struct {
    Total int64 // Total # of bytes transferred
}

// Write implements the io.Writer interface.  
// 
// Always completes and never returns an error.
func (wc *WriteCounter) Write(p []byte) (int, error) {
    n := len(p)
    wc.Total += int64(n)
    fmt.Printf("Read %d bytes for a total of %d\n", n, wc.Total)
    return n, nil
}

func main() {

    // ...    

    // Wrap it with our custom io.Reader.
    src = io.TeeReader(src, &WriteCounter{})

    // ...
}

playground

Answer 3

如果我理解正确，您希望在数据传输时显示读取的字节数。大概是为了维护某种进度条什么的。在这种情况下，您可以使用 Go 的组合数据结构将读取器或写入器包装在自定义的 io.Reader 或 io.Writer 实现中。

它只是将各自的Read 或Write 调用转发到底层流，同时对它们返回的(int, error) 值做一些额外的工作。这是您可以在Go playground 上运行的示例。

package main

import (
    "bytes"
    "fmt"
    "io"
    "os"
    "strings"
)

// PassThru wraps an existing io.Reader.
//
// It simply forwards the Read() call, while displaying
// the results from individual calls to it.
type PassThru struct {
    io.Reader
    total int64 // Total # of bytes transferred
}

// Read 'overrides' the underlying io.Reader's Read method.
// This is the one that will be called by io.Copy(). We simply
// use it to keep track of byte counts and then forward the call.
func (pt *PassThru) Read(p []byte) (int, error) {
    n, err := pt.Reader.Read(p)
    pt.total += int64(n)

    if err == nil {
        fmt.Println("Read", n, "bytes for a total of", pt.total)
    }

    return n, err
}

func main() {
    var src io.Reader    // Source file/url/etc
    var dst bytes.Buffer // Destination file/buffer/etc

    // Create some random input data.
    src = bytes.NewBufferString(strings.Repeat("Some random input data", 1000))

    // Wrap it with our custom io.Reader.
    src = &PassThru{Reader: src}

    count, err := io.Copy(&dst, src)
    if err != nil {
        fmt.Println(err)
        os.Exit(1)
    }

    fmt.Println("Transferred", count, "bytes")
}

它生成的输出是这样的：

Read 512 bytes for a total of 512
Read 1024 bytes for a total of 1536
Read 2048 bytes for a total of 3584
Read 4096 bytes for a total of 7680
Read 8192 bytes for a total of 15872
Read 6128 bytes for a total of 22000
Transferred 22000 bytes

Answer 4

grab Go 包实现了文件下载的进度更新（和许多其他功能）。

在下载过程中打印进度更新的示例包含在以下演练中：http://cavaliercoder.com/blog/downloading-large-files-in-go.html

您基本上可以调用grab.GetAsync，它在新的 Go 例程中下载，然后从调用线程监视返回的grab.Response 的BytesTransferred 或Progress。