如何使用适用于 GOlang 的 AWS 开发工具包清空 S3 存储桶？

Question

目标：使用适用于 GOlang 的 AWS 开发工具包清空现有 S3 存储桶。

Answer 1

AWS SDK 现在有 BatchDeleteIterator 可以完成这项工作。通过Amazon docs 提供的示例。

package main

import (
    "github.com/aws/aws-sdk-go/aws"
    "github.com/aws/aws-sdk-go/aws/session"
    "github.com/aws/aws-sdk-go/service/s3"
    "github.com/aws/aws-sdk-go/service/s3/s3manager"

    "fmt"
    "os"
)

//    go run s3_delete_objects BUCKET

func main() {
    if len(os.Args) != 2 {
        exitErrorf("Bucket name required\nUsage: %s BUCKET", os.Args[0])
    }

    bucket := os.Args[1]

    // Initialize a session in us-west-2 that the SDK will use to load
    // credentials from the shared credentials file ~/.aws/credentials.
    sess, _ := session.NewSession(&aws.Config{
        Region: aws.String("us-west-2")},
    )

    // Create S3 service client
    svc := s3.New(sess)

    // Setup BatchDeleteIterator to iterate through a list of objects.
    iter := s3manager.NewDeleteListIterator(svc, &s3.ListObjectsInput{
        Bucket: aws.String(bucket),
    })

    // Traverse iterator deleting each object
    if err := s3manager.NewBatchDeleteWithClient(svc).Delete(aws.BackgroundContext(), iter); err != nil {
        exitErrorf("Unable to delete objects from bucket %q, %v", bucket, err)
    }

    fmt.Printf("Deleted object(s) from bucket: %s", bucket)
}

func exitErrorf(msg string, args ...interface{}) {
    fmt.Fprintf(os.Stderr, msg+"\n", args...)
    os.Exit(1)
}

Answer 2

注意：这些是代码 sn-ps，可能需要您自己进行更改才能运行。

您需要实现以下方法：

  //EmptyBucket empties the Amazon S3 bucket
    func (s awsS3) EmptyBucket(bucket string) error {
        log.Info("removing objects from S3 bucket : ", bucket)
        params := &s3.ListObjectsInput{
            Bucket: aws.String(bucket),
        }
        for {
            //Requesting for batch of objects from s3 bucket
            objects, err := s.Client.ListObjects(params)
            if err != nil {
                return err
            }
            //Checks if the bucket is already empty
            if len((*objects).Contents) == 0 {
                log.Info("Bucket is already empty")
                return nil
             }
            log.Info("First object in batch | ", *(objects.Contents[0].Key))

            //creating an array of pointers of ObjectIdentifier
            objectsToDelete := make([]*s3.ObjectIdentifier, 0, 1000)
            for _, object := range (*objects).Contents {
                obj := s3.ObjectIdentifier{
                    Key: object.Key,
                }
                objectsToDelete = append(objectsToDelete, &obj)
            }
            //Creating JSON payload for bulk delete
            deleteArray := s3.Delete{Objects: objectsToDelete}
            deleteParams := &s3.DeleteObjectsInput{
                Bucket: aws.String(bucket),
                Delete: &deleteArray,
            }
            //Running the Bulk delete job (limit 1000)
            _, err = s.Client.DeleteObjects(deleteParams)
            if err != nil {
                return err
            }
            if *(*objects).IsTruncated { //if there are more objects in the bucket, IsTruncated = true
                params.Marker = (*deleteParams).Delete.Objects[len((*deleteParams).Delete.Objects)-1].Key
                log.Info("Requesting next batch | ", *(params.Marker))
            } else { //if all objects in the bucket have been cleaned up.
                break
            }
        }
        log.Info("Emptied S3 bucket : ", bucket)
        return nil
    } 

更新：最新版本的适用于 GO 的 AWS 开发工具包解决了我之前遇到的问题。

Answer 3

适用于 Go 的 AWS 开发工具包具有 Amazon S3 批处理抽象。看看here。

Answer 4

不要忘记默认情况下 ListObjects 最多只能返回 1000 个存储桶项。如果您可能有超过 1000 个，请检查返回值的 IsTruncated 属性。如果为 true，则使用返回值中的 NextMarker 属性来获取接下来的 1000 个项目。

在 Go 开发指南中查看我的示例：http://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/s3-example-basic-bucket-operations.html#s3-examples-bucket-ops-delete-all-bucket-items