According to the docs, DeleteBucket states,
All objects (including all object versions and delete markers) in the bucket must be deleted before the bucket itself can be deleted.
Now, to delete the versions from a versioning-enabled bucket, we can
- use DeleteObject, which states,
To remove a specific version, you must be the bucket owner and you must use the version Id subresource. Using this subresource permanently deletes the version.
- use DeleteObjects, which similarly states,
In the XML, you provide the object key names, and optionally, version IDs if you want to delete a specific version of the object from a versioning-enabled bucket.
I put together a sample program, that I tested against LocalStack, after using the following commands(prerequisites - Docker, Docker Compose, AWS CLI) to create a bucket and populate it with files to include versions.
curl -O https://raw.githubusercontent.com/localstack/localstack/master/docker-compose.yml
export SERVICES="s3"
docker-compose up
export AWS_ACCESS_KEY_ID="test"
export AWS_SECRET_ACCESS_KEY="test"
export AWS_DEFAULT_REGION="us-east-1"
aws --endpoint-url=http://localhost:4566 s3 mb s3://testbucket
aws --endpoint-url=http://localhost:4566 s3api put-bucket-versioning --bucket testbucket --versioning-configuration Status=Enabled
for i in 1 2 3; do
aws --endpoint-url=http://localhost:4566 s3 cp main.go s3://testbucket/main.go
aws --endpoint-url=http://localhost:4566 s3 cp go.mod s3://testbucket/go.mod
done
aws --endpoint-url=http://localhost:4566 s3api list-object-versions --bucket testbucket
Set the following environment variables before running it
export AWS_ENDPOINT="http://localhost:4566"
export S3_BUCKET="testbucket"
package main
import (
"context"
"fmt"
"log"
"os"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
)
type s3Client struct {
*s3.Client
}
func main() {
awsEndpoint := os.Getenv("AWS_ENDPOINT")
bucketName := os.Getenv("S3_BUCKET")
cfg, err := config.LoadDefaultConfig(context.TODO(),
config.WithEndpointResolverWithOptions(aws.EndpointResolverWithOptionsFunc(
func(service, region string, options ...interface{}) (aws.Endpoint, error) {
return aws.Endpoint{
URL: awsEndpoint,
HostnameImmutable: true,
}, nil
})),
)
if err != nil {
log.Fatalf("Cannot load the AWS configs: %s", err)
}
serviceClient := s3.NewFromConfig(cfg)
client := &s3Client{
Client: serviceClient,
}
fmt.Printf(">>> Bucket: %s\n", bucketName)
objects, err := client.listObjects(bucketName)
if err != nil {
log.Fatal(err)
}
if len(objects) > 0 {
fmt.Printf(">>> List objects in the bucket: \n")
for _, object := range objects {
fmt.Printf("%s\n", object)
}
} else {
fmt.Printf(">>> No objects in the bucket.\n")
}
if client.versioningEnabled(bucketName) {
fmt.Printf(">>> Versioning is enabled.\n")
objectVersions, err := client.listObjectVersions(bucketName)
if err != nil {
log.Fatal(err)
}
if len(objectVersions) > 0 {
fmt.Printf(">>> List objects with versions: \n")
for key, versions := range objectVersions {
fmt.Printf("%s: ", key)
for _, version := range versions {
fmt.Printf("\n\t%s ", version)
}
fmt.Println()
}
}
if len(objectVersions) > 0 {
fmt.Printf(">>> Delete objects with versions.\n")
if err := client.deleteObjects(bucketName, objectVersions); err != nil {
log.Fatal(err)
}
objectVersions, err = client.listObjectVersions(bucketName)
if err != nil {
log.Fatal(err)
}
if len(objectVersions) > 0 {
fmt.Printf(">>> List objects with versions after deletion: \n")
for key, version := range objectVersions {
fmt.Printf("%s: %s\n", key, version)
}
} else {
fmt.Printf(">>> No objects in the bucket after deletion.\n")
}
}
}
fmt.Printf(">>> Delete the bucket.\n")
if err := client.deleteBucket(bucketName); err != nil {
log.Fatal(err)
}
}
func (c *s3Client) versioningEnabled(bucket string) bool {
output, err := c.GetBucketVersioning(context.TODO(), &s3.GetBucketVersioningInput{
Bucket: aws.String(bucket),
})
if err != nil {
return false
}
return output.Status == "Enabled"
}
func (c *s3Client) listObjects(bucket string) ([]string, error) {
var objects []string
output, err := c.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
Bucket: aws.String(bucket),
})
if err != nil {
return nil, err
}
for _, object := range output.Contents {
objects = append(objects, aws.ToString(object.Key))
}
return objects, nil
}
func (c *s3Client) listObjectVersions(bucket string) (map[string][]string, error) {
var objectVersions = make(map[string][]string)
output, err := c.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
Bucket: aws.String(bucket),
})
if err != nil {
return nil, err
}
for _, object := range output.Versions {
if _, ok := objectVersions[aws.ToString(object.Key)]; ok {
objectVersions[aws.ToString(object.Key)] = append(objectVersions[aws.ToString(object.Key)], aws.ToString(object.VersionId))
} else {
objectVersions[aws.ToString(object.Key)] = []string{aws.ToString(object.VersionId)}
}
}
return objectVersions, err
}
func (c *s3Client) deleteObjects(bucket string, objectVersions map[string][]string) error {
var identifiers []types.ObjectIdentifier
for key, versions := range objectVersions {
for _, version := range versions {
identifiers = append(identifiers, types.ObjectIdentifier{
Key: aws.String(key),
VersionId: aws.String(version),
})
}
}
_, err := c.DeleteObjects(context.TODO(), &s3.DeleteObjectsInput{
Bucket: aws.String(bucket),
Delete: &types.Delete{
Objects: identifiers,
},
})
if err != nil {
return err
}
return nil
}
func (c *s3Client) deleteBucket(bucket string) error {
_, err := c.DeleteBucket(context.TODO(), &s3.DeleteBucketInput{
Bucket: aws.String(bucket),
})
if err != nil {
return err
}
return nil
}
Note in the listObjectVersions method, I am mapping the VersionIds with the Keys.
for _, object := range output.Versions {
if _, ok := objectVersions[aws.ToString(object.Key)]; ok {
objectVersions[aws.ToString(object.Key)] = append(objectVersions[aws.ToString(object.Key)], aws.ToString(object.VersionId))
} else {
objectVersions[aws.ToString(object.Key)] = []string{aws.ToString(object.VersionId)}
}
}
Then in the deleteObjects method, when passing the ObjectIdentifiers, I pass the Key and the ObjectIds for all the versions of an object.
for key, versions := range objectVersions {
for _, version := range versions {
identifiers = append(identifiers, types.ObjectIdentifier{
Key: aws.String(key),
VersionId: aws.String(version),
})
}
}
