Skip to content

S3 Provider

The S3 provider enables secure and efficient integration with Amazon S3 and S3-compatible storage services like MinIO, DigitalOcean Spaces, and Backblaze B2. It provides a comprehensive API for bucket and object operations with built-in security features and performance optimizations.

Key Features

  • Multi-Cloud Support: Works with AWS S3, MinIO, DigitalOcean Spaces, Backblaze B2, and other S3-compatible services
  • Security: TLS/SSL encryption, secure credential handling, presigned URLs
  • Performance: Automatic multipart uploads, connection pooling, concurrent operations
  • Advanced Operations: Range downloads, object copying, metadata management
  • Encryption: Server-side encryption (SSE-S3, SSE-KMS, SSE-C) on uploads and copies
  • Object Lock / WORM: Retention (Governance/Compliance), legal holds, bucket-level default retention
  • Versioning: List object versions and target specific versions on reads and deletes
  • Logging: Comprehensive operation logging with timing and performance metrics

Basic Usage

Creating a Client

package main

import (
    "context"
    "log"

    "github.com/oddbit-project/blueprint/provider/s3"
    blueprintLog "github.com/oddbit-project/blueprint/log"
)

func main() {
    // Create configuration
    config := s3.NewConfig()
    config.Endpoint = "localhost:9000"  // MinIO endpoint
    config.Region = "us-east-1"
    config.AccessKeyID = "minioadmin"
    config.UseSSL = false  // Disable for local MinIO

    // Set secret key via environment variable
    config.PasswordEnvVar = "S3_SECRET_KEY"

    // Create logger
    logger := blueprintLog.New("s3-client")

    // Create client
    client, err := s3.NewClient(config, logger)
    if err != nil {
        log.Fatal(err)
    }

    // Connect to S3
    if err := client.Connect(context.Background()); err != nil {
        log.Fatal(err)
    }

    defer client.Close()
}

Configuration Options

The S3 provider supports comprehensive configuration through the Config struct:

type Config struct {
    // Connection settings
    Endpoint    string `json:"endpoint"`    // S3 endpoint URL
    Region      string `json:"region"`      // AWS region
    AccessKeyID string `json:"accessKeyId"` // Access key ID

    // Secure credential handling
    secure.DefaultCredentialConfig  // Secret key from environment/file

    // Optional defaults
    DefaultBucket string `json:"defaultBucket"`

    // Behavior settings
    ForcePathStyle bool `json:"forcePathStyle"` // Path-style addressing
    UseAccelerate  bool `json:"useAccelerate"`  // S3 transfer acceleration
    UseSSL         bool `json:"useSSL"`         // SSL/TLS encryption

    // Timeout settings (seconds)
    TimeoutSeconds       int `json:"timeoutSeconds"`       // Default: 300
    UploadTimeoutSeconds int `json:"uploadTimeoutSeconds"` // Default: 1800

    // Multipart upload settings
    MultipartThreshold int64 `json:"multipartThreshold"` // Default: 100MB
    PartSize           int64 `json:"partSize"`           // Default: 10MB
    MaxUploadParts     int   `json:"maxUploadParts"`     // Default: 10000
    Concurrency        int   `json:"concurrency"`        // Default: 5

    // TLS configuration
    tls.ClientConfig

    // Connection pooling settings
    MaxIdleConns        int           `json:"maxIdleConns"`        // Default: 50
    MaxIdleConnsPerHost int           `json:"maxIdleConnsPerHost"` // Default: 10
    MaxConnsPerHost     int           `json:"maxConnsPerHost"`     // Default: 20
    IdleConnTimeout     time.Duration `json:"idleConnTimeout"`     // Default: 60s

    // Retry settings
    MaxRetries int    `json:"maxRetries"` // Default: 3
    RetryMode  string `json:"retryMode"`  // "standard" or "adaptive"
}

Bucket Operations

Creating and Managing Buckets

// Create a bucket
bucket, err := client.Bucket("my-bucket")
if err != nil {
    log.Fatal(err)
}

// Create bucket with options
bucketOpts := s3.BucketOptions{
    Region: "us-west-2",
    ACL:    "private",
}
if err := bucket.Create(ctx, bucketOpts); err != nil {
    log.Fatal(err)
}

// Check if bucket exists
exists, err := bucket.Exists(ctx)
if err != nil {
    log.Fatal(err)
}

// List all buckets
buckets, err := client.ListBuckets(ctx)
if err != nil {
    log.Fatal(err)
}

for _, bucket := range buckets {
    fmt.Printf("Bucket: %s, Created: %s\n", 
        bucket.Name, bucket.CreationDate)
}

// Delete bucket (must be empty)
if err := bucket.Delete(ctx); err != nil {
    log.Fatal(err)
}

Object Operations

Basic Upload and Download

// Upload object
file, err := os.Open("local-file.txt")
if err != nil {
    log.Fatal(err)
}
defer file.Close()

stat, _ := file.Stat()
err = bucket.PutObject(ctx, "remote-file.txt", file, stat.Size())
if err != nil {
    log.Fatal(err)
}

// Download object
reader, err := bucket.GetObject(ctx, "remote-file.txt")
if err != nil {
    log.Fatal(err)
}
defer reader.Close()

// Write to local file
outFile, err := os.Create("downloaded-file.txt")
if err != nil {
    log.Fatal(err)
}
defer outFile.Close()

_, err = io.Copy(outFile, reader)
if err != nil {
    log.Fatal(err)
}

Upload with Options

// Upload with metadata and options
options := s3.ObjectOptions{
    ContentType:        "text/plain",
    CacheControl:       "max-age=3600",
    ContentDisposition: "attachment; filename=data.txt",
    Metadata: map[string]string{
        "author":      "user123",
        "description": "Important data file",
    },
    Tags: map[string]string{
        "project":     "analytics",
        "environment": "production",
    },
    StorageClass: "STANDARD_IA",
}

err = bucket.PutObject(ctx, "data.txt", file, size, options)

Streaming Operations

// Stream upload (unknown size)
err = bucket.PutObjectStream(ctx, "stream-file.txt", reader)

// Stream download to writer
var buf bytes.Buffer
err = bucket.GetObjectStream(ctx, "large-file.bin", &buf)

Advanced Upload Options

// Advanced upload with detailed control
uploadOpts := s3.UploadOptions{
    ObjectOptions: s3.ObjectOptions{
        ContentType: "application/octet-stream",
        Metadata: map[string]string{
            "upload-method": "advanced",
        },
    },
    LeavePartsOnError: false,  // Clean up on failure
    MaxUploadParts:    5000,   // Custom part limit
    Concurrency:       10,     // Parallel uploads
}

err = bucket.PutObjectAdvanced(ctx, "large-file.bin", reader, size, uploadOpts)

Advanced Download Operations

Range Downloads

// Download specific byte range
start := int64(1024)
end := int64(4095)
reader, err := bucket.GetObjectRange(ctx, "large-file.bin", start, end)
if err != nil {
    log.Fatal(err)
}
defer reader.Close()

// Download range to writer
var buf bytes.Buffer
err = bucket.GetObjectStreamRange(ctx, "file.bin", &buf, start, end)

Advanced Download Options

// Advanced download with options
downloadOpts := s3.DownloadOptions{
    StartByte:   &start,
    EndByte:     &end,
    Concurrency: 5,
    PartSize:    int64(10 * 1024 * 1024), // 10MB parts
}

var output bytes.Buffer
err = bucket.GetObjectAdvanced(ctx, "file.bin", &output, downloadOpts)

Object Listing and Metadata

List Objects

// List all objects
objects, err := bucket.ListObjects(ctx)
if err != nil {
    log.Fatal(err)
}

// List with options
listOpts := s3.ListOptions{
    Prefix:     "documents/",
    MaxKeys:    100,
    StartAfter: "documents/file-050.txt",
}

filteredObjects, err := bucket.ListObjects(ctx, listOpts)
for _, obj := range filteredObjects {
    fmt.Printf("Object: %s, Size: %d, Modified: %s\n",
        obj.Key, obj.Size, obj.LastModified)
}

// List all object versions (including delete markers) on a versioned bucket
versions, err := bucket.ListObjects(ctx, s3.ListOptions{Prefix: "report.pdf", Versions: true})
for _, v := range versions {
    fmt.Printf("Version: %s, Latest: %t, DeleteMarker: %t\n",
        v.VersionID, v.IsLatest, v.IsDeleteMarker)
}

Object Metadata and Existence

// Check if object exists
exists, err := bucket.ObjectExists(ctx, "file.txt")

// Get object metadata
info, err := bucket.HeadObject(ctx, "file.txt")
if err != nil {
    log.Fatal(err)
}

fmt.Printf("Size: %d, ETag: %s, ContentType: %s\n",
    info.Size, info.ETag, info.ContentType)

Object Operations

// Copy object
err = bucket.CopyObject(ctx, "source.txt", "destination-bucket", "dest.txt")

// Upload and learn which version was written
v, err := bucket.PutObjectInfo(ctx, "report.pdf", reader, size)
// v.Bucket, v.Key, v.VersionID, v.ETag, v.Size

// Copy a specific source version and get the version the copy created
c, err := bucket.CopyObjectVersion(ctx, s3.CopySource{Name: "report.pdf", VersionID: v.VersionID},
    "destination-bucket", "report-copy.pdf")

// Delete object
err = bucket.DeleteObject(ctx, "old-file.txt")

// Delete a specific version, bypassing GOVERNANCE retention (WORM buckets)
err = bucket.DeleteObject(ctx, "old-file.txt", s3.DeleteOptions{
    VersionID:        "versionId",
    GovernanceBypass: true,
})

Presigned URLs

Presigned URLs enable secure temporary access to S3 objects without exposing credentials:

// Generate presigned URL for download (valid for 1 hour)
downloadURL, err := bucket.PresignGetObject(ctx, "file.txt", time.Hour)
if err != nil {
    log.Fatal(err)
}
fmt.Printf("Download URL: %s\n", downloadURL)

// Generate presigned URL for upload (valid for 30 minutes)
uploadURL, err := bucket.PresignPutObject(ctx, "new-file.txt", 30*time.Minute)
if err != nil {
    log.Fatal(err)
}

// Client can now upload directly using the presigned URL
// curl -X PUT --upload-file local.txt "$uploadURL"

// Generate presigned URL for metadata access
headURL, err := bucket.PresignHeadObject(ctx, "file.txt", time.Hour)

Object Lock (WORM)

Object Lock provides write-once-read-many (WORM) protection: objects (or specific versions) cannot be deleted or overwritten until a retention period expires or a legal hold is removed.

Important: Object Lock can only be enabled when the bucket is created — it cannot be turned on for an existing bucket. Enabling Object Lock implicitly enables versioning on the bucket.

Enabling Object Lock on a Bucket

bucket, err := client.Bucket("compliance-bucket")
if err != nil {
    log.Fatal(err)
}

// Object Lock must be enabled at creation time
err = bucket.Create(ctx, s3.BucketOptions{ObjectLocking: true})
if err != nil {
    log.Fatal(err)
}

Retention Modes

  • s3.RetentionGovernance — objects are protected, but users with the s3:BypassGovernanceRetention permission can shorten or remove retention using GovernanceBypass.
  • s3.RetentionCompliance — objects are protected from everyone, including the root account, until the retention date passes. Retention cannot be shortened or removed.

Setting Retention at Upload Time

Retention and legal hold can be applied during upload through ObjectOptions:

err = bucket.PutObject(ctx, "report.pdf", reader, size, s3.ObjectOptions{
    LockMode:        s3.RetentionGovernance,
    RetainUntilDate: time.Now().Add(30 * 24 * time.Hour), // locked for 30 days
})

Managing Retention on Existing Objects

// Apply or extend retention
err = bucket.SetObjectRetention(ctx, "report.pdf", s3.RetentionOptions{
    Mode:            s3.RetentionGovernance,
    RetainUntilDate: time.Now().Add(7 * 24 * time.Hour),
})

// Read current retention state
ret, err := bucket.GetObjectRetention(ctx, "report.pdf")
if err != nil {
    log.Fatal(err)
}
fmt.Printf("Mode: %s, RetainUntil: %s\n", ret.Mode, ret.RetainUntilDate)

// Shorten or remove a GOVERNANCE retention (requires bypass permission)
err = bucket.SetObjectRetention(ctx, "report.pdf", s3.RetentionOptions{
    Mode:             s3.RetentionGovernance,
    RetainUntilDate:  time.Now().Add(1 * time.Hour),
    GovernanceBypass: true,
})

A legal hold protects an object indefinitely, independent of any retention period, until it is explicitly removed.

// Place a legal hold
err = bucket.SetObjectLegalHold(ctx, "report.pdf", true)

// Check legal hold status
held, err := bucket.GetObjectLegalHold(ctx, "report.pdf")

// Release the legal hold
err = bucket.SetObjectLegalHold(ctx, "report.pdf", false)

// Place or release a legal hold on a specific version
err = bucket.SetObjectLegalHoldVersion(ctx, "report.pdf", versionID, true)

Note: GetObjectLegalHold returns false (not an error) when no legal hold was ever set.

Bucket-Level Default Retention

A bucket can define a default retention that is automatically applied to every newly uploaded object that does not specify its own retention.

// Apply COMPLIANCE retention of 1 year by default
err = bucket.SetObjectLockConfig(ctx, s3.RetentionCompliance, 1, s3.ValidityYears)

// Or 30 days by default
err = bucket.SetObjectLockConfig(ctx, s3.RetentionGovernance, 30, s3.ValidityDays)

// Read the current configuration
cfg, err := bucket.GetObjectLockConfig(ctx)
if err != nil {
    log.Fatal(err)
}
fmt.Printf("Enabled: %t, Mode: %s, Validity: %d %s\n",
    cfg.Enabled, cfg.Mode, cfg.Validity, cfg.Unit)

// Clear the default retention (Object Lock stays enabled on the bucket)
err = bucket.SetObjectLockConfig(ctx, "", 0, "")

Validity units are s3.ValidityDays and s3.ValidityYears.

Deleting Locked Objects

Enabling Object Lock also enables versioning. A plain DeleteObject (no VersionID) writes a delete marker and leaves the locked version intact. To remove a specific version that is under GOVERNANCE retention, pass its version ID with GovernanceBypass (requires the s3:BypassGovernanceRetention permission). Obtain the version id from HeadObject or a version-aware ListObjects:

head, err := bucket.HeadObject(ctx, "report.pdf")
if err != nil {
    log.Fatal(err)
}

err = bucket.DeleteObject(ctx, "report.pdf", s3.DeleteOptions{
    VersionID:        head.VersionID,
    GovernanceBypass: true,
})

COMPLIANCE-locked versions cannot be deleted by anyone until the retention period expires.

Error Handling

The S3 provider defines specific error constants for common scenarios:

const (
    ErrNilConfig           = "Config is nil"
    ErrMissingEndpoint     = "missing endpoint"
    ErrMissingRegion       = "missing region"
    ErrBucketNotFound      = "bucket not found"
    ErrBucketAlreadyExists = "bucket already exists"
    ErrObjectNotFound      = "object not found"
    ErrClientNotConnected  = "client not connected"
    ErrTLSRequiresSSL      = "tlsEnable requires useSSL"
)

Handle errors appropriately:

if err := bucket.Create(ctx); err != nil {
    if err == s3.ErrBucketAlreadyExists {
        log.Println("Bucket already exists, continuing...")
    } else {
        log.Fatal("Failed to create bucket:", err)
    }
}

// Check for object existence
exists, err := bucket.ObjectExists(ctx, "file.txt")
if err != nil {
    log.Fatal("Error checking object:", err)
}
if !exists {
    log.Println("Object does not exist")
}

Performance Optimization

Multipart Upload Configuration

The provider automatically uses multipart uploads for large files:

config := s3.NewConfig()
config.MultipartThreshold = 50 * 1024 * 1024  // 50MB threshold
config.PartSize = 5 * 1024 * 1024             // 5MB parts
config.Concurrency = 10                       // 10 concurrent uploads
config.MaxUploadParts = 5000                  // Maximum parts

Timeout Configuration

Configure timeouts based on your use case:

config.TimeoutSeconds = 300        // 5 minutes for regular operations
config.UploadTimeoutSeconds = 3600 // 1 hour for large file uploads

Connection Pooling

Optimize HTTP connection reuse for better performance:

// Default connection pooling (suitable for most use cases)
config.MaxIdleConns = 50        // Total idle connections across all hosts
config.MaxIdleConnsPerHost = 10 // Idle connections per host
config.MaxConnsPerHost = 20     // Maximum connections per host
config.IdleConnTimeout = 60 * time.Second // Keep idle connections for 60s

// High-performance configuration for busy workloads
config.MaxIdleConns = 200
config.MaxIdleConnsPerHost = 50
config.MaxConnsPerHost = 100
config.IdleConnTimeout = 300 * time.Second

// Conservative configuration for resource-constrained environments
config.MaxIdleConns = 10
config.MaxIdleConnsPerHost = 2
config.MaxConnsPerHost = 5
config.IdleConnTimeout = 30 * time.Second

Connection Settings

config.MaxRetries = 5               // Increase for unreliable networks
config.RetryMode = "adaptive"       // Use adaptive retry logic

Connection Pooling Best Practices

Performance Tuning Guidelines

Connection pooling significantly improves performance by reusing HTTP connections. Choose settings based on your workload:

For High-Throughput Applications: 

config := s3.NewConfig()
// Allow more connections for concurrent operations
config.MaxIdleConns = 200
config.MaxIdleConnsPerHost = 50
config.MaxConnsPerHost = 100
config.IdleConnTimeout = 5 * time.Minute // Keep connections longer

For Batch Processing: 

config := s3.NewConfig()
// Moderate pooling for sustained workloads
config.MaxIdleConns = 100
config.MaxIdleConnsPerHost = 20
config.MaxConnsPerHost = 50
config.IdleConnTimeout = 2 * time.Minute

For Resource-Constrained Environments: 

config := s3.NewConfig()
// Minimal connection usage
config.MaxIdleConns = 10
config.MaxIdleConnsPerHost = 2
config.MaxConnsPerHost = 5
config.IdleConnTimeout = 30 * time.Second

Monitoring Connection Pool Efficiency

// Enable verbose logging to monitor connection reuse
logger := blueprintLog.New("s3-client")

// The provider logs connection establishment and reuse patterns
client, err := s3.NewClient(config, logger)

Connection Pool Sizing Rules

  1. MaxIdleConns: Set to 2-4x your expected concurrent operations
  2. MaxIdleConnsPerHost: Typically 10-25% of MaxIdleConns
  3. MaxConnsPerHost: Should accommodate peak concurrent requests per endpoint
  4. IdleConnTimeout: Balance between connection reuse and resource cleanup
  5. Short workloads: 30-60 seconds
  6. Long-running services: 2-5 minutes
  7. Always-on applications: 5-10 minutes

Workload-Specific Examples

Web Application with S3 Integration: 

// Handles user uploads/downloads with moderate concurrency
config.MaxIdleConns = 50
config.MaxIdleConnsPerHost = 15
config.MaxConnsPerHost = 30
config.IdleConnTimeout = 2 * time.Minute

Data Pipeline Processing: 

// High-volume batch operations
config.MaxIdleConns = 150
config.MaxIdleConnsPerHost = 30
config.MaxConnsPerHost = 60
config.IdleConnTimeout = 5 * time.Minute

Microservice with Occasional S3 Access: 

// Minimal overhead for infrequent operations
config.MaxIdleConns = 20
config.MaxIdleConnsPerHost = 5
config.MaxConnsPerHost = 10
config.IdleConnTimeout = 60 * time.Second

Security Features

Secure Credential Handling

// Use environment variables for secrets
config.PasswordEnvVar = "S3_SECRET_KEY"

// Or use encrypted files
config.PasswordFile = "/secure/path/secret.key"

TLS Configuration

// Enable SSL/TLS for production
config.UseSSL = true

// Custom TLS configuration
config.ClientConfig = tls.ClientConfig{
    TLSEnable: true,
    TLSCert:   "/path/to/client.crt",
    TLSKey:    "/path/to/client.key",
    TLSCA:     "/path/to/ca.crt",
}

The custom TLS settings are applied only with TLSEnable, over SSL. Setting any of them (TLSCA, TLSCert, TLSKey, TLSInsecureSkipVerify or a key password) without TLSEnable makes Validate() fail with tls.ErrTLSNotEnabled, and TLSEnable with UseSSL false fails with ErrTLSRequiresSSL, instead of the settings being silently dropped (#116). With TLSEnable and UseSSL, TLSCert and TLSKey must be set together, and a key password requires both: otherwise Validate() fails with tls.ErrTLSIncompleteKeyPair, instead of connecting without the client certificate (#115).

Server-Side Encryption

Encryption options are applied on PutObject* and CopyObject:

// SSE-S3 (server-managed keys)
options := s3.ObjectOptions{
    ServerSideEncryption: s3.SSEAlgorithmAES256,
}

// SSE-KMS
options := s3.ObjectOptions{
    ServerSideEncryption:    s3.SSEAlgorithmKMS,
    SSEKMSKeyId:             "arn:aws:kms:region:account:key/key-id",
    SSEKMSEncryptionContext: map[string]string{"project": "analytics"},
}

// SSE-C (customer-provided key, base64-encoded 32-byte key)
options := s3.ObjectOptions{
    SSECustomerKey: base64Key,
}

A customer-provided key (SSE-C) takes precedence over ServerSideEncryption. Note: reads of SSE-C objects require the same key to be supplied; the high-level GetObject/HeadObject do not yet accept an SSE-C key, so use MinioClient() directly for SSE-C downloads. aws:kms:dsse is not supported by the underlying client and returns an error.

To copy an object whose source is SSE-C encrypted, supply the source key via SourceSSECustomerKey (and, if the destination should be encrypted, the destination SSE options too):

err = bucket.CopyObject(ctx, "src.txt", "dst-bucket", "dst.txt", s3.ObjectOptions{
    SourceSSECustomerKey: srcBase64Key, // decrypt the SSE-C source
    SSECustomerKey:       dstBase64Key, // optional: re-encrypt the destination
})

Object Versions

GetObject, HeadObject, GetObjectRetention, and GetObjectLegalHold accept an optional version id to target a specific version:

reader, err := bucket.GetObject(ctx, "report.pdf", versionID)
info, err := bucket.HeadObject(ctx, "report.pdf", versionID)

To pin a version, record the one each write returns and read by that id:

v, err := bucket.PutObjectInfo(ctx, "report.pdf", reader, size)
// store v.VersionID, then later:
reader, err := bucket.GetObject(ctx, "report.pdf", v.VersionID)
err = bucket.SetObjectLegalHoldVersion(ctx, "report.pdf", v.VersionID, true)

Notes:

  • VersionID == "" means the bucket is not versioned: the write is not pinned. On a bucket with versioning suspended the server may return "" or "null"; neither identifies a version that later writes cannot overwrite.
  • For copies, ObjectVersion.Bucket is the destination bucket and Size is -1 (S3 does not report it).
  • ETag is not a content MD5 for multipart uploads or SSE-KMS/SSE-C objects.
  • Copies are single-request (5 GiB source limit on AWS) and use the default operation timeout. A copy that times out on the client may still complete on the server, leaving a version the caller did not record. A copy whose response carries no ETag fails with ErrEmptyCopyResult.
  • Copies apply ObjectOptions.LegalHold, and LockMode with RetainUntilDate, to the destination. Set LockMode and RetainUntilDate together: if only one is set, the copy is written without either (a put with only one fails). Lock options on a copy into a bucket without Object Lock fail.
  • Version-pinned reads are available on GetObject, HeadObject, retention and legal hold only, not on stream, range, advanced or presigned reads.

Integration Examples

AWS S3

config := s3.NewConfig()
config.Region = "us-west-2"
config.UseSSL = true
config.AccessKeyID = "AKIA..."
config.PasswordEnvVar = "AWS_SECRET_ACCESS_KEY"

MinIO

config := s3.NewConfig()
config.Endpoint = "localhost:9000"
config.Region = "us-east-1"
config.UseSSL = false
config.ForcePathStyle = true
config.AccessKeyID = "minioadmin"
config.PasswordEnvVar = "MINIO_SECRET_KEY"

DigitalOcean Spaces

config := s3.NewConfig()
config.Endpoint = "nyc3.digitaloceanspaces.com"
config.Region = "us-east-1"
config.UseSSL = true
config.AccessKeyID = "your-spaces-key"
config.PasswordEnvVar = "DO_SPACES_SECRET"

Testing

Integration tests spin up MinIO via testcontainers and require a running Docker daemon. They are skipped automatically in short mode.

# Unit tests only (no Docker required)
go test -short ./provider/s3/...

# Full suite including integration tests (requires Docker)
go test ./provider/s3/...

# Run benchmarks
go test -bench=. ./provider/s3/...

Server-side encryption requires a TLS connection, so the SSE integration test (TestIntegrationSSE) starts a dedicated TLS- and KMS-enabled MinIO container with a self-signed certificate; no extra setup is needed.

CLI Sample

A complete CLI sample is available in samples/s3-client/ demonstrating all features including connection pooling:

cd samples/s3-client
go build -o s3-cli main.go

# Basic operations
./s3-cli create-bucket my-bucket
./s3-cli upload my-bucket file.txt
./s3-cli download my-bucket file.txt downloaded.txt
./s3-cli presign-get my-bucket file.txt 60

# Connection pooling examples
./s3-cli --max-idle-conns 100 --max-conns-per-host 50 upload my-bucket large-file.zip
./s3-cli --verbose upload my-bucket file.txt  # Shows connection pool configuration

Troubleshooting

Common Issues

Connection Refused - Verify endpoint URL and port - Check if S3 service is running - Validate network connectivity

Access Denied - Verify access key and secret key - Check IAM permissions - Ensure bucket policies allow operation

SSL/TLS Errors - For local development, use UseSSL: false - Verify SSL certificates for production - Check TLS configuration

Multipart Upload Failures - Adjust part size and concurrency - Increase upload timeout - Check available memory and disk space

Poor Performance - Tune connection pooling settings for your workload - Increase MaxIdleConns and MaxConnsPerHost for high concurrency - Monitor connection reuse with verbose logging - Consider increasing IdleConnTimeout for sustained workloads

Logging

Enable detailed logging for debugging:

logger := blueprintLog.New("s3-client")

// The provider logs:
// - Connection establishment
// - Operation timing and performance
// - Error details
// - Security events

API Reference

Interfaces

The provider implements two main interfaces:

ClientInterface - Connection and bucket management - Connect(ctx context.Context) error - Close() error - IsConnected() bool - CreateBucket(ctx context.Context, bucket string, opts ...BucketOptions) error - DeleteBucket(ctx context.Context, bucket string) error - ListBuckets(ctx context.Context) ([]BucketInfo, error) - BucketExists(ctx context.Context, bucket string) (bool, error)

BucketInterface - Object operations - Upload: PutObject, PutObjectStream, PutObjectMultipart, PutObjectAdvanced, PutObjectInfo - Download: GetObject, GetObjectStream, GetObjectRange, GetObjectAdvanced - Management: DeleteObject, ListObjects, ObjectExists, HeadObject, CopyObject, CopyObjectVersion - Presigned URLs: PresignGetObject, PresignPutObject, PresignHeadObject - Object Lock / WORM: SetObjectRetention, GetObjectRetention, SetObjectLegalHold, SetObjectLegalHoldVersion, GetObjectLegalHold, SetObjectLockConfig, GetObjectLockConfig

Data Types

ObjectInfo - Object metadata 

type ObjectInfo struct {
    Key          string
    Size         int64
    LastModified time.Time
    ETag         string
    StorageClass string
    ContentType  string
    // Versioning fields
    VersionID      string
    IsLatest       bool
    IsDeleteMarker bool
}

ObjectVersion - The version a put or copy wrote 

type ObjectVersion struct {
    Bucket    string // destination bucket
    Key       string
    VersionID string // empty when the bucket is not versioned ("" or "null" when versioning is suspended)
    ETag      string // not an MD5 for multipart, SSE-KMS or SSE-C objects
    Size      int64  // -1 for copies
}

CopySource - Source of a copy within the bucket 

type CopySource struct {
    Name      string
    VersionID string // empty = latest
}

ObjectOptions - Upload/copy options 

type ObjectOptions struct {
    ContentType        string
    CacheControl       string
    ContentDisposition string
    ContentEncoding    string
    ContentLanguage    string
    Metadata           map[string]string
    Tags               map[string]string
    StorageClass       string
    // Server-side encryption options
    ServerSideEncryption    string            // AES256 or aws:kms (aws:kms:dsse is not supported)
    SSEKMSKeyId             string            // KMS key ID for SSE-KMS
    SSEKMSEncryptionContext map[string]string // KMS encryption context
    SSECustomerKey          string            // SSE-C key (base64); takes precedence over ServerSideEncryption
    SourceSSECustomerKey    string            // SSE-C key (base64) to decrypt an SSE-C source on CopyObject
    // Object Lock / WORM options (require an object-lock-enabled bucket)
    LockMode        string    // RetentionGovernance or RetentionCompliance
    RetainUntilDate time.Time // retain-until date for LockMode
    LegalHold       string    // LegalHoldEnabled or LegalHoldDisabled
}

DeleteOptions - Object deletion parameters 

type DeleteOptions struct {
    VersionID        string // delete a specific object version
    GovernanceBypass bool   // bypass GOVERNANCE retention (requires s3:BypassGovernanceRetention)
    ForceDelete      bool   // MinIO extension: force delete of a WORM-locked object
}

RetentionOptions - Object retention parameters 

type RetentionOptions struct {
    Mode             string    // RetentionGovernance or RetentionCompliance
    RetainUntilDate  time.Time // date until which the object is locked
    GovernanceBypass bool      // bypass governance mode to shorten/remove retention
    VersionID        string    // optional object version
}

ObjectRetention - Retention state returned by GetObjectRetention 

type ObjectRetention struct {
    Mode            string
    RetainUntilDate time.Time
}

ObjectLockConfig - Bucket Object Lock configuration 

type ObjectLockConfig struct {
    Enabled  bool   // Object Lock enabled on the bucket
    Mode     string // default retention mode, empty if no default set
    Validity uint   // default retention validity
    Unit     string // ValidityDays or ValidityYears
}

The S3 provider offers a robust, secure, and performant solution for S3-compatible storage integration with comprehensive features suitable for both development and production environments.