Skip to content

S3 Provider

The S3 provider enables secure and efficient integration with Amazon S3 and S3-compatible storage services like MinIO, DigitalOcean Spaces, and Backblaze B2. It provides a comprehensive API for bucket and object operations with built-in security features and performance optimizations.

Key Features

  • Multi-Cloud Support: Works with AWS S3, MinIO, DigitalOcean Spaces, Backblaze B2, and other S3-compatible services
  • Security: TLS/SSL encryption, secure credential handling, presigned URLs
  • Performance: Automatic multipart uploads, connection pooling, concurrent operations
  • Advanced Operations: Range downloads, object copying, metadata management
  • Logging: Comprehensive operation logging with timing and performance metrics

Basic Usage

Creating a Client

package main

import (
    "context"
    "log"

    "github.com/oddbit-project/blueprint/provider/s3"
    blueprintLog "github.com/oddbit-project/blueprint/log"
)

func main() {
    // Create configuration
    config := s3.NewConfig()
    config.Endpoint = "localhost:9000"  // MinIO endpoint
    config.Region = "us-east-1"
    config.AccessKeyID = "minioadmin"
    config.UseSSL = false  // Disable for local MinIO

    // Set secret key via environment variable
    config.PasswordEnvVar = "S3_SECRET_KEY"

    // Create logger
    logger := blueprintLog.NewLogger()

    // Create client
    client, err := s3.NewClient(config, logger)
    if err != nil {
        log.Fatal(err)
    }

    // Connect to S3
    if err := client.Connect(context.Background()); err != nil {
        log.Fatal(err)
    }

    defer client.Close()
}

Configuration Options

The S3 provider supports comprehensive configuration through the Config struct:

type Config struct {
    // Connection settings
    Endpoint    string `json:"endpoint"`    // S3 endpoint URL
    Region      string `json:"region"`      // AWS region
    AccessKeyID string `json:"accessKeyId"` // Access key ID

    // Secure credential handling
    secure.DefaultCredentialConfig  // Secret key from environment/file

    // Optional defaults
    DefaultBucket string `json:"defaultBucket"`

    // Behavior settings
    ForcePathStyle bool `json:"forcePathStyle"` // Path-style addressing
    UseAccelerate  bool `json:"useAccelerate"`  // S3 transfer acceleration
    UseSSL         bool `json:"useSSL"`         // SSL/TLS encryption

    // Timeout settings (seconds)
    TimeoutSeconds       int `json:"timeoutSeconds"`       // Default: 300
    UploadTimeoutSeconds int `json:"uploadTimeoutSeconds"` // Default: 1800

    // Multipart upload settings
    MultipartThreshold int64 `json:"multipartThreshold"` // Default: 100MB
    PartSize           int64 `json:"partSize"`           // Default: 10MB
    MaxUploadParts     int   `json:"maxUploadParts"`     // Default: 10000
    Concurrency        int   `json:"concurrency"`        // Default: 5

    // TLS configuration
    tls.ClientConfig

    // Connection pooling settings
    MaxIdleConns        int           `json:"maxIdleConns"`        // Default: 50
    MaxIdleConnsPerHost int           `json:"maxIdleConnsPerHost"` // Default: 10
    MaxConnsPerHost     int           `json:"maxConnsPerHost"`     // Default: 20
    IdleConnTimeout     time.Duration `json:"idleConnTimeout"`     // Default: 60s

    // Retry settings
    MaxRetries int    `json:"maxRetries"` // Default: 3
    RetryMode  string `json:"retryMode"`  // "standard" or "adaptive"
}

Bucket Operations

Creating and Managing Buckets

// Create a bucket
bucket, err := client.Bucket("my-bucket")
if err != nil {
    log.Fatal(err)
}

// Create bucket with options
bucketOpts := s3.BucketOptions{
    Region: "us-west-2",
    ACL:    "private",
}
if err := bucket.Create(ctx, bucketOpts); err != nil {
    log.Fatal(err)
}

// Check if bucket exists
exists, err := bucket.Exists(ctx)
if err != nil {
    log.Fatal(err)
}

// List all buckets
buckets, err := client.ListBuckets(ctx)
if err != nil {
    log.Fatal(err)
}

for _, bucket := range buckets {
    fmt.Printf("Bucket: %s, Created: %s\n", 
        bucket.Name, bucket.CreationDate)
}

// Delete bucket (must be empty)
if err := bucket.Delete(ctx); err != nil {
    log.Fatal(err)
}

Object Operations

Basic Upload and Download

// Upload object
file, err := os.Open("local-file.txt")
if err != nil {
    log.Fatal(err)
}
defer file.Close()

stat, _ := file.Stat()
err = bucket.PutObject(ctx, "remote-file.txt", file, stat.Size())
if err != nil {
    log.Fatal(err)
}

// Download object
reader, err := bucket.GetObject(ctx, "remote-file.txt")
if err != nil {
    log.Fatal(err)
}
defer reader.Close()

// Write to local file
outFile, err := os.Create("downloaded-file.txt")
if err != nil {
    log.Fatal(err)
}
defer outFile.Close()

_, err = io.Copy(outFile, reader)
if err != nil {
    log.Fatal(err)
}

Upload with Options

// Upload with metadata and options
options := s3.ObjectOptions{
    ContentType:        "text/plain",
    CacheControl:       "max-age=3600",
    ContentDisposition: "attachment; filename=data.txt",
    Metadata: map[string]string{
        "author":      "user123",
        "description": "Important data file",
    },
    Tags: map[string]string{
        "project":     "analytics",
        "environment": "production",
    },
    StorageClass: "STANDARD_IA",
}

err = bucket.PutObject(ctx, "data.txt", file, size, options)

Streaming Operations

// Stream upload (unknown size)
err = bucket.PutObjectStream(ctx, "stream-file.txt", reader)

// Stream download to writer
var buf bytes.Buffer
err = bucket.GetObjectStream(ctx, "large-file.bin", &buf)

Advanced Upload Options

// Advanced upload with detailed control
uploadOpts := s3.UploadOptions{
    ObjectOptions: s3.ObjectOptions{
        ContentType: "application/octet-stream",
        Metadata: map[string]string{
            "upload-method": "advanced",
        },
    },
    LeavePartsOnError: false,  // Clean up on failure
    MaxUploadParts:    5000,   // Custom part limit
    Concurrency:       10,     // Parallel uploads
}

err = bucket.PutObjectAdvanced(ctx, "large-file.bin", reader, size, uploadOpts)

Advanced Download Operations

Range Downloads

// Download specific byte range
start := int64(1024)
end := int64(4095)
reader, err := bucket.GetObjectRange(ctx, "large-file.bin", start, end)
if err != nil {
    log.Fatal(err)
}
defer reader.Close()

// Download range to writer
var buf bytes.Buffer
err = bucket.GetObjectStreamRange(ctx, "file.bin", &buf, start, end)

Advanced Download Options

// Advanced download with options
downloadOpts := s3.DownloadOptions{
    StartByte:   &start,
    EndByte:     &end,
    Concurrency: 5,
    PartSize:    int64(10 * 1024 * 1024), // 10MB parts
}

var output bytes.Buffer
err = bucket.GetObjectAdvanced(ctx, "file.bin", &output, downloadOpts)

Object Listing and Metadata

List Objects

// List all objects
objects, err := bucket.ListObjects(ctx)
if err != nil {
    log.Fatal(err)
}

// List with options
listOpts := s3.ListOptions{
    Prefix:     "documents/",
    MaxKeys:    100,
    StartAfter: "documents/file-050.txt",
}

filteredObjects, err := bucket.ListObjects(ctx, listOpts)
for _, obj := range filteredObjects {
    fmt.Printf("Object: %s, Size: %d, Modified: %s\n",
        obj.Key, obj.Size, obj.LastModified)
}

Object Metadata and Existence

// Check if object exists
exists, err := bucket.ObjectExists(ctx, "file.txt")

// Get object metadata
info, err := bucket.HeadObject(ctx, "file.txt")
if err != nil {
    log.Fatal(err)
}

fmt.Printf("Size: %d, ETag: %s, ContentType: %s\n",
    info.Size, info.ETag, info.ContentType)

Object Operations

// Copy object
err = bucket.CopyObject(ctx, "source.txt", "destination-bucket", "dest.txt")

// Delete object
err = bucket.DeleteObject(ctx, "old-file.txt")

Presigned URLs

Presigned URLs enable secure temporary access to S3 objects without exposing credentials:

// Generate presigned URL for download (valid for 1 hour)
downloadURL, err := bucket.PresignGetObject(ctx, "file.txt", time.Hour)
if err != nil {
    log.Fatal(err)
}
fmt.Printf("Download URL: %s\n", downloadURL)

// Generate presigned URL for upload (valid for 30 minutes)
uploadURL, err := bucket.PresignPutObject(ctx, "new-file.txt", 30*time.Minute)
if err != nil {
    log.Fatal(err)
}

// Client can now upload directly using the presigned URL
// curl -X PUT --upload-file local.txt "$uploadURL"

// Generate presigned URL for metadata access
headURL, err := bucket.PresignHeadObject(ctx, "file.txt", time.Hour)

Error Handling

The S3 provider defines specific error constants for common scenarios:

const (
    ErrNilConfig           = "Config is nil"
    ErrMissingEndpoint     = "missing endpoint"
    ErrMissingRegion       = "missing region"
    ErrBucketNotFound      = "bucket not found"
    ErrBucketAlreadyExists = "bucket already exists"
    ErrObjectNotFound      = "object not found"
    ErrClientNotConnected  = "client not connected"
)

Handle errors appropriately:

if err := bucket.Create(ctx); err != nil {
    if err == s3.ErrBucketAlreadyExists {
        log.Println("Bucket already exists, continuing...")
    } else {
        log.Fatal("Failed to create bucket:", err)
    }
}

// Check for object existence
exists, err := bucket.ObjectExists(ctx, "file.txt")
if err != nil {
    log.Fatal("Error checking object:", err)
}
if !exists {
    log.Println("Object does not exist")
}

Performance Optimization

Multipart Upload Configuration

The provider automatically uses multipart uploads for large files:

config := s3.NewConfig()
config.MultipartThreshold = 50 * 1024 * 1024  // 50MB threshold
config.PartSize = 5 * 1024 * 1024             // 5MB parts
config.Concurrency = 10                       // 10 concurrent uploads
config.MaxUploadParts = 5000                  // Maximum parts

Timeout Configuration

Configure timeouts based on your use case:

config.TimeoutSeconds = 300        // 5 minutes for regular operations
config.UploadTimeoutSeconds = 3600 // 1 hour for large file uploads

Connection Pooling

Optimize HTTP connection reuse for better performance:

// Default connection pooling (suitable for most use cases)
config.MaxIdleConns = 50        // Total idle connections across all hosts
config.MaxIdleConnsPerHost = 10 // Idle connections per host
config.MaxConnsPerHost = 20     // Maximum connections per host
config.IdleConnTimeout = 60 * time.Second // Keep idle connections for 60s

// High-performance configuration for busy workloads
config.MaxIdleConns = 200
config.MaxIdleConnsPerHost = 50
config.MaxConnsPerHost = 100
config.IdleConnTimeout = 300 * time.Second

// Conservative configuration for resource-constrained environments
config.MaxIdleConns = 10
config.MaxIdleConnsPerHost = 2
config.MaxConnsPerHost = 5
config.IdleConnTimeout = 30 * time.Second

Connection Settings

config.MaxRetries = 5               // Increase for unreliable networks
config.RetryMode = "adaptive"       // Use adaptive retry logic

Connection Pooling Best Practices

Performance Tuning Guidelines

Connection pooling significantly improves performance by reusing HTTP connections. Choose settings based on your workload:

For High-Throughput Applications: 

config := s3.NewConfig()
// Allow more connections for concurrent operations
config.MaxIdleConns = 200
config.MaxIdleConnsPerHost = 50
config.MaxConnsPerHost = 100
config.IdleConnTimeout = 5 * time.Minute // Keep connections longer

For Batch Processing: 

config := s3.NewConfig()
// Moderate pooling for sustained workloads
config.MaxIdleConns = 100
config.MaxIdleConnsPerHost = 20
config.MaxConnsPerHost = 50
config.IdleConnTimeout = 2 * time.Minute

For Resource-Constrained Environments: 

config := s3.NewConfig()
// Minimal connection usage
config.MaxIdleConns = 10
config.MaxIdleConnsPerHost = 2
config.MaxConnsPerHost = 5
config.IdleConnTimeout = 30 * time.Second

Monitoring Connection Pool Efficiency

// Enable verbose logging to monitor connection reuse
logger := blueprintLog.NewLogger()
logger.SetLevel(blueprintLog.DebugLevel)

// The provider logs connection establishment and reuse patterns
client, err := s3.NewClient(config, logger)

Connection Pool Sizing Rules

  1. MaxIdleConns: Set to 2-4x your expected concurrent operations
  2. MaxIdleConnsPerHost: Typically 10-25% of MaxIdleConns
  3. MaxConnsPerHost: Should accommodate peak concurrent requests per endpoint
  4. IdleConnTimeout: Balance between connection reuse and resource cleanup
  5. Short workloads: 30-60 seconds
  6. Long-running services: 2-5 minutes
  7. Always-on applications: 5-10 minutes

Workload-Specific Examples

Web Application with S3 Integration: 

// Handles user uploads/downloads with moderate concurrency
config.MaxIdleConns = 50
config.MaxIdleConnsPerHost = 15
config.MaxConnsPerHost = 30
config.IdleConnTimeout = 2 * time.Minute

Data Pipeline Processing: 

// High-volume batch operations
config.MaxIdleConns = 150
config.MaxIdleConnsPerHost = 30
config.MaxConnsPerHost = 60
config.IdleConnTimeout = 5 * time.Minute

Microservice with Occasional S3 Access: 

// Minimal overhead for infrequent operations
config.MaxIdleConns = 20
config.MaxIdleConnsPerHost = 5
config.MaxConnsPerHost = 10
config.IdleConnTimeout = 60 * time.Second

Security Features

Secure Credential Handling

// Use environment variables for secrets
config.PasswordEnvVar = "S3_SECRET_KEY"

// Or use encrypted files
config.PasswordFile = "/secure/path/secret.key"

TLS Configuration

// Enable SSL/TLS for production
config.UseSSL = true

// Custom TLS configuration
config.ClientConfig = tls.ClientConfig{
    CertFile:           "/path/to/client.crt",
    KeyFile:            "/path/to/client.key",
    CAFile:             "/path/to/ca.crt",
    InsecureSkipVerify: false,
}

Server-Side Encryption

options := s3.ObjectOptions{
    ServerSideEncryption: s3.SSEAlgorithmAES256,
    // Or use KMS
    ServerSideEncryption: s3.SSEAlgorithmKMS,
    SSEKMSKeyId:         "arn:aws:kms:region:account:key/key-id",
}

Integration Examples

AWS S3

config := s3.NewConfig()
config.Region = "us-west-2"
config.UseSSL = true
config.AccessKeyID = "AKIA..."
config.PasswordEnvVar = "AWS_SECRET_ACCESS_KEY"

MinIO

config := s3.NewConfig()
config.Endpoint = "localhost:9000"
config.Region = "us-east-1"
config.UseSSL = false
config.ForcePathStyle = true
config.AccessKeyID = "minioadmin"
config.PasswordEnvVar = "MINIO_SECRET_KEY"

DigitalOcean Spaces

config := s3.NewConfig()
config.Endpoint = "nyc3.digitaloceanspaces.com"
config.Region = "us-east-1"
config.UseSSL = true
config.AccessKeyID = "your-spaces-key"
config.PasswordEnvVar = "DO_SPACES_SECRET"

Testing

The provider includes comprehensive test coverage:

# Run all tests
go test -v ./provider/s3/...

# Run integration tests (requires running S3 service)
go test -v -tags=integration ./provider/s3/...

# Run benchmarks
go test -v -bench=. ./provider/s3/...

CLI Sample

A complete CLI sample is available in samples/s3-client/ demonstrating all features including connection pooling:

cd samples/s3-client
go build -o s3-cli main.go

# Basic operations
./s3-cli create-bucket my-bucket
./s3-cli upload my-bucket file.txt
./s3-cli download my-bucket file.txt downloaded.txt
./s3-cli presign-get my-bucket file.txt 60

# Connection pooling examples
./s3-cli --max-idle-conns 100 --max-conns-per-host 50 upload my-bucket large-file.zip
./s3-cli --verbose upload my-bucket file.txt  # Shows connection pool configuration

Troubleshooting

Common Issues

Connection Refused - Verify endpoint URL and port - Check if S3 service is running - Validate network connectivity

Access Denied - Verify access key and secret key - Check IAM permissions - Ensure bucket policies allow operation

SSL/TLS Errors - For local development, use UseSSL: false - Verify SSL certificates for production - Check TLS configuration

Multipart Upload Failures - Adjust part size and concurrency - Increase upload timeout - Check available memory and disk space

Poor Performance - Tune connection pooling settings for your workload - Increase MaxIdleConns and MaxConnsPerHost for high concurrency - Monitor connection reuse with verbose logging - Consider increasing IdleConnTimeout for sustained workloads

Logging

Enable detailed logging for debugging:

logger := blueprintLog.NewLogger()
logger.SetLevel(blueprintLog.DebugLevel)

// The provider logs:
// - Connection establishment
// - Operation timing and performance
// - Error details
// - Security events

API Reference

Interfaces

The provider implements two main interfaces:

ClientInterface - Connection and bucket management - Connect(ctx context.Context) error - Close() error - IsConnected() bool - CreateBucket(ctx context.Context, bucket string, opts ...BucketOptions) error - DeleteBucket(ctx context.Context, bucket string) error - ListBuckets(ctx context.Context) ([]BucketInfo, error) - BucketExists(ctx context.Context, bucket string) (bool, error)

BucketInterface - Object operations - Upload: PutObject, PutObjectStream, PutObjectMultipart, PutObjectAdvanced - Download: GetObject, GetObjectStream, GetObjectRange, GetObjectAdvanced - Management: DeleteObject, ListObjects, ObjectExists, HeadObject, CopyObject - Presigned URLs: PresignGetObject, PresignPutObject, PresignHeadObject

Data Types

ObjectInfo - Object metadata 

type ObjectInfo struct {
    Key          string
    Size         int64
    LastModified time.Time
    ETag         string
    StorageClass string
    ContentType  string
}

ObjectOptions - Upload/copy options 

type ObjectOptions struct {
    ContentType        string
    CacheControl       string
    ContentDisposition string
    ContentEncoding    string
    ContentLanguage    string
    Metadata           map[string]string
    Tags               map[string]string
    StorageClass       string
    // Server-side encryption options
    ServerSideEncryption    string
    SSEKMSKeyId             string
    SSEKMSEncryptionContext map[string]string
    // ... additional encryption options
}

The S3 provider offers a robust, secure, and performant solution for S3-compatible storage integration with comprehensive features suitable for both development and production environments.