refactor(storage): migrate S3 clients to tigrisdata/storage-go

autocert/s3cache/s3cache.go

@@ -8,8 +8,8 @@ import (
 	"path"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
-	awsConfig "github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
+	storage "github.com/tigrisdata/storage-go"
 	"golang.org/x/crypto/acme/autocert"
 )
 
@@ -19,15 +19,11 @@ type Options struct {
 }
 
 func New(ctx context.Context, opts Options) (autocert.Cache, error) {
-	cfg, err := awsConfig.LoadDefaultConfig(ctx)
+	s3c, err := storage.New(ctx, storage.WithGlobalEndpoint(), storage.WithPathStyle(true))
 	if err != nil {
 		return nil, err
 	}
 
-	s3c := s3.NewFromConfig(cfg, func(o *s3.Options) {
-		o.UsePathStyle = true
-	})
-
 	return &impl{
 		cli:    s3c,
 		bucket: opts.Bucket,
@@ -36,7 +32,7 @@ func New(ctx context.Context, opts Options) (autocert.Cache, error) {
 }
 
 type impl struct {
-	cli            *s3.Client
+	cli            *storage.Client
 	bucket, prefix string
 }
 

cmd/future-sight/main.go

@@ -16,16 +16,15 @@ import (
 
 	"buf.build/go/protovalidate"
 	"github.com/aws/aws-sdk-go-v2/aws"
-	"github.com/aws/aws-sdk-go-v2/credentials"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/nats-io/nats.go"
 	"github.com/redis/go-redis/v9"
+	storage "github.com/tigrisdata/storage-go"
 	"golang.org/x/sync/errgroup"
 	"google.golang.org/protobuf/proto"
 	pb "within.website/x/gen/within/website/x/futuresight/v1"
 	"within.website/x/internal"
 	"within.website/x/internal/xesite"
-	"within.website/x/web/useragent"
 )
 
 var (
@@ -49,18 +48,15 @@ func main() {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	creds := credentials.NewStaticCredentialsProvider(*awsAccessKeyID, *awsSecretKey, "")
-
-	s3c := s3.New(s3.Options{
-		AppID:            useragent.GenUserAgent("future-sight-push", "https://xeiaso.net"),
-		BaseEndpoint:     awsEndpointS3,
-		ClientLogMode:    aws.LogRetries | aws.LogRequest | aws.LogResponse,
-		Credentials:      creds,
-		EndpointResolver: s3.EndpointResolverFromURL(*awsEndpointS3),
-		//Logger:           logging.NewStandardLogger(os.Stderr),
-		UsePathStyle: *usePathStyle,
-		Region:       *awsRegion,
-	})
+	s3c, err := storage.New(ctx,
+		storage.WithEndpoint(*awsEndpointS3),
+		storage.WithRegion(*awsRegion),
+		storage.WithPathStyle(*usePathStyle),
+		storage.WithAccessKeypair(*awsAccessKeyID, *awsSecretKey),
+	)
+	if err != nil {
+		log.Fatal(err)
+	}
 
 	slog.Debug("details",
 		"awsAccessKeyID", *awsAccessKeyID,
@@ -148,7 +144,7 @@ func main() {
 }
 
 type Server struct {
-	s3c *s3.Client
+	s3c *storage.Client
 	vk  *redis.Client
 	nc  *nats.Conn
 	zs  *xesite.ZipServer

cmd/relayd/telemetry.go

@@ -10,25 +10,24 @@ import (
 	"time"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
-	awsConfig "github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/google/uuid"
+	storage "github.com/tigrisdata/storage-go"
 	"within.website/x/bundler"
 	relayd "within.website/x/gen/within/website/x/relayd/v1"
 )
 
 var (
 	telemetryEnable          = flag.Bool("telemetry-enable", false, "if true, enable request telemetry bundling to object storage")
 	telemetryBucket          = flag.String("telemetry-bucket", "relayd-logs", "object storage bucket to dump logs to")
-	telemetryPathStyle       = flag.Bool("telemetry-path-style", false, "if true, use s3 path style")
 	telemetryHost            = flag.String("telemetry-host", "", "hostname to disambiguate telemetry")
 	telemetryBundleCount     = flag.Int("telemetry-bundle-count", 512, "maximum number of items per telemetry bundle")
 	telemetryContextDeadline = flag.Duration("telemetry-context-deadline", time.Minute, "maximum time for the telemetry context deadline")
 )
 
 type TelemetrySink struct {
 	sink *bundler.Bundler[*relayd.RequestLog]
-	s3c  *s3.Client
+	s3c  *storage.Client
 	host string
 }
 
@@ -39,15 +38,11 @@ func NewTelemetrySink(ctx context.Context) (*TelemetrySink, error) {
 
 	slog.Info("telemetry enabled")
 
-	cfg, err := awsConfig.LoadDefaultConfig(ctx)
+	s3c, err := storage.New(ctx, storage.WithGlobalEndpoint(), storage.WithPathStyle(true))
 	if err != nil {
 		return nil, err
 	}
 
-	s3c := s3.NewFromConfig(cfg, func(o *s3.Options) {
-		o.UsePathStyle = true
-	})
-
 	result := &TelemetrySink{
 		s3c:  s3c,
 		host: *telemetryHost,

cmd/stickers/main.go

@@ -15,10 +15,7 @@ import (
 	"time"
 
 	"github.com/a-h/templ"
-	"github.com/aws/aws-sdk-go-v2/aws"
-	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
-	awsConfig "github.com/aws/aws-sdk-go-v2/config"
-	"github.com/aws/aws-sdk-go-v2/service/s3"
+	simplestorage "github.com/tigrisdata/storage-go/simplestorage"
 	"within.website/x/htmx"
 	"within.website/x/internal"
 	"within.website/x/xess"
@@ -44,18 +41,14 @@ func main() {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	cfg, err := awsConfig.LoadDefaultConfig(ctx)
+	sc, err := simplestorage.New(ctx,
+		simplestorage.WithBucket(*bucketName),
+		simplestorage.WithFlyEndpoint(),
+	)
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	s3c := s3.NewFromConfig(cfg, func(o *s3.Options) {
-		o.UsePathStyle = false
-	})
-
-	presigner := Presigner{s3.NewPresignClient(s3c)}
-	_ = presigner
-
 	mux := http.NewServeMux()
 	htmx.Mount(mux)
 	xess.Mount(mux)
@@ -91,10 +84,7 @@ func main() {
 		key := fmt.Sprintf("%s/%s/%s.%s", *folderName, name, mood, format)
 
 		if !inCache(name, mood) {
-			_, err := s3c.HeadObject(r.Context(), &s3.HeadObjectInput{
-				Bucket: aws.String(*bucketName),
-				Key:    aws.String(key),
-			})
+			_, err := sc.Head(r.Context(), key)
 			if err != nil {
 				slog.Error("can't head key", "format", format, "bucket", *bucketName, "key", key, "err", err)
 
@@ -123,7 +113,7 @@ func main() {
 			setCache(name, mood)
 		}
 
-		req, err := presigner.GetObject(r.Context(), key, 3600)
+		url, err := sc.PresignURL(r.Context(), http.MethodGet, key, time.Hour)
 		if err != nil {
 			slog.Error("can't presign get for key", "format", format, "bucket", *bucketName, "key", key, "err", err)
 
@@ -152,7 +142,7 @@ func main() {
 		w.Header().Add("Cache-Control", "max-age=3599")
 		w.Header().Add("Expires", time.Now().Add(time.Hour-time.Second).Format(http.TimeFormat))
 
-		brandedURL := strings.ReplaceAll(req.URL, "xedn.fly.storage.tigris.dev", "files.xeiaso.net")
+		brandedURL := strings.ReplaceAll(url, "xedn.fly.storage.tigris.dev", "files.xeiaso.net")
 
 		http.Redirect(w, r, brandedURL, http.StatusTemporaryRedirect)
 	})
@@ -161,34 +151,6 @@ func main() {
 	log.Fatal(http.ListenAndServe(*bind, mux))
 }
 
-// Presigner encapsulates the Amazon Simple Storage Service (Amazon S3) presign actions
-// used in the examples.
-// It contains PresignClient, a client that is used to presign requests to Amazon S3.
-// Presigned requests contain temporary credentials and can be made from any HTTP client.
-type Presigner struct {
-	PresignClient *s3.PresignClient
-}
-
-// GetObject makes a presigned request that can be used to get an object from a bucket.
-// The presigned request is valid for the specified number of seconds.
-func (presigner Presigner) GetObject(
-	ctx context.Context,
-	objectKey string,
-	lifetimeSecs int64,
-) (*v4.PresignedHTTPRequest, error) {
-	request, err := presigner.PresignClient.PresignGetObject(ctx, &s3.GetObjectInput{
-		Bucket: aws.String(*bucketName),
-		Key:    aws.String(objectKey),
-	}, func(opts *s3.PresignOptions) {
-		opts.Expires = time.Duration(lifetimeSecs * int64(time.Second))
-	})
-	if err != nil {
-		log.Printf("Couldn't get a presigned request to get %v:%v. Here's why: %v\n",
-			bucketName, objectKey, err)
-	}
-	return request, err
-}
-
 func inCache(character, mood string) bool {
 	cacheLock.RLock()
 	defer cacheLock.RUnlock()

cmd/uploud/main.go

@@ -17,13 +17,13 @@ import (
 	"path/filepath"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
-	awsConfig "github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/disintegration/imaging"
 	"github.com/gen2brain/avif"
 	_ "github.com/gen2brain/heic"
 	"github.com/gen2brain/jpegxl"
 	"github.com/gen2brain/webp"
+	storage "github.com/tigrisdata/storage-go"
 	"golang.org/x/sync/errgroup"
 	"within.website/x/internal"
 )
@@ -93,15 +93,11 @@ func main() {
 		log.Fatal(err)
 	}
 
-	cfg, err := awsConfig.LoadDefaultConfig(ctx)
+	s3c, err := storage.New(ctx, storage.WithGlobalEndpoint(), storage.WithPathStyle(true))
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	s3c := s3.NewFromConfig(cfg, func(o *s3.Options) {
-		o.UsePathStyle = true
-	})
-
 	for _, finfo := range files {
 		log.Printf("uploading %s", finfo.Name())
 		fin, err := os.Open(filepath.Join(td, finfo.Name()))

cmd/xedn/uplodr/main.go

@@ -17,17 +17,16 @@ import (
 	"time"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
-	"github.com/aws/aws-sdk-go-v2/credentials"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/disintegration/imaging"
 	"github.com/gen2brain/avif"
 	_ "github.com/gen2brain/heic"
 	"github.com/gen2brain/jpegxl"
 	"github.com/gen2brain/webp"
+	storage "github.com/tigrisdata/storage-go"
 	"google.golang.org/grpc"
 	pb "within.website/x/gen/within/website/x/xedn/uplodr/v1"
 	"within.website/x/internal"
-	"within.website/x/tigris"
 )
 
 var (
@@ -81,19 +80,22 @@ func main() {
 }
 
 type Server struct {
-	tc  *s3.Client
-	b2c *s3.Client
+	tc  *storage.Client
+	b2c *storage.Client
 
 	pb.UnimplementedImageServer
 }
 
 func New(ctx context.Context) (*Server, error) {
-	tc, err := tigris.Client(ctx)
+	tc, err := storage.New(ctx, storage.WithFlyEndpoint())
 	if err != nil {
 		return nil, fmt.Errorf("failed to create Tigris client: %w", err)
 	}
 
-	b2c := mkB2Client()
+	b2c, err := mkB2Client(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create Backblaze B2 client: %w", err)
+	}
 
 	return &Server{
 		tc:  tc,
@@ -327,15 +329,11 @@ var mimeTypes = map[string]string{
 	".css":  "text/css",
 }
 
-func mkB2Client() *s3.Client {
-	s3Config := aws.Config{
-		Credentials:  credentials.NewStaticCredentialsProvider(*b2KeyID, *b2KeySecret, ""),
-		BaseEndpoint: aws.String("https://s3.us-west-001.backblazeb2.com"),
-		Region:       "us-west-001",
-	}
-	s3Client := s3.NewFromConfig(s3Config, (func(o *s3.Options) {
-		o.UsePathStyle = true
-		o.Region = "us-west-001"
-	}))
-	return s3Client
+func mkB2Client(ctx context.Context) (*storage.Client, error) {
+	return storage.New(ctx,
+		storage.WithEndpoint("https://s3.us-west-001.backblazeb2.com"),
+		storage.WithRegion("us-west-001"),
+		storage.WithPathStyle(true),
+		storage.WithAccessKeypair(*b2KeyID, *b2KeySecret),
+	)
 }