diff --git a/drivers/all.go b/drivers/all.go index fb68d0395..ea9865166 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -52,6 +52,7 @@ import ( _ "github.com/OpenListTeam/OpenList/v4/drivers/misskey" _ "github.com/OpenListTeam/OpenList/v4/drivers/mopan" _ "github.com/OpenListTeam/OpenList/v4/drivers/netease_music" + _ "github.com/OpenListTeam/OpenList/v4/drivers/obs" _ "github.com/OpenListTeam/OpenList/v4/drivers/onedrive" _ "github.com/OpenListTeam/OpenList/v4/drivers/onedrive_app" _ "github.com/OpenListTeam/OpenList/v4/drivers/onedrive_sharelink" diff --git a/drivers/obs/driver.go b/drivers/obs/driver.go new file mode 100644 index 000000000..fd1b119c6 --- /dev/null +++ b/drivers/obs/driver.go @@ -0,0 +1,255 @@ +package obs + +import ( + "bytes" + "context" + "fmt" + "net/url" + stdpath "path" + "strings" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/huaweicloud/huaweicloud-sdk-go-obs/obs" + log "github.com/sirupsen/logrus" +) + +type OBS struct { + model.Storage + Addition + client *obs.ObsClient + linkClient *obs.ObsClient + directUploadClient *obs.ObsClient + + config driver.Config +} + +func (d *OBS) Config() driver.Config { + return d.config +} + +func (d *OBS) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *OBS) Init(ctx context.Context) error { + if d.Region == "" { + d.Region = "cn-north-4" + } + if d.SignURLExpire == 0 { + d.SignURLExpire = 4 + } + + // 创建主客户端 + client, err := d.createClient() + if err != nil { + return fmt.Errorf("failed to create main client: %w", err) + } + d.client = client + + // 创建linkClient(用于生成下载链接) + linkClient, err := d.createLinkClient() + if err != nil { + return fmt.Errorf("failed to create link client: %w", err) + } + d.linkClient = linkClient + + // 创建directUploadClient(用于直接上传) + if d.EnableDirectUpload { + directUploadClient, err := d.createDirectUploadClient() + if err != nil { + return fmt.Errorf("failed to create direct upload client: %w", err) + } + d.directUploadClient = directUploadClient + } + + return nil +} + +func (d *OBS) Drop(ctx context.Context) error { + // OBS SDK的客户端不需要显式关闭 + return nil +} + +func (d *OBS) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + if d.ListObjectVersion == "v2" { + return d.listV2(dir.GetPath(), args) + } + return d.listV1(dir.GetPath(), args) +} + +func (d *OBS) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + path := getKey(file.GetPath(), false) + fileName := stdpath.Base(path) + + input := &obs.CreateSignedUrlInput{ + Bucket: d.Bucket, + Key: path, + Method: obs.HttpMethodGet, + } + + if d.CustomHost == "" { + disposition := fmt.Sprintf(`attachment; filename*=UTF-8''%s`, url.PathEscape(fileName)) + if d.AddFilenameToDisposition { + disposition = utils.GenerateContentDisposition(fileName) + } + input.QueryParams = map[string]string{ + "response-content-disposition": disposition, + } + } + + var link model.Link + var err error + + if d.CustomHost != "" { + if d.EnableCustomHostPresign { + output, err := d.linkClient.CreateSignedUrl(input) + if err != nil { + return nil, fmt.Errorf("failed to create signed URL: %w", err) + } + link.URL = output.SignedUrl + } else { + // 构建URL + scheme := "https" + host := d.CustomHost + if d.ForcePathStyle { + link.URL = fmt.Sprintf("%s://%s/%s/%s", scheme, host, d.Bucket, path) + } else { + link.URL = fmt.Sprintf("%s://%s.%s/%s", scheme, d.Bucket, host, path) + } + } + + if d.RemoveBucket { + parsedURL, parseErr := url.Parse(link.URL) + if parseErr != nil { + log.Errorf("Failed to parse URL for bucket removal: %v, URL: %s", parseErr, link.URL) + return nil, fmt.Errorf("failed to parse URL for bucket removal: %w", parseErr) + } + + path := parsedURL.Path + bucketPrefix := "/" + d.Bucket + if strings.HasPrefix(path, bucketPrefix) { + path = strings.TrimPrefix(path, bucketPrefix) + if path == "" { + path = "/" + } + parsedURL.Path = path + link.URL = parsedURL.String() + log.Debugf("Removed bucket '%s' from URL path: %s -> %s", d.Bucket, bucketPrefix, path) + } else { + log.Warnf("URL path does not contain expected bucket prefix '%s': %s", bucketPrefix, path) + } + } + } else { + if common.ShouldProxy(d, fileName) { + output, err := d.linkClient.CreateSignedUrl(input) + if err != nil { + return nil, fmt.Errorf("failed to create signed URL: %w", err) + } + link.URL = output.SignedUrl + } else { + output, err := d.linkClient.CreateSignedUrl(input) + if err != nil { + return nil, fmt.Errorf("failed to create signed URL: %w", err) + } + link.URL = output.SignedUrl + } + } + + if err != nil { + return nil, err + } + return &link, nil +} + +func (d *OBS) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + return d.Put(ctx, &model.Object{ + Path: stdpath.Join(parentDir.GetPath(), dirName), + }, &stream.FileStream{ + Obj: &model.Object{ + Name: getPlaceholderName(d.Placeholder), + Modified: time.Now(), + }, + Reader: bytes.NewReader([]byte{}), + Mimetype: "application/octet-stream", + }, func(float64) {}) +} + +func (d *OBS) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + err := d.Copy(ctx, srcObj, dstDir) + if err != nil { + return err + } + return d.Remove(ctx, srcObj) +} + +func (d *OBS) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + err := d.copy(ctx, srcObj.GetPath(), stdpath.Join(stdpath.Dir(srcObj.GetPath()), newName), srcObj.IsDir()) + if err != nil { + return err + } + return d.Remove(ctx, srcObj) +} + +func (d *OBS) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + return d.copy(ctx, srcObj.GetPath(), stdpath.Join(dstDir.GetPath(), srcObj.GetName()), srcObj.IsDir()) +} + +func (d *OBS) Remove(ctx context.Context, obj model.Obj) error { + if obj.IsDir() { + return d.removeDir(ctx, obj.GetPath()) + } + return d.removeFile(obj.GetPath()) +} + +func (d *OBS) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { + key := getKey(stdpath.Join(dstDir.GetPath(), s.GetName()), false) + contentType := s.GetMimetype() + log.Debugln("key:", key) + + // 使用PutObject直接上传 + input := &obs.PutObjectInput{} + input.Bucket = d.Bucket + input.Key = key + input.Body = driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{Reader: s, UpdateProgress: up}) + input.ContentType = contentType + _, err := d.client.PutObject(input) + return err +} + +func (d *OBS) GetDirectUploadTools() []string { + if !d.EnableDirectUpload { + return nil + } + return []string{"HttpDirect"} +} + +func (d *OBS) GetDirectUploadInfo(ctx context.Context, _ string, dstDir model.Obj, fileName string, _ int64) (any, error) { + if !d.EnableDirectUpload { + return nil, errs.NotImplement + } + path := getKey(stdpath.Join(dstDir.GetPath(), fileName), false) + + input := &obs.CreateSignedUrlInput{ + Bucket: d.Bucket, + Key: path, + Method: obs.HttpMethodPut, + } + + output, err := d.directUploadClient.CreateSignedUrl(input) + if err != nil { + return nil, fmt.Errorf("failed to create signed URL for direct upload: %w", err) + } + + return &model.HttpDirectUploadInfo{ + UploadURL: output.SignedUrl, + Method: "PUT", + }, nil +} + +var _ driver.Driver = (*OBS)(nil) diff --git a/drivers/obs/meta.go b/drivers/obs/meta.go new file mode 100644 index 000000000..3c3bb83c6 --- /dev/null +++ b/drivers/obs/meta.go @@ -0,0 +1,38 @@ +package obs + +import ( + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/op" +) + +type Addition struct { + driver.RootPath + Bucket string `json:"bucket" required:"true"` + Endpoint string `json:"endpoint" required:"true"` + Region string `json:"region"` + AccessKeyID string `json:"access_key_id" required:"true"` + SecretAccessKey string `json:"secret_access_key" required:"true"` + CustomHost string `json:"custom_host"` + EnableCustomHostPresign bool `json:"enable_custom_host_presign"` + SignURLExpire int `json:"sign_url_expire" type:"number" default:"4"` + Placeholder string `json:"placeholder"` + ForcePathStyle bool `json:"force_path_style"` + ListObjectVersion string `json:"list_object_version" type:"select" options:"v1,v2" default:"v1"` + RemoveBucket bool `json:"remove_bucket" help:"Remove bucket name from path when using custom host."` + AddFilenameToDisposition bool `json:"add_filename_to_disposition" help:"Add filename to Content-Disposition header."` + EnableDirectUpload bool `json:"enable_direct_upload" default:"false"` + DirectUploadHost string `json:"direct_upload_host" required:"false"` +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &OBS{ + config: driver.Config{ + Name: "OBS", + DefaultRoot: "/", + LocalSort: true, + CheckStatus: true, + }, + } + }) +} diff --git a/drivers/obs/types.go b/drivers/obs/types.go new file mode 100644 index 000000000..34ad4c803 --- /dev/null +++ b/drivers/obs/types.go @@ -0,0 +1,3 @@ +package obs + +// 此文件用于定义辅助类型(如有需要) diff --git a/drivers/obs/util.go b/drivers/obs/util.go new file mode 100644 index 000000000..d92f3c002 --- /dev/null +++ b/drivers/obs/util.go @@ -0,0 +1,255 @@ +package obs + +import ( + "context" + "fmt" + "net/http" + stdpath "path" + "strings" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/huaweicloud/huaweicloud-sdk-go-obs/obs" +) + +// getKey 处理路径格式,去除开头的/ +func getKey(path string, isDir bool) string { + path = strings.TrimPrefix(path, "/") + if isDir && path != "" && !strings.HasSuffix(path, "/") { + path += "/" + } + return path +} + +// getPlaceholderName 获取占位文件名 +func getPlaceholderName(placeholder string) string { + if placeholder == "" { + return ".placeholder" + } + return placeholder +} + +// obsObjectToObj 将OBS对象元数据转换为model.Obj +func obsObjectToObj(object obs.Content) model.Obj { + isDir := strings.HasSuffix(object.Key, "/") + name := stdpath.Base(object.Key) + if isDir && name == "" { + name = stdpath.Base(strings.TrimSuffix(object.Key, "/")) + } + return &model.Object{ + Name: name, + Size: object.Size, + Modified: object.LastModified, + IsFolder: isDir, + Path: "/" + object.Key, + } +} + +// obsPrefixToObj 将OBS前缀转换为目录对象 +func obsPrefixToObj(prefix string) model.Obj { + key := strings.TrimSuffix(prefix, "/") + name := stdpath.Base(key) + return &model.Object{ + Name: name, + Size: 0, + Modified: time.Time{}, + IsFolder: true, + Path: "/" + prefix, + } +} + +// createClient 创建主客户端 +func (d *OBS) createClient() (*obs.ObsClient, error) { + ak := d.AccessKeyID + sk := d.SecretAccessKey + endpoint := d.Endpoint + + // 使用WithPathStyle配置路径风格 + if d.ForcePathStyle { + return obs.New(ak, sk, endpoint, obs.WithPathStyle(true)) + } + return obs.New(ak, sk, endpoint) +} + +// createLinkClient 创建用于生成下载链接的客户端 +func (d *OBS) createLinkClient() (*obs.ObsClient, error) { + ak := d.AccessKeyID + sk := d.SecretAccessKey + endpoint := d.Endpoint + + if d.CustomHost != "" { + endpoint = d.CustomHost + } + + if d.ForcePathStyle { + return obs.New(ak, sk, endpoint, obs.WithPathStyle(true)) + } + return obs.New(ak, sk, endpoint) +} + +// createDirectUploadClient 创建用于直接上传的客户端 +func (d *OBS) createDirectUploadClient() (*obs.ObsClient, error) { + ak := d.AccessKeyID + sk := d.SecretAccessKey + endpoint := d.Endpoint + + if d.DirectUploadHost != "" { + endpoint = d.DirectUploadHost + } + + if d.ForcePathStyle { + return obs.New(ak, sk, endpoint, obs.WithPathStyle(true)) + } + return obs.New(ak, sk, endpoint) +} + +// listV1 使用v1 API列举对象 +func (d *OBS) listV1(path string, args model.ListArgs) ([]model.Obj, error) { + prefix := getKey(path, true) + if prefix != "" && !strings.HasSuffix(prefix, "/") { + prefix += "/" + } + + input := &obs.ListObjectsInput{ + Bucket: d.Bucket, + } + input.Prefix = prefix + input.Delimiter = "/" + + var res []model.Obj + + for { + output, err := d.client.ListObjects(input) + if err != nil { + return nil, fmt.Errorf("failed to list objects: %w", err) + } + + // 处理目录 + for _, prefix := range output.CommonPrefixes { + res = append(res, obsPrefixToObj(prefix)) + } + + // 处理文件 + for _, content := range output.Contents { + // 跳过目录标记对象 + if strings.HasSuffix(content.Key, "/") && content.Size == 0 { + continue + } + res = append(res, obsObjectToObj(content)) + } + + // 检查是否还有更多结果 + if !output.IsTruncated { + break + } + input.Marker = output.NextMarker + } + + return res, nil +} + +// listV2 使用v2 API列举对象(OBS SDK v3不支持ListObjectsV2,使用ListObjects代替) +func (d *OBS) listV2(path string, args model.ListArgs) ([]model.Obj, error) { + // OBS SDK v3不支持ListObjectsV2,回退到v1 + return d.listV1(path, args) +} + +// removeFile 删除单个文件 +func (d *OBS) removeFile(path string) error { + key := getKey(path, false) + input := &obs.DeleteObjectInput{ + Bucket: d.Bucket, + Key: key, + } + _, err := d.client.DeleteObject(input) + return err +} + +// removeDir 递归删除目录 +func (d *OBS) removeDir(ctx context.Context, path string) error { + objs, err := op.List(ctx, d, path, model.ListArgs{}) + if err != nil { + return err + } + for _, obj := range objs { + cSrc := stdpath.Join(path, obj.GetName()) + if obj.IsDir() { + err = d.removeDir(ctx, cSrc) + } else { + err = d.removeFile(cSrc) + } + if err != nil { + return err + } + } + _ = d.removeFile(stdpath.Join(path, getPlaceholderName(d.Placeholder))) + _ = d.removeFile(stdpath.Join(path, d.Placeholder)) + return nil +} + +// copy 复制对象 +func (d *OBS) copy(ctx context.Context, srcPath, dstPath string, isDir bool) error { + if isDir { + return d.copyDir(ctx, srcPath, dstPath) + } + return d.copyFile(ctx, srcPath, dstPath) +} + +// copyFile 复制单个文件 +func (d *OBS) copyFile(ctx context.Context, src string, dst string) error { + srcKey := getKey(src, false) + dstKey := getKey(dst, false) + + copyInput := &obs.CopyObjectInput{} + copyInput.Bucket = d.Bucket + copyInput.Key = dstKey + copyInput.CopySourceBucket = d.Bucket + copyInput.CopySourceKey = srcKey + _, err := d.client.CopyObject(copyInput) + return err +} + +// copyDir 复制目录 +func (d *OBS) copyDir(ctx context.Context, src string, dst string) error { + objs, err := op.List(ctx, d, src, model.ListArgs{S3ShowPlaceholder: true}) + if err != nil { + return err + } + for _, obj := range objs { + cSrc := stdpath.Join(src, obj.GetName()) + cDst := stdpath.Join(dst, obj.GetName()) + if obj.IsDir() { + err = d.copyDir(ctx, cSrc, cDst) + } else { + err = d.copyFile(ctx, cSrc, cDst) + } + if err != nil { + return err + } + } + return nil +} + +// handleError 处理OBS错误 +func handleError(err error) error { + if err == nil { + return nil + } + + // 检查是否为OBS错误 + if obsError, ok := err.(obs.ObsError); ok { + switch obsError.StatusCode { + case http.StatusNotFound: + return fmt.Errorf("object not found: %s", obsError.Message) + case http.StatusForbidden: + return fmt.Errorf("access denied: %s", obsError.Message) + case http.StatusBadRequest: + return fmt.Errorf("bad request: %s", obsError.Message) + default: + return fmt.Errorf("OBS error (status %d): %s", obsError.StatusCode, obsError.Message) + } + } + + return err +} diff --git a/go.mod b/go.mod index 2fc141f2e..439e857c7 100644 --- a/go.mod +++ b/go.mod @@ -113,6 +113,7 @@ require ( github.com/glebarez/go-sqlite v1.21.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect + github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.9+incompatible // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect github.com/jcmturner/gofork v1.7.6 // indirect diff --git a/go.sum b/go.sum index 0f69ce117..0ccb2e6ed 100644 --- a/go.sum +++ b/go.sum @@ -385,6 +385,8 @@ github.com/henrybear327/Proton-API-Bridge v1.0.0 h1:gjKAaWfKu++77WsZTHg6FUyPC5W0 github.com/henrybear327/Proton-API-Bridge v1.0.0/go.mod h1:gunH16hf6U74W2b9CGDaWRadiLICsoJ6KRkSt53zLts= github.com/henrybear327/go-proton-api v1.0.0 h1:zYi/IbjLwFAW7ltCeqXneUGJey0TN//Xo851a/BgLXw= github.com/henrybear327/go-proton-api v1.0.0/go.mod h1:w63MZuzufKcIZ93pwRgiOtxMXYafI8H74D77AxytOBc= +github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.9+incompatible h1:zUhCrGMMpJxZGAB30GbQzluDhQuPENxRQfxss7KlpKU= +github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.9+incompatible/go.mod h1:l7VUhRbTKCzdOacdT4oWCwATKyvZqUOlOqr0Ous3k4s= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/boxo v0.12.0 h1:AXHg/1ONZdRQHQLgG5JHsSC3XoE4DjCAMgK+asZvUcQ=