diff --git a/drivers/crypt/driver.go b/drivers/crypt/driver.go index 9e8b5c5c7..8a785b106 100644 --- a/drivers/crypt/driver.go +++ b/drivers/crypt/driver.go @@ -11,6 +11,7 @@ import ( "strings" "sync" + "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/fs" @@ -19,6 +20,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/sign" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" + "github.com/OpenListTeam/OpenList/v4/pkg/singleflight" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" rcCrypt "github.com/rclone/rclone/backend/crypt" @@ -30,7 +32,8 @@ import ( type Crypt struct { model.Storage Addition - cipher *rcCrypt.Cipher + cipher *rcCrypt.Cipher + thumbGroup singleflight.Group[struct{}] } const obfuscatedPrefix = "___Obfuscated___" @@ -146,7 +149,7 @@ func (d *Crypt) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ Mask: mask &^ model.Temp, // discarding hash as it's encrypted } - if !d.Thumbnail || !strings.HasPrefix(args.ReqPath, "/") { + if !d.Thumbnail || !strings.HasPrefix(args.ReqPath, "/") || objRes.IsFolder || utils.GetFileType(name) != conf.IMAGE { result = append(result, objRes) continue } @@ -170,7 +173,7 @@ func (a Addition) GetRootPath() string { return a.RemotePath } -func (d *Crypt) Get(ctx context.Context, path string) (model.Obj, error) { +func (d *Crypt) getActual(ctx context.Context, path string) (model.Obj, error) { firstTryIsFolder, secondTry := guessPath(path) remoteFullPath := stdpath.Join(d.RemotePath, d.encryptPath(path, firstTryIsFolder)) remoteObj, err := fs.Get(ctx, remoteFullPath, &fs.GetArgs{NoLog: true}) @@ -231,10 +234,39 @@ func (d *Crypt) Get(ctx context.Context, path string) (model.Obj, error) { }, nil } +func (d *Crypt) Get(ctx context.Context, path string) (model.Obj, error) { + obj, err := d.getActual(ctx, path) + if err == nil { + return obj, nil + } + if !errs.IsObjectNotFound(err) || !isThumbPath(path) { + return nil, err + } + sourcePath, ok := thumbSourcePath(path) + if !ok { + return nil, err + } + sourceObj, sourceErr := op.Get(ctx, d, sourcePath) + if sourceErr != nil || sourceObj.IsDir() || utils.GetFileType(sourceObj.GetName()) != conf.IMAGE { + return nil, err + } + return d.newThumbObject(path, sourceObj), nil +} + // https://github.com/rclone/rclone/blob/v1.67.0/backend/crypt/cipher.go#L37 const fileHeaderSize = 32 func (d *Crypt) Link(ctx context.Context, file model.Obj, _ model.LinkArgs) (*model.Link, error) { + if thumb, ok := file.(*thumbObject); ok { + if err := d.ensureThumb(ctx, thumb); err != nil { + return nil, err + } + generatedObj, err := d.getActual(ctx, thumb.thumbPath) + if err != nil { + return nil, err + } + return d.Link(ctx, generatedObj, model.LinkArgs{}) + } remoteStorage, remoteActualPath, err := op.GetStorageAndActualPath(file.GetPath()) if err != nil { return nil, err diff --git a/drivers/crypt/meta.go b/drivers/crypt/meta.go index 6a4659910..b7fb5144d 100644 --- a/drivers/crypt/meta.go +++ b/drivers/crypt/meta.go @@ -15,7 +15,7 @@ type Addition struct { EncryptedSuffix string `json:"encrypted_suffix" required:"true" default:".bin" help:"for advanced user only! encrypted files will have this suffix"` FileNameEncoding string `json:"filename_encoding" type:"select" required:"true" options:"base64,base32,base32768" default:"base64" help:"for advanced user only!"` - Thumbnail bool `json:"thumbnail" required:"true" default:"false" help:"enable thumbnail which pre-generated under .thumbnails folder"` + Thumbnail bool `json:"thumbnail" required:"true" default:"false" help:"enable thumbnails under .thumbnails folder; missing image thumbnails will be generated on access"` ShowHidden bool `json:"show_hidden" default:"true" required:"false" help:"show hidden directories and files"` } diff --git a/drivers/crypt/types.go b/drivers/crypt/types.go index 283fd7b6a..8f5f8c125 100644 --- a/drivers/crypt/types.go +++ b/drivers/crypt/types.go @@ -1 +1,9 @@ package crypt + +import "github.com/OpenListTeam/OpenList/v4/internal/model" + +type thumbObject struct { + model.Object + thumbPath string + sourceObj model.Obj +} diff --git a/drivers/crypt/util.go b/drivers/crypt/util.go index e0f2bef74..ff9a7d31d 100644 --- a/drivers/crypt/util.go +++ b/drivers/crypt/util.go @@ -1,9 +1,29 @@ package crypt import ( + "bytes" + "context" + "fmt" + "io" + "os" stdpath "path" "path/filepath" "strings" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/disintegration/imaging" + ffmpeg "github.com/u2takey/ffmpeg-go" +) + +const ( + thumbDirName = ".thumbnails" + thumbExt = ".webp" + thumbWidth = 144 ) // will give the best guessing based on the path @@ -27,3 +47,134 @@ func (d *Crypt) encryptPath(path string, isFolder bool) string { dir, fileName := filepath.Split(path) return stdpath.Join(d.cipher.EncryptDirName(dir), d.cipher.EncryptFileName(fileName)) } + +func isThumbPath(path string) bool { + path = utils.FixAndCleanPath(path) + return stdpath.Base(stdpath.Dir(path)) == thumbDirName && strings.HasSuffix(stdpath.Base(path), thumbExt) +} + +func thumbSourcePath(path string) (string, bool) { + path = utils.FixAndCleanPath(path) + if !isThumbPath(path) { + return "", false + } + name := strings.TrimSuffix(stdpath.Base(path), thumbExt) + if name == "" { + return "", false + } + parentDir := stdpath.Dir(stdpath.Dir(path)) + return stdpath.Join(parentDir, name), true +} + +func thumbTargetDir(path string) string { + return stdpath.Dir(utils.FixAndCleanPath(path)) +} + +func (d *Crypt) newThumbObject(path string, sourceObj model.Obj) *thumbObject { + path = utils.FixAndCleanPath(path) + return &thumbObject{ + Object: model.Object{ + Path: stdpath.Join(d.RemotePath, d.encryptPath(path, false)), + Name: stdpath.Base(path), + Modified: sourceObj.ModTime(), + Ctime: sourceObj.CreateTime(), + }, + thumbPath: path, + sourceObj: sourceObj, + } +} + +func (d *Crypt) ensureThumb(ctx context.Context, thumb *thumbObject) error { + if _, err := d.getActual(ctx, thumb.thumbPath); err == nil { + return nil + } + _, err, _ := d.thumbGroup.Do(thumb.thumbPath, func() (struct{}, error) { + if _, err := d.getActual(ctx, thumb.thumbPath); err == nil { + return struct{}{}, nil + } + buf, err := d.buildThumb(ctx, thumb.sourceObj) + if err != nil { + return struct{}{}, err + } + file := &stream.FileStream{ + Obj: &model.Object{ + Name: stdpath.Base(thumb.thumbPath), + Size: int64(buf.Len()), + Modified: thumb.sourceObj.ModTime(), + }, + Reader: bytes.NewReader(buf.Bytes()), + Mimetype: "image/webp", + } + if err := op.Put(ctx, d, thumbTargetDir(thumb.thumbPath), file, nil); err != nil { + return struct{}{}, err + } + if _, err := d.getActual(ctx, thumb.thumbPath); err != nil { + return struct{}{}, err + } + return struct{}{}, nil + }) + return err +} + +func (d *Crypt) buildThumb(ctx context.Context, sourceObj model.Obj) (*bytes.Buffer, error) { + sourceFile, err := d.openSourceTempFile(ctx, sourceObj) + if err != nil { + return nil, err + } + defer func() { + _ = sourceFile.Close() + _ = os.Remove(sourceFile.Name()) + }() + + image, err := imaging.Decode(sourceFile, imaging.AutoOrientation(true)) + if err != nil { + return nil, err + } + thumbImg := imaging.Resize(image, thumbWidth, 0, imaging.Lanczos) + + tmpPNG, err := os.CreateTemp(conf.Conf.TempDir, "crypt-thumb-*.png") + if err != nil { + return nil, err + } + tmpPNGPath := tmpPNG.Name() + defer func() { + _ = tmpPNG.Close() + _ = os.Remove(tmpPNGPath) + }() + if err := imaging.Encode(tmpPNG, thumbImg, imaging.PNG); err != nil { + return nil, err + } + if err := tmpPNG.Close(); err != nil { + return nil, err + } + + buf := bytes.NewBuffer(nil) + cmd := ffmpeg.Input(tmpPNGPath). + Output("pipe:", ffmpeg.KwArgs{"vcodec": "libwebp", "f": "webp"}). + GlobalArgs("-loglevel", "error"). + Silent(true). + WithOutput(buf, io.Discard) + if err := cmd.Run(); err != nil { + return nil, fmt.Errorf("encode webp failed: %w", err) + } + if buf.Len() == 0 { + return nil, fmt.Errorf("encode webp failed: empty output") + } + return buf, nil +} + +func (d *Crypt) openSourceTempFile(ctx context.Context, sourceObj model.Obj) (*os.File, error) { + link, err := d.Link(ctx, sourceObj, model.LinkArgs{}) + if err != nil { + return nil, err + } + defer link.Close() + + reader, err := link.RangeReader.RangeRead(ctx, http_range.Range{Length: -1}) + if err != nil { + return nil, err + } + defer reader.Close() + + return utils.CreateTempFile(reader, sourceObj.GetSize()) +} diff --git a/drivers/crypt/util_test.go b/drivers/crypt/util_test.go new file mode 100644 index 000000000..81fc22318 --- /dev/null +++ b/drivers/crypt/util_test.go @@ -0,0 +1,57 @@ +package crypt + +import "testing" + +func TestIsThumbPath(t *testing.T) { + t.Parallel() + + cases := []struct { + path string + want bool + }{ + {path: "/photos/.thumbnails/cat.jpg.webp", want: true}, + {path: "/.thumbnails/cat.jpg.webp", want: true}, + {path: "/photos/cat.jpg.webp", want: false}, + {path: "/photos/.thumbnails/cat.jpg.png", want: false}, + {path: "/photos/.thumbnails/", want: false}, + } + + for _, tc := range cases { + if got := isThumbPath(tc.path); got != tc.want { + t.Fatalf("isThumbPath(%q) = %v, want %v", tc.path, got, tc.want) + } + } +} + +func TestThumbSourcePath(t *testing.T) { + t.Parallel() + + cases := []struct { + path string + want string + ok bool + }{ + {path: "/photos/.thumbnails/cat.jpg.webp", want: "/photos/cat.jpg", ok: true}, + {path: "/.thumbnails/cat.jpg.webp", want: "/cat.jpg", ok: true}, + {path: "/photos/.thumbnails/cat.jpg.png", ok: false}, + {path: "/photos/cat.jpg.webp", ok: false}, + } + + for _, tc := range cases { + got, ok := thumbSourcePath(tc.path) + if ok != tc.ok { + t.Fatalf("thumbSourcePath(%q) ok = %v, want %v", tc.path, ok, tc.ok) + } + if got != tc.want { + t.Fatalf("thumbSourcePath(%q) = %q, want %q", tc.path, got, tc.want) + } + } +} + +func TestThumbTargetDir(t *testing.T) { + t.Parallel() + + if got := thumbTargetDir("/photos/.thumbnails/cat.jpg.webp"); got != "/photos/.thumbnails" { + t.Fatalf("thumbTargetDir() = %q, want %q", got, "/photos/.thumbnails") + } +}