mirror of https://github.com/usememos/memos
feat: pre-signed URL for S3 storage (#2855)
Adds automatically background refresh of all external links if they are belongs to the current blob (S3) storage. The feature is disabled by default in order to keep backward compatibility. The background go-routine spawns once during startup and periodically signs and updates external links if that links belongs to current S3 storage. The original idea was to sign external links on-demand, however, with current architecture it will require duplicated code in plenty of places. If do it, the changes will be quite invasive and in the end pointless: I believe, the architecture will be eventually updated to give more scalable way for pluggable storage. For example - Upload/Download interface without hard dependency on external link. There are stubs already, but I don't feel confident enough to change significant part of the application architecture.pull/2860/head
Aleksandr Baryshnikov
1 year ago
committed by
GitHub
parent
cbcec80c5d
commit
fa17dce046
@ -0,0 +1,140 @@
|
||||
package jobs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"go.uber.org/zap"
|
||||
|
||||
apiv1 "github.com/usememos/memos/api/v1"
|
||||
"github.com/usememos/memos/internal/log"
|
||||
"github.com/usememos/memos/plugin/storage/s3"
|
||||
"github.com/usememos/memos/store"
|
||||
)
|
||||
|
||||
// RunPreSignLinks is a background job that pre-signs external links stored in the database.
|
||||
// It uses S3 client to generate presigned URLs and updates the corresponding resources in the store.
|
||||
func RunPreSignLinks(ctx context.Context, dataStore *store.Store) {
|
||||
for {
|
||||
started := time.Now()
|
||||
if err := signExternalLinks(ctx, dataStore); err != nil {
|
||||
log.Warn("failed sign external links", zap.Error(err))
|
||||
} else {
|
||||
log.Info("links pre-signed", zap.Duration("duration", time.Since(started)))
|
||||
}
|
||||
select {
|
||||
case <-time.After(s3.LinkLifetime / 2):
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func signExternalLinks(ctx context.Context, dataStore *store.Store) error {
|
||||
const pageSize = 32
|
||||
|
||||
objectStore, err := findObjectStorage(ctx, dataStore)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "find object storage")
|
||||
}
|
||||
if objectStore == nil || !objectStore.Config.PreSign {
|
||||
// object storage not set or not supported
|
||||
return nil
|
||||
}
|
||||
|
||||
var offset int
|
||||
var limit = pageSize
|
||||
for {
|
||||
resources, err := dataStore.ListResources(ctx, &store.FindResource{
|
||||
GetBlob: false,
|
||||
Limit: &limit,
|
||||
Offset: &offset,
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "list resources, offset %d", offset)
|
||||
}
|
||||
|
||||
for _, res := range resources {
|
||||
if res.ExternalLink == "" {
|
||||
// not for object store
|
||||
continue
|
||||
}
|
||||
if strings.Contains(res.ExternalLink, "?") && time.Since(time.Unix(res.UpdatedTs, 0)) < s3.LinkLifetime/2 {
|
||||
// resource not signed (hack for migration)
|
||||
// resource was recently updated - skipping
|
||||
continue
|
||||
}
|
||||
newLink, err := objectStore.PreSignLink(ctx, res.ExternalLink)
|
||||
if err != nil {
|
||||
log.Warn("failed pre-sign link", zap.Int32("resource", res.ID), zap.String("link", res.ExternalLink), zap.Error(err))
|
||||
continue // do not fail - we may want update left over links too
|
||||
}
|
||||
now := time.Now().Unix()
|
||||
// we may want to use here transaction and batch update in the future
|
||||
_, err = dataStore.UpdateResource(ctx, &store.UpdateResource{
|
||||
ID: res.ID,
|
||||
UpdatedTs: &now,
|
||||
ExternalLink: &newLink,
|
||||
})
|
||||
if err != nil {
|
||||
// something with DB - better to stop here
|
||||
return errors.Wrapf(err, "update resource %d link to %q", res.ID, newLink)
|
||||
}
|
||||
}
|
||||
|
||||
offset += limit
|
||||
if len(resources) < limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// findObjectStorage returns current default storage if it's S3-compatible or nil otherwise.
|
||||
// Returns error only in case of internal problems (ie: database or configuration issues).
|
||||
// May return nil client and nil error.
|
||||
func findObjectStorage(ctx context.Context, dataStore *store.Store) (*s3.Client, error) {
|
||||
systemSettingStorageServiceID, err := dataStore.GetSystemSetting(ctx, &store.FindSystemSetting{Name: apiv1.SystemSettingStorageServiceIDName.String()})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "Failed to find SystemSettingStorageServiceIDName")
|
||||
}
|
||||
|
||||
storageServiceID := apiv1.DefaultStorage
|
||||
if systemSettingStorageServiceID != nil {
|
||||
err = json.Unmarshal([]byte(systemSettingStorageServiceID.Value), &storageServiceID)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "Failed to unmarshal storage service id")
|
||||
}
|
||||
}
|
||||
storage, err := dataStore.GetStorage(ctx, &store.FindStorage{ID: &storageServiceID})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "Failed to find StorageServiceID")
|
||||
}
|
||||
|
||||
if storage == nil {
|
||||
return nil, nil // storage not configured - not an error, just return empty ref
|
||||
}
|
||||
storageMessage, err := apiv1.ConvertStorageFromStore(storage)
|
||||
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "Failed to ConvertStorageFromStore")
|
||||
}
|
||||
if storageMessage.Type != apiv1.StorageS3 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
s3Config := storageMessage.Config.S3Config
|
||||
return s3.NewClient(ctx, &s3.Config{
|
||||
AccessKey: s3Config.AccessKey,
|
||||
SecretKey: s3Config.SecretKey,
|
||||
EndPoint: s3Config.EndPoint,
|
||||
Region: s3Config.Region,
|
||||
Bucket: s3Config.Bucket,
|
||||
URLPrefix: s3Config.URLPrefix,
|
||||
URLSuffix: s3Config.URLSuffix,
|
||||
PreSign: s3Config.PreSign,
|
||||
})
|
||||
}
|
||||
Loading…
Reference in New Issue