diff --git a/api/storage.go b/api/storage.go index 73208a891..d578e6013 100644 --- a/api/storage.go +++ b/api/storage.go @@ -7,18 +7,22 @@ type Storage struct { UpdatedTs int64 `json:"updatedTs"` Name string `json:"name"` EndPoint string `json:"endPoint"` + Region string `json:"region"` AccessKey string `json:"accessKey"` SecretKey string `json:"secretKey"` Bucket string `json:"bucket"` + URLPrefix string `json:"urlPrefix"` } type StorageCreate struct { CreatorID int `json:"creatorId"` Name string `json:"name"` EndPoint string `json:"endPoint"` + Region string `json:"region"` AccessKey string `json:"accessKey"` SecretKey string `json:"secretKey"` Bucket string `json:"bucket"` + URLPrefix string `json:"urlPrefix"` } type StoragePatch struct { @@ -26,13 +30,17 @@ type StoragePatch struct { UpdatedTs *int64 Name *string `json:"name"` EndPoint *string `json:"endPoint"` + Region *string `json:"region"` AccessKey *string `json:"accessKey"` SecretKey *string `json:"secretKey"` Bucket *string `json:"bucket"` + URLPrefix *string `json:"urlPrefix"` } type StorageFind struct { - CreatorID *int `json:"creatorId"` + ID *int `json:"id"` + Name *string `json:"name"` + CreatorID *int `json:"creatorId"` } type StorageDelete struct { diff --git a/api/system_setting.go b/api/system_setting.go index e41368413..1e569256d 100644 --- a/api/system_setting.go +++ b/api/system_setting.go @@ -23,6 +23,8 @@ const ( SystemSettingAdditionalScriptName SystemSettingName = "additionalScript" // SystemSettingCustomizedProfileName is the key type of customized server profile. SystemSettingCustomizedProfileName SystemSettingName = "customizedProfile" + // SystemSettingStorageServiceName is the key type of sotrage service name. + SystemSettingStorageServiceName SystemSettingName = "storageServiceName" ) // CustomizedProfile is the struct definition for SystemSettingCustomizedProfileName system setting item. @@ -55,6 +57,8 @@ func (key SystemSettingName) String() string { return "additionalScript" case SystemSettingCustomizedProfileName: return "customizedProfile" + case SystemSettingStorageServiceName: + return "storageServiceName" } return "" } @@ -127,6 +131,8 @@ func (upsert SystemSettingUpsert) Validate() error { if !slices.Contains(UserSettingAppearanceValue, customizedProfile.Appearance) { return fmt.Errorf("invalid appearance value") } + } else if upsert.Name == SystemSettingStorageServiceName { + return nil } else { return fmt.Errorf("invalid system setting name") } diff --git a/go.mod b/go.mod index 58b03121f..5c7da413b 100644 --- a/go.mod +++ b/go.mod @@ -20,11 +20,26 @@ require ( ) require ( + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.22 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.28 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.22 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.29 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.19 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.23 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.22 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.22 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.12.1 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.18.3 // indirect + github.com/aws/smithy-go v1.13.5 // indirect github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/golang-jwt/jwt v3.2.2+incompatible // indirect github.com/gorilla/context v1.1.1 // indirect github.com/gorilla/securecookie v1.1.1 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/labstack/gommon v0.3.1 // indirect github.com/mattn/go-colorable v0.1.12 // indirect @@ -44,6 +59,11 @@ require ( ) require ( + github.com/aws/aws-sdk-go-v2 v1.17.4 + github.com/aws/aws-sdk-go-v2/config v1.18.12 + github.com/aws/aws-sdk-go-v2/credentials v1.13.12 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.51 + github.com/aws/aws-sdk-go-v2/service/s3 v1.30.2 github.com/pkg/errors v0.9.1 github.com/segmentio/analytics-go v3.1.0+incompatible github.com/stretchr/testify v1.8.0 diff --git a/go.sum b/go.sum index 1764369a4..c45448819 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,41 @@ +github.com/aws/aws-sdk-go-v2 v1.17.4 h1:wyC6p9Yfq6V2y98wfDsj6OnNQa4w2BLGCLIxzNhwOGY= +github.com/aws/aws-sdk-go-v2 v1.17.4/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10/go.mod h1:VeTZetY5KRJLuD/7fkQXMU6Mw7H5m/KP2J5Iy9osMno= +github.com/aws/aws-sdk-go-v2/config v1.18.12 h1:fKs/I4wccmfrNRO9rdrbMO1NgLxct6H9rNMiPdBxHWw= +github.com/aws/aws-sdk-go-v2/config v1.18.12/go.mod h1:J36fOhj1LQBr+O4hJCiT8FwVvieeoSGOtPuvhKlsNu8= +github.com/aws/aws-sdk-go-v2/credentials v1.13.12 h1:Cb+HhuEnV19zHRaYYVglwvdHGMJWbdsyP4oHhw04xws= +github.com/aws/aws-sdk-go-v2/credentials v1.13.12/go.mod h1:37HG2MBroXK3jXfxVGtbM2J48ra2+Ltu+tmwr/jO0KA= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.22 h1:3aMfcTmoXtTZnaT86QlVaYh+BRMbvrrmZwIQ5jWqCZQ= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.22/go.mod h1:YGSIJyQ6D6FjKMQh16hVFSIUD54L4F7zTGePqYMYYJU= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.51 h1:iTFYCAdKzSAjGnVIUe88Hxvix0uaBqr0Rv7qJEOX5hE= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.51/go.mod h1:7Grl2gV+dx9SWrUIgwwlUvU40t7+lOSbx34XwfmsTkY= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.28 h1:r+XwaCLpIvCKjBIYy/HVZujQS9tsz5ohHG3ZIe0wKoE= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.28/go.mod h1:3lwChorpIM/BhImY/hy+Z6jekmN92cXGPI1QJasVPYY= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.22 h1:7AwGYXDdqRQYsluvKFmWoqpcOQJ4bH634SkYf3FNj/A= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.22/go.mod h1:EqK7gVrIGAHyZItrD1D8B0ilgwMD1GiWAmbU4u/JHNk= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.29 h1:J4xhFd6zHhdF9jPP0FQJ6WknzBboGMBNjKOv4iTuw4A= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.29/go.mod h1:TwuqRBGzxjQJIwH16/fOZodwXt2Zxa9/cwJC5ke4j7s= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.19 h1:FGvpyTg2LKEmMrLlpjOgkoNp9XF5CGeyAyo33LdqZW8= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.19/go.mod h1:8W88sW3PjamQpKFUQvHWWKay6ARsNvZnzU7+a4apubw= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 h1:y2+VQzC6Zh2ojtV2LoC0MNwHWc6qXv/j2vrQtlftkdA= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11/go.mod h1:iV4q2hsqtNECrfmlXyord9u4zyuFEJX9eLgLpSPzWA8= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.23 h1:c5+bNdV8E4fIPteWx4HZSkqI07oY9exbfQ7JH7Yx4PI= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.23/go.mod h1:1jcUfF+FAOEwtIcNiHPaV4TSoZqkUIPzrohmD7fb95c= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.22 h1:LjFQf8hFuMO22HkV5VWGLBvmCLBCLPivUAmpdpnp4Vs= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.22/go.mod h1:xt0Au8yPIwYXf/GYPy/vl4K3CgwhfQMYbrH7DlUUIws= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.22 h1:ISLJ2BKXe4zzyZ7mp5ewKECiw0U7KpLgS3S6OxY9Cm0= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.22/go.mod h1:QFVbqK54XArazLvn2wvWMRBi/jGrWii46qbr5DyPGjc= +github.com/aws/aws-sdk-go-v2/service/s3 v1.30.2 h1:5EQWIFO+Hc8E2hFcXQJ1vm6ufl/PMt/6RVRDZRju2vM= +github.com/aws/aws-sdk-go-v2/service/s3 v1.30.2/go.mod h1:SXDHd6fI2RhqB7vmAzyYQCTQnpZrIprVJvYxpzW3JAM= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.1 h1:lQKN/LNa3qqu2cDOQZybP7oL4nMGGiFqob0jZJaR8/4= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.1/go.mod h1:IgV8l3sj22nQDd5qcAGY0WenwCzCphqdbFOpfktZPrI= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.1 h1:0bLhH6DRAqox+g0LatcjGKjjhU6Eudyys6HB6DJVPj8= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.1/go.mod h1:O1YSOg3aekZibh2SngvCRRG+cRHKKlYgxf/JBF/Kr/k= +github.com/aws/aws-sdk-go-v2/service/sts v1.18.3 h1:s49mSnsBZEXjfGBkRfmK+nPqzT7Lt3+t2SmAKNyHblw= +github.com/aws/aws-sdk-go-v2/service/sts v1.18.3/go.mod h1:b+psTJn33Q4qGoDaM7ZiOVVG8uVjGI6HaZ8WBHdgDgU= +github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= +github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= @@ -7,6 +45,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8= @@ -17,6 +57,10 @@ github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyC github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1 h1:DHd3rPN5lE3Ts3D8rKkQ8x/0kqfeNmBAaiSi+o7FsgI= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -89,6 +133,8 @@ golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9/go.mod h1:tRJNPiyCQ0inRvYxb gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/plugin/storage/s3/s3.go b/plugin/storage/s3/s3.go new file mode 100644 index 000000000..7fee14c64 --- /dev/null +++ b/plugin/storage/s3/s3.go @@ -0,0 +1,67 @@ +package s3 + +import ( + "context" + "fmt" + "io" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/feature/s3/manager" + awss3 "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/usememos/memos/api" +) + +type Client struct { + Client *awss3.Client + BucketName string + URLPrefix string +} + +func NewClient(ctx context.Context, storage *api.Storage) (*Client, error) { + resolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) { + return aws.Endpoint{ + URL: storage.EndPoint, + SigningRegion: storage.Region, + }, nil + }) + + cfg, err := config.LoadDefaultConfig(ctx, + config.WithEndpointResolverWithOptions(resolver), + config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(storage.AccessKey, storage.SecretKey, "")), + ) + if err != nil { + return nil, err + } + + client := awss3.NewFromConfig(cfg) + + return &Client{ + Client: client, + BucketName: storage.Bucket, + URLPrefix: storage.URLPrefix, + }, nil +} + +func (client *Client) UploadFile(ctx context.Context, filename string, fileType string, src io.Reader, storage *api.Storage) (*string, error) { + uploader := manager.NewUploader(client.Client) + resp, err := uploader.Upload(ctx, &awss3.PutObjectInput{ + Bucket: aws.String(client.BucketName), + Key: aws.String(filename), + Body: src, + ContentType: aws.String(fileType), + ACL: types.ObjectCannedACL(*aws.String("public-read")), + }) + if err != nil { + return nil, err + } + var link string + if storage.URLPrefix == "" { + link = resp.Location + } else { + link = fmt.Sprintf("%s/%s", storage.URLPrefix, filename) + } + return &link, nil +} diff --git a/server/resource.go b/server/resource.go index 3d79b0a54..bb55466a7 100644 --- a/server/resource.go +++ b/server/resource.go @@ -11,12 +11,12 @@ import ( "strings" "time" + "github.com/labstack/echo/v4" "github.com/pkg/errors" "github.com/usememos/memos/api" "github.com/usememos/memos/common" metric "github.com/usememos/memos/plugin/metrics" - - "github.com/labstack/echo/v4" + "github.com/usememos/memos/plugin/storage/s3" ) const ( @@ -85,18 +85,48 @@ func (s *Server) registerResourceRoutes(g *echo.Group) { } defer src.Close() - fileBytes, err := io.ReadAll(src) - if err != nil { - return echo.NewHTTPError(http.StatusInternalServerError, "Failed to read file").SetInternal(err) + var resourceCreate *api.ResourceCreate + systemSettingStorageServiceName := api.SystemSettingStorageServiceName + systemSetting, err := s.Store.FindSystemSetting(ctx, &api.SystemSettingFind{Name: &systemSettingStorageServiceName}) + if err != nil && common.ErrorCode(err) != common.NotFound { + return echo.NewHTTPError(http.StatusInternalServerError, "Failed to find storage").SetInternal(err) } + if common.ErrorCode(err) == common.NotFound || systemSetting.Value == "" { + fileBytes, err := io.ReadAll(src) + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, "Failed to read file").SetInternal(err) + } + resourceCreate = &api.ResourceCreate{ + CreatorID: userID, + Filename: filename, + Type: filetype, + Size: size, + Blob: fileBytes, + } + } else { + storage, err := s.Store.FindStorage(ctx, &api.StorageFind{Name: &systemSetting.Value}) + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, "Failed to find storage").SetInternal(err) + } - resourceCreate := &api.ResourceCreate{ - CreatorID: userID, - Filename: filename, - Type: filetype, - Size: size, - Blob: fileBytes, + s3client, err := s3.NewClient(ctx, storage) + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, "Failed to new s3 client").SetInternal(err) + } + + link, err := s3client.UploadFile(ctx, filename, filetype, src, storage) + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, "Failed to upload via s3 client").SetInternal(err) + } + + resourceCreate = &api.ResourceCreate{ + CreatorID: userID, + Filename: filename, + Type: filetype, + ExternalLink: *link, + } } + resource, err := s.Store.CreateResource(ctx, resourceCreate) if err != nil { return echo.NewHTTPError(http.StatusInternalServerError, "Failed to create resource").SetInternal(err) diff --git a/server/server.go b/server/server.go index 2863db7fe..f43361f9a 100644 --- a/server/server.go +++ b/server/server.go @@ -115,6 +115,7 @@ func NewServer(ctx context.Context, profile *profile.Profile) (*Server, error) { s.registerShortcutRoutes(apiGroup) s.registerResourceRoutes(apiGroup) s.registerTagRoutes(apiGroup) + s.registerStorageRoutes(apiGroup) return s, nil } diff --git a/server/storage.go b/server/storage.go new file mode 100644 index 000000000..9c77f1fd1 --- /dev/null +++ b/server/storage.go @@ -0,0 +1,143 @@ +package server + +import ( + "encoding/json" + "fmt" + "net/http" + "strconv" + "time" + + "github.com/labstack/echo/v4" + "github.com/usememos/memos/api" + "github.com/usememos/memos/common" +) + +func (s *Server) registerStorageRoutes(g *echo.Group) { + g.POST("/storage", func(c echo.Context) error { + ctx := c.Request().Context() + userID, ok := c.Get(getUserIDContextKey()).(int) + if !ok { + return echo.NewHTTPError(http.StatusUnauthorized, "Missing user in session") + } + + user, err := s.Store.FindUser(ctx, &api.UserFind{ + ID: &userID, + }) + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, "Failed to find user").SetInternal(err) + } + if user == nil || user.Role != api.Host { + return echo.NewHTTPError(http.StatusUnauthorized, "Unauthorized") + } + + storageCreate := &api.StorageCreate{} + if err := json.NewDecoder(c.Request().Body).Decode(storageCreate); err != nil { + return echo.NewHTTPError(http.StatusBadRequest, "Malformatted post storage request").SetInternal(err) + } + + storageCreate.CreatorID = userID + storage, err := s.Store.CreateStorage(ctx, storageCreate) + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, "Failed to create storage").SetInternal(err) + } + + c.Response().Header().Set(echo.HeaderContentType, echo.MIMEApplicationJSONCharsetUTF8) + if err := json.NewEncoder(c.Response().Writer).Encode(composeResponse(storage)); err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, "Failed to encode storage response").SetInternal(err) + } + return nil + }) + + g.PATCH("/storage/:storageId", func(c echo.Context) error { + ctx := c.Request().Context() + userID, ok := c.Get(getUserIDContextKey()).(int) + if !ok { + return echo.NewHTTPError(http.StatusUnauthorized, "Missing user in session") + } + + storageID, err := strconv.Atoi(c.Param("storageId")) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("ID is not a number: %s", c.Param("storageId"))).SetInternal(err) + } + + storage, err := s.Store.FindStorage(ctx, &api.StorageFind{ + ID: &storageID, + }) + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, "Failed to find storage").SetInternal(err) + } + if storage.CreatorID != userID { + return echo.NewHTTPError(http.StatusUnauthorized, "Unauthorized") + } + + currentTs := time.Now().Unix() + storagePatch := &api.StoragePatch{ + ID: storageID, + UpdatedTs: ¤tTs, + } + if err := json.NewDecoder(c.Request().Body).Decode(storagePatch); err != nil { + return echo.NewHTTPError(http.StatusBadRequest, "Malformatted patch storage request").SetInternal(err) + } + + storage, err = s.Store.PatchStorage(ctx, storagePatch) + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, "Failed to patch storage").SetInternal(err) + } + + c.Response().Header().Set(echo.HeaderContentType, echo.MIMEApplicationJSONCharsetUTF8) + if err := json.NewEncoder(c.Response().Writer).Encode(composeResponse(storage)); err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, "Failed to encode memo response").SetInternal(err) + } + return nil + }) + + g.GET("/storage", func(c echo.Context) error { + ctx := c.Request().Context() + storageList, err := s.Store.FindStorageList(ctx, &api.StorageFind{}) + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, "Failed to find storage list").SetInternal(err) + } + + c.Response().Header().Set(echo.HeaderContentType, echo.MIMEApplicationJSONCharsetUTF8) + if err := json.NewEncoder(c.Response().Writer).Encode(composeResponse(storageList)); err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, "Failed to encode storage list response").SetInternal(err) + } + return nil + }) + + g.DELETE("/storage/:storageId", func(c echo.Context) error { + ctx := c.Request().Context() + userID, ok := c.Get(getUserIDContextKey()).(int) + if !ok { + return echo.NewHTTPError(http.StatusUnauthorized, "Missing user in session") + } + + storageID, err := strconv.Atoi(c.Param("storageId")) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("ID is not a number: %s", c.Param("storageId"))).SetInternal(err) + } + + storage, err := s.Store.FindStorage(ctx, &api.StorageFind{ + ID: &storageID, + }) + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, "Failed to find storage").SetInternal(err) + } + if storage.CreatorID != userID { + return echo.NewHTTPError(http.StatusUnauthorized, "Unauthorized") + } + + storageDelete := &api.StorageDelete{ + ID: storageID, + } + + if err = s.Store.DeleteStorage(ctx, storageDelete); err != nil { + if common.ErrorCode(err) == common.NotFound { + return echo.NewHTTPError(http.StatusNotFound, fmt.Sprintf("Storage ID not found: %d", storageID)) + } + return echo.NewHTTPError(http.StatusInternalServerError, "Failed to delete storage").SetInternal(err) + } + + return c.JSON(http.StatusOK, true) + }) +} diff --git a/server/system.go b/server/system.go index 7cbb8ec46..4850c50ce 100644 --- a/server/system.go +++ b/server/system.go @@ -63,7 +63,7 @@ func (s *Server) registerSystemRoutes(g *echo.Group) { return echo.NewHTTPError(http.StatusInternalServerError, "Failed to find system setting list").SetInternal(err) } for _, systemSetting := range systemSettingList { - if systemSetting.Name == api.SystemSettingServerID || systemSetting.Name == api.SystemSettingSecretSessionName { + if systemSetting.Name == api.SystemSettingServerID || systemSetting.Name == api.SystemSettingSecretSessionName || systemSetting.Name == api.SystemSettingStorageServiceName { continue } diff --git a/store/db/migration/dev/LATEST__SCHEMA.sql b/store/db/migration/dev/LATEST__SCHEMA.sql index cc6cd3c2f..c4abd5fc5 100644 --- a/store/db/migration/dev/LATEST__SCHEMA.sql +++ b/store/db/migration/dev/LATEST__SCHEMA.sql @@ -110,9 +110,11 @@ CREATE TABLE storage ( creator_id INTEGER NOT NULL, created_ts BIGINT NOT NULL DEFAULT (strftime('%s', 'now')), updated_ts BIGINT NOT NULL DEFAULT (strftime('%s', 'now')), - name TEXT NOT NULL DEFAULT '', + name TEXT NOT NULL DEFAULT '' UNIQUE, end_point TEXT NOT NULL DEFAULT '', + region TEXT NOT NULL DEFAULT '', access_key TEXT NOT NULL DEFAULT '', secret_key TEXT NOT NULL DEFAULT '', - bucket TEXT NOT NULL DEFAULT '' + bucket TEXT NOT NULL DEFAULT '', + url_prefix TEXT NOT NULL DEFAULT '' ); \ No newline at end of file diff --git a/store/storage.go b/store/storage.go index 514792ecc..5638dab8b 100644 --- a/store/storage.go +++ b/store/storage.go @@ -17,9 +17,11 @@ type storageRaw struct { UpdatedTs int64 Name string EndPoint string + Region string AccessKey string SecretKey string Bucket string + URLPrefix string } func (raw *storageRaw) toStorage() *api.Storage { @@ -30,9 +32,11 @@ func (raw *storageRaw) toStorage() *api.Storage { UpdatedTs: raw.UpdatedTs, Name: raw.Name, EndPoint: raw.EndPoint, + Region: raw.Region, AccessKey: raw.AccessKey, SecretKey: raw.SecretKey, Bucket: raw.Bucket, + URLPrefix: raw.URLPrefix, } } @@ -94,6 +98,26 @@ func (s *Store) FindStorageList(ctx context.Context, find *api.StorageFind) ([]* return list, nil } +func (s *Store) FindStorage(ctx context.Context, find *api.StorageFind) (*api.Storage, error) { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return nil, FormatError(err) + } + defer tx.Rollback() + + list, err := findStorageRawList(ctx, tx, find) + if err != nil { + return nil, err + } + + if len(list) == 0 { + return nil, &common.Error{Code: common.NotFound, Err: fmt.Errorf("not found")} + } + + storageRaw := list[0] + return storageRaw.toStorage(), nil +} + func (s *Store) DeleteStorage(ctx context.Context, delete *api.StorageDelete) error { tx, err := s.db.BeginTx(ctx, nil) if err != nil { @@ -113,16 +137,16 @@ func (s *Store) DeleteStorage(ctx context.Context, delete *api.StorageDelete) er } func createStorageRaw(ctx context.Context, tx *sql.Tx, create *api.StorageCreate) (*storageRaw, error) { - set := []string{"creator_id", "name", "end_point", "access_key", "secret_key", "bucket"} - args := []interface{}{create.CreatorID, create.Name, create.AccessKey, create.SecretKey, create.Bucket} - placeholder := []string{"?", "?", "?", "?", "?", "?"} + set := []string{"creator_id", "name", "end_point", "region", "access_key", "secret_key", "bucket", "url_prefix"} + args := []interface{}{create.CreatorID, create.Name, create.EndPoint, create.Region, create.AccessKey, create.SecretKey, create.Bucket, create.URLPrefix} + placeholder := []string{"?", "?", "?", "?", "?", "?", "?", "?"} query := ` INSERT INTO storage ( ` + strings.Join(set, ", ") + ` ) VALUES (` + strings.Join(placeholder, ",") + `) - RETURNING id, creator_id, created_ts, updated_ts, name, end_point, access_key, secret_key, bucket + RETURNING id, creator_id, created_ts, updated_ts, name, end_point, region, access_key, secret_key, bucket, url_prefix ` var storageRaw storageRaw if err := tx.QueryRowContext(ctx, query, args...).Scan( @@ -132,9 +156,11 @@ func createStorageRaw(ctx context.Context, tx *sql.Tx, create *api.StorageCreate &storageRaw.UpdatedTs, &storageRaw.Name, &storageRaw.EndPoint, + &storageRaw.Region, &storageRaw.AccessKey, &storageRaw.SecretKey, &storageRaw.Bucket, + &storageRaw.URLPrefix, ); err != nil { return nil, FormatError(err) } @@ -153,6 +179,9 @@ func patchStorageRaw(ctx context.Context, tx *sql.Tx, patch *api.StoragePatch) ( if v := patch.EndPoint; v != nil { set, args = append(set, "end_point = ?"), append(args, *v) } + if v := patch.Region; v != nil { + set, args = append(set, "region = ?"), append(args, *v) + } if v := patch.AccessKey; v != nil { set, args = append(set, "access_key = ?"), append(args, *v) } @@ -162,6 +191,9 @@ func patchStorageRaw(ctx context.Context, tx *sql.Tx, patch *api.StoragePatch) ( if v := patch.Bucket; v != nil { set, args = append(set, "bucket = ?"), append(args, *v) } + if v := patch.URLPrefix; v != nil { + set, args = append(set, "url_prefix = ?"), append(args, *v) + } args = append(args, patch.ID) @@ -169,7 +201,7 @@ func patchStorageRaw(ctx context.Context, tx *sql.Tx, patch *api.StoragePatch) ( UPDATE storage SET ` + strings.Join(set, ", ") + ` WHERE id = ? - RETURNING id, creator_id, created_ts, updated_ts, name, end_point, access_key, secret_key, bucket + RETURNING id, creator_id, created_ts, updated_ts, name, end_point, region, access_key, secret_key, bucket, url_prefix ` var storageRaw storageRaw @@ -180,9 +212,11 @@ func patchStorageRaw(ctx context.Context, tx *sql.Tx, patch *api.StoragePatch) ( &storageRaw.UpdatedTs, &storageRaw.Name, &storageRaw.EndPoint, + &storageRaw.Region, &storageRaw.AccessKey, &storageRaw.SecretKey, &storageRaw.Bucket, + &storageRaw.URLPrefix, ); err != nil { return nil, FormatError(err) } @@ -193,6 +227,9 @@ func patchStorageRaw(ctx context.Context, tx *sql.Tx, patch *api.StoragePatch) ( func findStorageRawList(ctx context.Context, tx *sql.Tx, find *api.StorageFind) ([]*storageRaw, error) { where, args := []string{"1 = 1"}, []interface{}{} + if v := find.Name; v != nil { + where, args = append(where, "name = ?"), append(args, *v) + } if v := find.CreatorID; v != nil { where, args = append(where, "creator_id = ?"), append(args, *v) } @@ -204,9 +241,11 @@ func findStorageRawList(ctx context.Context, tx *sql.Tx, find *api.StorageFind) created_ts, name, end_point, + region, access_key, secret_key, - bucket + bucket, + url_prefix FROM storage WHERE ` + strings.Join(where, " AND ") + ` ORDER BY created_ts DESC @@ -226,9 +265,11 @@ func findStorageRawList(ctx context.Context, tx *sql.Tx, find *api.StorageFind) &storageRaw.CreatedTs, &storageRaw.Name, &storageRaw.EndPoint, + &storageRaw.Region, &storageRaw.AccessKey, &storageRaw.SecretKey, &storageRaw.Bucket, + &storageRaw.URLPrefix, ); err != nil { return nil, FormatError(err) } diff --git a/store/system_setting.go b/store/system_setting.go index e72b5dc74..0302ac3b5 100644 --- a/store/system_setting.go +++ b/store/system_setting.go @@ -122,7 +122,7 @@ func findSystemSettingList(ctx context.Context, tx *sql.Tx, find *api.SystemSett query := ` SELECT name, - value, + value, description FROM system_setting WHERE ` + strings.Join(where, " AND ")