mirror of https://github.com/usememos/memos
refactor: update storage setting
Steven
10 months ago
parent
f25c7d9b24
commit
320963098f
@ -1,110 +0,0 @@
|
||||
package resourcepresign
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/usememos/memos/plugin/storage/s3"
|
||||
storepb "github.com/usememos/memos/proto/gen/store"
|
||||
"github.com/usememos/memos/store"
|
||||
)
|
||||
|
||||
// RunPreSignLinks is a background runner that pre-signs external links stored in the database.
|
||||
// It uses S3 client to generate presigned URLs and updates the corresponding resources in the store.
|
||||
func RunPreSignLinks(ctx context.Context, dataStore *store.Store) {
|
||||
for {
|
||||
if err := signExternalLinks(ctx, dataStore); err != nil {
|
||||
slog.Error("failed to pre-sign links", err)
|
||||
} else {
|
||||
slog.Debug("pre-signed links")
|
||||
}
|
||||
select {
|
||||
case <-time.After(s3.LinkLifetime / 2):
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func signExternalLinks(ctx context.Context, dataStore *store.Store) error {
|
||||
objectStore, err := findObjectStorage(ctx, dataStore)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "find object storage")
|
||||
}
|
||||
if objectStore == nil || !objectStore.Config.PreSign {
|
||||
// object storage not set or not supported
|
||||
return nil
|
||||
}
|
||||
|
||||
resources, err := dataStore.ListResources(ctx, &store.FindResource{
|
||||
GetBlob: false,
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "list resources")
|
||||
}
|
||||
|
||||
for _, resource := range resources {
|
||||
if resource.ExternalLink == "" {
|
||||
// not for object store
|
||||
continue
|
||||
}
|
||||
if strings.Contains(resource.ExternalLink, "?") && time.Since(time.Unix(resource.UpdatedTs, 0)) < s3.LinkLifetime/2 {
|
||||
// resource not signed (hack for migration)
|
||||
// resource was recently updated - skipping
|
||||
continue
|
||||
}
|
||||
|
||||
newLink, err := objectStore.PreSignLink(ctx, resource.ExternalLink)
|
||||
if err != nil {
|
||||
slog.Error("failed to pre-sign link", err)
|
||||
continue
|
||||
}
|
||||
|
||||
now := time.Now().Unix()
|
||||
if _, err := dataStore.UpdateResource(ctx, &store.UpdateResource{
|
||||
ID: resource.ID,
|
||||
UpdatedTs: &now,
|
||||
ExternalLink: &newLink,
|
||||
}); err != nil {
|
||||
return errors.Wrapf(err, "update resource %d link to %q", resource.ID, newLink)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// findObjectStorage returns current default storage if it's S3-compatible or nil otherwise.
|
||||
// Returns error only in case of internal problems (ie: database or configuration issues).
|
||||
// May return nil client and nil error.
|
||||
func findObjectStorage(ctx context.Context, dataStore *store.Store) (*s3.Client, error) {
|
||||
workspaceStorageSetting, err := dataStore.GetWorkspaceStorageSetting(ctx)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "Failed to find workspaceStorageSetting")
|
||||
}
|
||||
if workspaceStorageSetting.StorageType != storepb.WorkspaceStorageSetting_STORAGE_TYPE_EXTERNAL || workspaceStorageSetting.ActivedExternalStorageId == nil {
|
||||
return nil, nil
|
||||
}
|
||||
storage, err := dataStore.GetStorage(ctx, &store.FindStorage{ID: workspaceStorageSetting.ActivedExternalStorageId})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "Failed to find storage")
|
||||
}
|
||||
if storage == nil || storage.Type != storepb.Storage_S3 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
s3Config := storage.Config.GetS3Config()
|
||||
return s3.NewClient(ctx, &s3.Config{
|
||||
AccessKey: s3Config.AccessKey,
|
||||
SecretKey: s3Config.SecretKey,
|
||||
EndPoint: s3Config.EndPoint,
|
||||
Region: s3Config.Region,
|
||||
Bucket: s3Config.Bucket,
|
||||
URLPrefix: s3Config.UrlPrefix,
|
||||
URLSuffix: s3Config.UrlSuffix,
|
||||
PreSign: s3Config.PreSign,
|
||||
})
|
||||
}
|
||||
@ -1,257 +0,0 @@
|
||||
import { Button, IconButton, Input, Checkbox, Typography } from "@mui/joy";
|
||||
import React, { useEffect, useState } from "react";
|
||||
import { toast } from "react-hot-toast";
|
||||
import { storageServiceClient } from "@/grpcweb";
|
||||
import { S3Config, Storage, Storage_Type } from "@/types/proto/api/v1/storage_service";
|
||||
import { useTranslate } from "@/utils/i18n";
|
||||
import { generateDialog } from "./Dialog";
|
||||
import Icon from "./Icon";
|
||||
import LearnMore from "./LearnMore";
|
||||
import RequiredBadge from "./RequiredBadge";
|
||||
|
||||
interface Props extends DialogProps {
|
||||
storage?: Storage;
|
||||
confirmCallback?: () => void;
|
||||
}
|
||||
|
||||
const CreateStorageServiceDialog: React.FC<Props> = (props: Props) => {
|
||||
const t = useTranslate();
|
||||
const { destroy, storage, confirmCallback } = props;
|
||||
const [basicInfo, setBasicInfo] = useState({
|
||||
title: "",
|
||||
});
|
||||
const [type] = useState<Storage_Type>(Storage_Type.S3);
|
||||
const [s3Config, setS3Config] = useState<S3Config>({
|
||||
endPoint: "",
|
||||
region: "",
|
||||
accessKey: "",
|
||||
secretKey: "",
|
||||
path: "",
|
||||
bucket: "",
|
||||
urlPrefix: "",
|
||||
urlSuffix: "",
|
||||
preSign: false,
|
||||
});
|
||||
const isCreating = storage === undefined;
|
||||
|
||||
useEffect(() => {
|
||||
if (storage) {
|
||||
setBasicInfo({
|
||||
title: storage.title,
|
||||
});
|
||||
if (storage.type === "S3") {
|
||||
setS3Config(S3Config.fromPartial(storage.config?.s3Config || {}));
|
||||
}
|
||||
}
|
||||
}, []);
|
||||
|
||||
const handleCloseBtnClick = () => {
|
||||
destroy();
|
||||
};
|
||||
|
||||
const allowConfirmAction = () => {
|
||||
if (basicInfo.title === "") {
|
||||
return false;
|
||||
}
|
||||
if (type === "S3") {
|
||||
if (
|
||||
s3Config.endPoint === "" ||
|
||||
s3Config.region === "" ||
|
||||
s3Config.accessKey === "" ||
|
||||
s3Config.secretKey === "" ||
|
||||
s3Config.bucket === ""
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
};
|
||||
|
||||
const handleConfirmBtnClick = async () => {
|
||||
try {
|
||||
if (isCreating) {
|
||||
await storageServiceClient.createStorage({
|
||||
storage: Storage.fromPartial({
|
||||
title: basicInfo.title,
|
||||
type: type,
|
||||
config: {
|
||||
s3Config: s3Config,
|
||||
},
|
||||
}),
|
||||
});
|
||||
} else {
|
||||
await storageServiceClient.updateStorage({
|
||||
storage: Storage.fromPartial({
|
||||
id: storage?.id,
|
||||
title: basicInfo.title,
|
||||
type: type,
|
||||
config: {
|
||||
s3Config: s3Config,
|
||||
},
|
||||
}),
|
||||
updateMask: ["title", "config"],
|
||||
});
|
||||
}
|
||||
} catch (error: any) {
|
||||
console.error(error);
|
||||
toast.error(error.response.data.message);
|
||||
}
|
||||
if (confirmCallback) {
|
||||
confirmCallback();
|
||||
}
|
||||
destroy();
|
||||
};
|
||||
|
||||
const setPartialS3Config = (state: Partial<S3Config>) => {
|
||||
setS3Config({
|
||||
...s3Config,
|
||||
...state,
|
||||
});
|
||||
};
|
||||
|
||||
return (
|
||||
<>
|
||||
<div className="dialog-header-container">
|
||||
<span>{t(isCreating ? "setting.storage-section.create-storage" : "setting.storage-section.update-storage")}</span>
|
||||
<IconButton size="sm" onClick={handleCloseBtnClick}>
|
||||
<Icon.X className="w-5 h-auto" />
|
||||
</IconButton>
|
||||
</div>
|
||||
<div className="dialog-content-container min-w-[19rem]">
|
||||
<Typography className="!mb-1" level="body-md">
|
||||
{t("common.name")}
|
||||
<RequiredBadge />
|
||||
</Typography>
|
||||
<Input
|
||||
className="mb-2"
|
||||
placeholder={t("common.name")}
|
||||
value={basicInfo.title}
|
||||
onChange={(e) =>
|
||||
setBasicInfo({
|
||||
...basicInfo,
|
||||
title: e.target.value,
|
||||
})
|
||||
}
|
||||
fullWidth
|
||||
/>
|
||||
<Typography className="!mb-1" level="body-md">
|
||||
{t("setting.storage-section.endpoint")}
|
||||
<RequiredBadge />
|
||||
</Typography>
|
||||
<Input
|
||||
className="mb-2"
|
||||
placeholder={t("setting.storage-section.s3-compatible-url")}
|
||||
value={s3Config.endPoint}
|
||||
onChange={(e) => setPartialS3Config({ endPoint: e.target.value })}
|
||||
fullWidth
|
||||
/>
|
||||
<Typography className="!mb-1" level="body-md">
|
||||
{t("setting.storage-section.region")}
|
||||
<RequiredBadge />
|
||||
</Typography>
|
||||
<Input
|
||||
className="mb-2"
|
||||
placeholder={t("setting.storage-section.region-placeholder")}
|
||||
value={s3Config.region}
|
||||
onChange={(e) => setPartialS3Config({ region: e.target.value })}
|
||||
fullWidth
|
||||
/>
|
||||
<Typography className="!mb-1" level="body-md">
|
||||
{t("setting.storage-section.accesskey")}
|
||||
<RequiredBadge />
|
||||
</Typography>
|
||||
<Input
|
||||
className="mb-2"
|
||||
placeholder={t("setting.storage-section.accesskey-placeholder")}
|
||||
value={s3Config.accessKey}
|
||||
onChange={(e) => setPartialS3Config({ accessKey: e.target.value })}
|
||||
fullWidth
|
||||
/>
|
||||
<Typography className="!mb-1" level="body-md">
|
||||
{t("setting.storage-section.secretkey")}
|
||||
<RequiredBadge />
|
||||
</Typography>
|
||||
<Input
|
||||
className="mb-2"
|
||||
placeholder={t("setting.storage-section.secretkey-placeholder")}
|
||||
value={s3Config.secretKey}
|
||||
onChange={(e) => setPartialS3Config({ secretKey: e.target.value })}
|
||||
fullWidth
|
||||
/>
|
||||
<Typography className="!mb-1" level="body-md">
|
||||
{t("setting.storage-section.bucket")}
|
||||
<RequiredBadge />
|
||||
</Typography>
|
||||
<Input
|
||||
className="mb-2"
|
||||
placeholder={t("setting.storage-section.bucket-placeholder")}
|
||||
value={s3Config.bucket}
|
||||
onChange={(e) => setPartialS3Config({ bucket: e.target.value })}
|
||||
fullWidth
|
||||
/>
|
||||
<div className="flex flex-row items-center mb-1">
|
||||
<Typography level="body-md">{t("setting.storage-section.path")}</Typography>
|
||||
<LearnMore
|
||||
className="ml-1"
|
||||
title={t("setting.storage-section.path-description")}
|
||||
url="https://usememos.com/docs/advanced-settings/local-storage"
|
||||
/>
|
||||
</div>
|
||||
<Input
|
||||
className="mb-2"
|
||||
placeholder={t("setting.storage-section.path-placeholder") + "/{year}/{month}/{filename}"}
|
||||
value={s3Config.path}
|
||||
onChange={(e) => setPartialS3Config({ path: e.target.value })}
|
||||
fullWidth
|
||||
/>
|
||||
<Typography className="!mb-1" level="body-md">
|
||||
{t("setting.storage-section.url-prefix")}
|
||||
</Typography>
|
||||
<Input
|
||||
className="mb-2"
|
||||
placeholder={t("setting.storage-section.url-prefix-placeholder")}
|
||||
value={s3Config.urlPrefix}
|
||||
onChange={(e) => setPartialS3Config({ urlPrefix: e.target.value })}
|
||||
fullWidth
|
||||
/>
|
||||
<Typography className="!mb-1" level="body-md">
|
||||
{t("setting.storage-section.url-suffix")}
|
||||
</Typography>
|
||||
<Input
|
||||
className="mb-2"
|
||||
placeholder={t("setting.storage-section.url-suffix-placeholder")}
|
||||
value={s3Config.urlSuffix}
|
||||
onChange={(e) => setPartialS3Config({ urlSuffix: e.target.value })}
|
||||
fullWidth
|
||||
/>
|
||||
<Checkbox
|
||||
className="mb-2"
|
||||
label={t("setting.storage-section.presign-placeholder")}
|
||||
checked={s3Config.preSign}
|
||||
onChange={(e) => setPartialS3Config({ preSign: e.target.checked })}
|
||||
/>
|
||||
<div className="mt-2 w-full flex flex-row justify-end items-center space-x-1">
|
||||
<Button variant="plain" color="neutral" onClick={handleCloseBtnClick}>
|
||||
{t("common.cancel")}
|
||||
</Button>
|
||||
<Button onClick={handleConfirmBtnClick} disabled={!allowConfirmAction()}>
|
||||
{t(isCreating ? "common.create" : "common.update")}
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
function showCreateStorageServiceDialog(storage?: Storage, confirmCallback?: () => void) {
|
||||
generateDialog(
|
||||
{
|
||||
className: "create-storage-service-dialog",
|
||||
dialogName: "create-storage-service-dialog",
|
||||
},
|
||||
CreateStorageServiceDialog,
|
||||
{ storage, confirmCallback },
|
||||
);
|
||||
}
|
||||
|
||||
export default showCreateStorageServiceDialog;
|
||||
Loading…
Reference in New Issue