From 96021e518a7fa193576cd509d169c2a9f91f1c3c Mon Sep 17 00:00:00 2001 From: Lincoln Nogueira Date: Sat, 13 May 2023 11:27:28 -0300 Subject: [PATCH] feat: add max upload size setting to UI & UI improvements (#1646) * Add preliminar Windows support for both development and production environments. Default profile.Data will be set to "C:\ProgramData\memos" on Windows. Folder will be created if it does not exist, as this behavior is expected for Windows applications. System service installation can be achieved with third-party tools, explained in docs/windows-service.md. Not sure if it's worth using https://github.com/kardianos/service to make service support built-in. This could be a nice addition alongside #1583 (add Windows artifacts) * feat: improve Windows support - Fix local file storage path handling on Windows - Improve Windows dev script * feat: add max upload size setting to UI & more - feat: add max upload size setting to UI - feat: max upload size setting is checked on UI during upload, but also enforced by the server - fix: overflowing mobile layout for Create SSO, Create Storage and other Settings dialogs - feat: add HelpButton component with some links to docs were appropriate - remove LearnMore component in favor of HelpButton - refactor: change some if/else to switch statements - refactor: inline some err == nil checks ! Existing databases without the new setting 'max-upload-size-mib' will show an upload error, but this can be user-fixed by simply setting the value on system settings UI. * improvements requested by @boojack --- api/system.go | 2 + api/system_setting.go | 108 ++++--- server/resource.go | 24 +- server/system.go | 34 ++- store/system_setting.go | 11 +- .../CreateIdentityProviderDialog.tsx | 46 ++- .../components/CreateStorageServiceDialog.tsx | 24 +- web/src/components/LearnMore.tsx | 21 -- .../Settings/PreferencesSection.tsx | 18 +- web/src/components/Settings/SSOSection.tsx | 73 +++-- .../components/Settings/StorageSection.tsx | 8 +- web/src/components/Settings/SystemSection.tsx | 63 +++- .../UpdateCustomizedProfileDialog.tsx | 7 +- .../components/UpdateLocalStorageDialog.tsx | 16 +- web/src/components/kit/HelpButton.tsx | 283 ++++++++++++++++++ web/src/locales/en.json | 19 +- web/src/locales/pt-BR.json | 21 +- web/src/store/module/global.ts | 1 + web/src/store/module/resource.ts | 15 +- web/src/types/modules/system.d.ts | 1 + 20 files changed, 591 insertions(+), 204 deletions(-) delete mode 100644 web/src/components/LearnMore.tsx create mode 100644 web/src/components/kit/HelpButton.tsx diff --git a/api/system.go b/api/system.go index 82e886586..1c5966b77 100644 --- a/api/system.go +++ b/api/system.go @@ -14,6 +14,8 @@ type SystemStatus struct { IgnoreUpgrade bool `json:"ignoreUpgrade"` // Disable public memos. DisablePublicMemos bool `json:"disablePublicMemos"` + // Max upload size. + MaxUploadSizeMiB int `json:"maxUploadSizeMiB"` // Additional style. AdditionalStyle string `json:"additionalStyle"` // Additional script. diff --git a/api/system_setting.go b/api/system_setting.go index 398ec92ad..75bdc35d8 100644 --- a/api/system_setting.go +++ b/api/system_setting.go @@ -2,7 +2,6 @@ package api import ( "encoding/json" - "errors" "fmt" "golang.org/x/exp/slices" @@ -21,6 +20,8 @@ const ( SystemSettingIgnoreUpgradeName SystemSettingName = "ignore-upgrade" // SystemSettingDisablePublicMemosName is the name of disable public memos setting. SystemSettingDisablePublicMemosName SystemSettingName = "disable-public-memos" + // SystemSettingMaxUploadSizeMiBName is the name of max upload size setting. + SystemSettingMaxUploadSizeMiBName SystemSettingName = "max-upload-size-mib" // SystemSettingAdditionalStyleName is the name of additional style. SystemSettingAdditionalStyleName SystemSettingName = "additional-style" // SystemSettingAdditionalScriptName is the name of additional script. @@ -68,6 +69,8 @@ func (key SystemSettingName) String() string { return "ignore-upgrade" case SystemSettingDisablePublicMemosName: return "disable-public-memos" + case SystemSettingMaxUploadSizeMiBName: + return "max-upload-size-mib" case SystemSettingAdditionalStyleName: return "additional-style" case SystemSettingAdditionalScriptName: @@ -97,40 +100,50 @@ type SystemSettingUpsert struct { Description string `json:"description"` } +const systemSettingUnmarshalError = `failed to unmarshal value from system setting "%v"` + func (upsert SystemSettingUpsert) Validate() error { - if upsert.Name == SystemSettingServerIDName { - return errors.New("update server id is not allowed") - } else if upsert.Name == SystemSettingAllowSignUpName { - value := false - err := json.Unmarshal([]byte(upsert.Value), &value) - if err != nil { - return fmt.Errorf("failed to unmarshal system setting allow signup value") + switch settingName := upsert.Name; settingName { + case SystemSettingServerIDName: + return fmt.Errorf("updating %v is not allowed", settingName) + + case SystemSettingAllowSignUpName: + var value bool + if err := json.Unmarshal([]byte(upsert.Value), &value); err != nil { + return fmt.Errorf(systemSettingUnmarshalError, settingName) } - } else if upsert.Name == SystemSettingIgnoreUpgradeName { - value := false - err := json.Unmarshal([]byte(upsert.Value), &value) - if err != nil { - return fmt.Errorf("failed to unmarshal system setting ignore upgrade value") + + case SystemSettingIgnoreUpgradeName: + var value bool + if err := json.Unmarshal([]byte(upsert.Value), &value); err != nil { + return fmt.Errorf(systemSettingUnmarshalError, settingName) } - } else if upsert.Name == SystemSettingDisablePublicMemosName { - value := false - err := json.Unmarshal([]byte(upsert.Value), &value) - if err != nil { - return fmt.Errorf("failed to unmarshal system setting disable public memos value") + + case SystemSettingDisablePublicMemosName: + var value bool + if err := json.Unmarshal([]byte(upsert.Value), &value); err != nil { + return fmt.Errorf(systemSettingUnmarshalError, settingName) } - } else if upsert.Name == SystemSettingAdditionalStyleName { - value := "" - err := json.Unmarshal([]byte(upsert.Value), &value) - if err != nil { - return fmt.Errorf("failed to unmarshal system setting additional style value") + + case SystemSettingMaxUploadSizeMiBName: + var value int + if err := json.Unmarshal([]byte(upsert.Value), &value); err != nil { + return fmt.Errorf(systemSettingUnmarshalError, settingName) } - } else if upsert.Name == SystemSettingAdditionalScriptName { - value := "" - err := json.Unmarshal([]byte(upsert.Value), &value) - if err != nil { - return fmt.Errorf("failed to unmarshal system setting additional script value") + + case SystemSettingAdditionalStyleName: + var value string + if err := json.Unmarshal([]byte(upsert.Value), &value); err != nil { + return fmt.Errorf(systemSettingUnmarshalError, settingName) } - } else if upsert.Name == SystemSettingCustomizedProfileName { + + case SystemSettingAdditionalScriptName: + var value string + if err := json.Unmarshal([]byte(upsert.Value), &value); err != nil { + return fmt.Errorf(systemSettingUnmarshalError, settingName) + } + + case SystemSettingCustomizedProfileName: customizedProfile := CustomizedProfile{ Name: "memos", LogoURL: "", @@ -139,36 +152,37 @@ func (upsert SystemSettingUpsert) Validate() error { Appearance: "system", ExternalURL: "", } - err := json.Unmarshal([]byte(upsert.Value), &customizedProfile) - if err != nil { - return fmt.Errorf("failed to unmarshal system setting customized profile value") + + if err := json.Unmarshal([]byte(upsert.Value), &customizedProfile); err != nil { + return fmt.Errorf(systemSettingUnmarshalError, settingName) } if !slices.Contains(UserSettingLocaleValue, customizedProfile.Locale) { - return fmt.Errorf("invalid locale value") + return fmt.Errorf(`invalid locale value for system setting "%v"`, settingName) } if !slices.Contains(UserSettingAppearanceValue, customizedProfile.Appearance) { - return fmt.Errorf("invalid appearance value") + return fmt.Errorf(`invalid appearance value for system setting "%v"`, settingName) } - } else if upsert.Name == SystemSettingStorageServiceIDName { + + case SystemSettingStorageServiceIDName: value := DatabaseStorage - err := json.Unmarshal([]byte(upsert.Value), &value) - if err != nil { - return fmt.Errorf("failed to unmarshal system setting storage service id value") + if err := json.Unmarshal([]byte(upsert.Value), &value); err != nil { + return fmt.Errorf(systemSettingUnmarshalError, settingName) } return nil - } else if upsert.Name == SystemSettingLocalStoragePathName { + + case SystemSettingLocalStoragePathName: value := "" - err := json.Unmarshal([]byte(upsert.Value), &value) - if err != nil { - return fmt.Errorf("failed to unmarshal system setting local storage path value") + if err := json.Unmarshal([]byte(upsert.Value), &value); err != nil { + return fmt.Errorf(systemSettingUnmarshalError, settingName) } - } else if upsert.Name == SystemSettingOpenAIConfigName { + + case SystemSettingOpenAIConfigName: value := OpenAIConfig{} - err := json.Unmarshal([]byte(upsert.Value), &value) - if err != nil { - return fmt.Errorf("failed to unmarshal system setting openai api config value") + if err := json.Unmarshal([]byte(upsert.Value), &value); err != nil { + return fmt.Errorf(systemSettingUnmarshalError, settingName) } - } else { + + default: return fmt.Errorf("invalid system setting name") } diff --git a/server/resource.go b/server/resource.go index 37a5088a0..6e7de32cc 100644 --- a/server/resource.go +++ b/server/resource.go @@ -25,8 +25,11 @@ import ( ) const ( - // The max file size is 32MB. - maxFileSize = 32 << 20 + // The upload memory buffer is 32 MiB. + // It should be kept low, so RAM usage doesn't get out of control. + // This is unrelated to maximum upload size limit, which is now set through system setting. + maxUploadBufferSizeBytes = 32 << 20 + MebiByte = 1024 * 1024 ) var fileKeyPattern = regexp.MustCompile(`\{[a-z]{1,9}\}`) @@ -67,8 +70,13 @@ func (s *Server) registerResourceRoutes(g *echo.Group) { return echo.NewHTTPError(http.StatusUnauthorized, "Missing user in session") } - if err := c.Request().ParseMultipartForm(maxFileSize); err != nil { - return echo.NewHTTPError(http.StatusBadRequest, "Upload file overload max size").SetInternal(err) + maxUploadSetting := s.Store.GetSystemSettingValueOrDefault(&ctx, api.SystemSettingMaxUploadSizeMiBName, "0") + var settingMaxUploadSizeBytes int + if settingMaxUploadSizeMiB, err := strconv.Atoi(maxUploadSetting); err == nil { + settingMaxUploadSizeBytes = settingMaxUploadSizeMiB * MebiByte + } else { + log.Warn("Failed to parse max upload size", zap.Error(err)) + settingMaxUploadSizeBytes = 0 } file, err := c.FormFile("file") @@ -79,6 +87,14 @@ func (s *Server) registerResourceRoutes(g *echo.Group) { return echo.NewHTTPError(http.StatusBadRequest, "Upload file not found").SetInternal(err) } + if file.Size > int64(settingMaxUploadSizeBytes) { + message := fmt.Sprintf("File size exceeds allowed limit of %d MiB", settingMaxUploadSizeBytes/MebiByte) + return echo.NewHTTPError(http.StatusBadRequest, message).SetInternal(err) + } + if err := c.Request().ParseMultipartForm(maxUploadBufferSizeBytes); err != nil { + return echo.NewHTTPError(http.StatusBadRequest, "Failed to parse upload data").SetInternal(err) + } + filetype := file.Header.Get("Content-Type") size := file.Size sourceFile, err := file.Open() diff --git a/server/system.go b/server/system.go index ac5214571..7b865ef28 100644 --- a/server/system.go +++ b/server/system.go @@ -44,6 +44,7 @@ func (s *Server) registerSystemRoutes(g *echo.Group) { AllowSignUp: false, IgnoreUpgrade: false, DisablePublicMemos: false, + MaxUploadSizeMiB: 32, AdditionalStyle: "", AdditionalScript: "", CustomizedProfile: api.CustomizedProfile{ @@ -74,27 +75,40 @@ func (s *Server) registerSystemRoutes(g *echo.Group) { continue } - if systemSetting.Name == api.SystemSettingAllowSignUpName { + switch systemSetting.Name { + case api.SystemSettingAllowSignUpName: systemStatus.AllowSignUp = baseValue.(bool) - } else if systemSetting.Name == api.SystemSettingIgnoreUpgradeName { + + case api.SystemSettingIgnoreUpgradeName: systemStatus.IgnoreUpgrade = baseValue.(bool) - } else if systemSetting.Name == api.SystemSettingDisablePublicMemosName { + + case api.SystemSettingDisablePublicMemosName: systemStatus.DisablePublicMemos = baseValue.(bool) - } else if systemSetting.Name == api.SystemSettingAdditionalStyleName { + + case api.SystemSettingMaxUploadSizeMiBName: + systemStatus.MaxUploadSizeMiB = int(baseValue.(float64)) + + case api.SystemSettingAdditionalStyleName: systemStatus.AdditionalStyle = baseValue.(string) - } else if systemSetting.Name == api.SystemSettingAdditionalScriptName { + + case api.SystemSettingAdditionalScriptName: systemStatus.AdditionalScript = baseValue.(string) - } else if systemSetting.Name == api.SystemSettingCustomizedProfileName { + + case api.SystemSettingCustomizedProfileName: customizedProfile := api.CustomizedProfile{} - err := json.Unmarshal([]byte(systemSetting.Value), &customizedProfile) - if err != nil { + if err := json.Unmarshal([]byte(systemSetting.Value), &customizedProfile); err != nil { return echo.NewHTTPError(http.StatusInternalServerError, "Failed to unmarshal system setting customized profile value").SetInternal(err) } systemStatus.CustomizedProfile = customizedProfile - } else if systemSetting.Name == api.SystemSettingStorageServiceIDName { + + case api.SystemSettingStorageServiceIDName: systemStatus.StorageServiceID = int(baseValue.(float64)) - } else if systemSetting.Name == api.SystemSettingLocalStoragePathName { + + case api.SystemSettingLocalStoragePathName: systemStatus.LocalStoragePath = baseValue.(string) + + default: + log.Warn("Unknown system setting name", zap.String("setting name", systemSetting.Name.String())) } } diff --git a/store/system_setting.go b/store/system_setting.go index 6cb428b96..0fe7ac71d 100644 --- a/store/system_setting.go +++ b/store/system_setting.go @@ -94,6 +94,15 @@ func (s *Store) FindSystemSetting(ctx context.Context, find *api.SystemSettingFi return systemSettingRaw.toSystemSetting(), nil } +func (s *Store) GetSystemSettingValueOrDefault(ctx *context.Context, find api.SystemSettingName, defaultValue string) string { + if setting, err := s.FindSystemSetting(*ctx, &api.SystemSettingFind{ + Name: find, + }); err == nil { + return setting.Value + } + return defaultValue +} + func upsertSystemSetting(ctx context.Context, tx *sql.Tx, upsert *api.SystemSettingUpsert) (*systemSettingRaw, error) { query := ` INSERT INTO system_setting ( @@ -127,7 +136,7 @@ func findSystemSettingList(ctx context.Context, tx *sql.Tx, find *api.SystemSett query := ` SELECT name, - value, + value, description FROM system_setting WHERE ` + strings.Join(where, " AND ") diff --git a/web/src/components/CreateIdentityProviderDialog.tsx b/web/src/components/CreateIdentityProviderDialog.tsx index 9883dcb49..528388b78 100644 --- a/web/src/components/CreateIdentityProviderDialog.tsx +++ b/web/src/components/CreateIdentityProviderDialog.tsx @@ -1,6 +1,6 @@ import { useEffect, useState } from "react"; import { toast } from "react-hot-toast"; -import { Button, Divider, Input, Radio, RadioGroup, Typography } from "@mui/joy"; +import { Button, Divider, Input, Option, Select, Typography } from "@mui/joy"; import * as api from "@/helpers/api"; import { UNKNOWN_ID } from "@/helpers/consts"; import { absolutifyLink } from "@/helpers/utils"; @@ -101,6 +101,7 @@ const CreateIdentityProviderDialog: React.FC = (props: Props) => { }, }, ]; + const identityProviderTypes = [...new Set(templateList.map((t) => t.type))]; const { confirmCallback, destroy, identityProvider } = props; const [basicInfo, setBasicInfo] = useState({ name: "", @@ -121,7 +122,7 @@ const CreateIdentityProviderDialog: React.FC = (props: Props) => { }, }); const [oauth2Scopes, setOAuth2Scopes] = useState(""); - const [seletedTemplate, setSelectedTemplate] = useState("GitHub"); + const [selectedTemplate, setSelectedTemplate] = useState("GitHub"); const isCreating = identityProvider === undefined; useEffect(() => { @@ -143,7 +144,7 @@ const CreateIdentityProviderDialog: React.FC = (props: Props) => { return; } - const template = templateList.find((t) => t.name === seletedTemplate); + const template = templateList.find((t) => t.name === selectedTemplate); if (template) { setBasicInfo({ name: template.name, @@ -155,7 +156,7 @@ const CreateIdentityProviderDialog: React.FC = (props: Props) => { setOAuth2Scopes(template.config.oauth2Config.scopes.join(" ")); } } - }, [seletedTemplate]); + }, [selectedTemplate]); const handleCloseBtnClick = () => { destroy(); @@ -229,37 +230,34 @@ const CreateIdentityProviderDialog: React.FC = (props: Props) => { return ( <>
-

{t("setting.sso-section." + (isCreating ? "create" : "update") + "-sso")}

-
-
+
{isCreating && ( <> {t("common.type")} - -
- -
-
+ {t("setting.sso-section.template")} - -
- {templateList.map((template) => ( - setSelectedTemplate(e.target.value)} - /> - ))} -
-
+ )} diff --git a/web/src/components/CreateStorageServiceDialog.tsx b/web/src/components/CreateStorageServiceDialog.tsx index ee5d1c451..108f47126 100644 --- a/web/src/components/CreateStorageServiceDialog.tsx +++ b/web/src/components/CreateStorageServiceDialog.tsx @@ -6,7 +6,7 @@ import * as api from "@/helpers/api"; import { generateDialog } from "./Dialog"; import Icon from "./Icon"; import RequiredBadge from "./RequiredBadge"; -import LearnMore from "./LearnMore"; +import HelpButton from "./kit/HelpButton"; interface Props extends DialogProps { storage?: ObjectStorage; @@ -106,15 +106,12 @@ const CreateStorageServiceDialog: React.FC = (props: Props) => { return ( <>
-

- {t("setting.storage-section." + (isCreating ? "create" : "update") + "-storage")} - -

-
-
+
{t("common.name")} @@ -186,13 +183,12 @@ const CreateStorageServiceDialog: React.FC = (props: Props) => { onChange={(e) => setPartialS3Config({ bucket: e.target.value })} fullWidth /> - - {t("setting.storage-section.path")} - - -

{t("setting.storage-section.path-description")}

- -
+
+ + {t("setting.storage-section.path")} + + +
{ - const { url, className } = props; - const { t } = useTranslation(); - - return ( - - {t("common.learn-more")} - - - ); -}; - -export default LearnMore; diff --git a/web/src/components/Settings/PreferencesSection.tsx b/web/src/components/Settings/PreferencesSection.tsx index 17eca991f..e22296470 100644 --- a/web/src/components/Settings/PreferencesSection.tsx +++ b/web/src/components/Settings/PreferencesSection.tsx @@ -1,4 +1,4 @@ -import { Select, Switch, Option } from "@mui/joy"; +import { Switch, Option, Select } from "@mui/joy"; import React from "react"; import { useTranslation } from "react-i18next"; import { useGlobalStore, useUserStore } from "@/store/module"; @@ -53,18 +53,18 @@ const PreferencesSection = () => {

{t("common.basic")}

- {t("common.language")} + {t("common.language")}
- {t("setting.preference-section.theme")} + {t("setting.preference-section.theme")}

{t("setting.preference")}

- {t("setting.preference-section.default-memo-visibility")} + {t("setting.preference-section.default-memo-visibility")}
- {t("setting.preference-section.daily-review-time-offset")} + {t("setting.preference-section.daily-review-time-offset")} +
- {t("setting.system-section.openai-api-key")} - - - {t("setting.system-section.openai-api-key-description")} - - - +
+ {t("setting.system-section.openai-api-key")} + +
= ({ destroy }: Props) => { }); }; - const handleDescriptionChanged = (e: React.ChangeEvent) => { + const handleDescriptionChanged = (e: React.ChangeEvent) => { setPartialState({ description: e.target.value as string, }); @@ -97,7 +98,7 @@ const UpdateCustomizedProfileDialog: React.FC = ({ destroy }: Props) => {
-
+

{t("setting.system-section.server-name")} ({t("setting.system-section.customize-server.default")}) @@ -106,7 +107,7 @@ const UpdateCustomizedProfileDialog: React.FC = ({ destroy }: Props) => {

{t("setting.system-section.customize-server.icon-url")}

{t("setting.system-section.customize-server.description")}

- +