From dc5d705f8ce6c4dfc9bfe39f2611487c98499def Mon Sep 17 00:00:00 2001 From: boojack Date: Sun, 6 Nov 2022 12:21:58 +0800 Subject: [PATCH] feat: vacuum records manually (#420) --- api/memo.go | 2 +- api/memo_organizer.go | 5 +++ api/memo_resource.go | 2 +- api/resource.go | 12 +++----- api/shortcut.go | 5 ++- api/user_setting.go | 4 +++ server/memo.go | 2 +- server/resource.go | 14 +++++++-- server/shortcut.go | 2 +- store/memo.go | 39 +++++++++++++++-------- store/memo_organizer.go | 68 +++++++++++++++++++++++++++++++++++++++++ store/memo_resource.go | 34 +++++++++++++++++++-- store/resource.go | 39 ++++++++++++++++------- store/shortcut.go | 37 +++++++++++++++++++--- store/store.go | 49 +++++++++++++++++++++++++++++ store/user.go | 12 +++++--- store/user_setting.go | 19 ++++++++++++ 17 files changed, 295 insertions(+), 50 deletions(-) diff --git a/api/memo.go b/api/memo.go index f9f8a346..cd76e1f3 100644 --- a/api/memo.go +++ b/api/memo.go @@ -90,5 +90,5 @@ type MemoFind struct { } type MemoDelete struct { - ID int `json:"id"` + ID int } diff --git a/api/memo_organizer.go b/api/memo_organizer.go index 4897a99f..7c99280c 100644 --- a/api/memo_organizer.go +++ b/api/memo_organizer.go @@ -19,3 +19,8 @@ type MemoOrganizerUpsert struct { UserID int Pinned bool `json:"pinned"` } + +type MemoOrganizerDelete struct { + MemoID *int + UserID *int +} diff --git a/api/memo_resource.go b/api/memo_resource.go index dc45fa5d..f2ceacf6 100644 --- a/api/memo_resource.go +++ b/api/memo_resource.go @@ -19,6 +19,6 @@ type MemoResourceFind struct { } type MemoResourceDelete struct { - MemoID int + MemoID *int ResourceID *int } diff --git a/api/resource.go b/api/resource.go index b00136cf..b833f913 100644 --- a/api/resource.go +++ b/api/resource.go @@ -40,18 +40,16 @@ type ResourceFind struct { MemoID *int } -type ResourceDelete struct { - ID int - - // Standard fields - CreatorID int -} - type ResourcePatch struct { ID int // Standard fields UpdatedTs *int64 + // Domain specific fields Filename *string `json:"filename"` } + +type ResourceDelete struct { + ID int +} diff --git a/api/shortcut.go b/api/shortcut.go index 7d596e06..b4aa803a 100644 --- a/api/shortcut.go +++ b/api/shortcut.go @@ -46,5 +46,8 @@ type ShortcutFind struct { } type ShortcutDelete struct { - ID int + ID *int + + // Standard fields + CreatorID *int } diff --git a/api/user_setting.go b/api/user_setting.go index ef0c405e..bf59b15a 100644 --- a/api/user_setting.go +++ b/api/user_setting.go @@ -156,3 +156,7 @@ type UserSettingFind struct { Key *UserSettingKey `json:"key"` } + +type UserSettingDelete struct { + UserID int +} diff --git a/server/memo.go b/server/memo.go index 7fb7ca8b..de9e82f2 100644 --- a/server/memo.go +++ b/server/memo.go @@ -500,7 +500,7 @@ func (s *Server) registerMemoRoutes(g *echo.Group) { } memoResourceDelete := &api.MemoResourceDelete{ - MemoID: memoID, + MemoID: &memoID, ResourceID: &resourceID, } if err := s.Store.DeleteMemoResource(ctx, memoResourceDelete); err != nil { diff --git a/server/resource.go b/server/resource.go index 25cff86a..06cd77a4 100644 --- a/server/resource.go +++ b/server/resource.go @@ -170,9 +170,19 @@ func (s *Server) registerResourceRoutes(g *echo.Group) { return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("ID is not a number: %s", c.Param("resourceId"))).SetInternal(err) } + resource, err := s.Store.FindResource(ctx, &api.ResourceFind{ + ID: &resourceID, + CreatorID: &userID, + }) + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, "Failed to find resource").SetInternal(err) + } + if resource == nil { + return echo.NewHTTPError(http.StatusNotFound, "Not find resource").SetInternal(err) + } + resourceDelete := &api.ResourceDelete{ - ID: resourceID, - CreatorID: userID, + ID: resourceID, } if err := s.Store.DeleteResource(ctx, resourceDelete); err != nil { if common.ErrorCode(err) == common.NotFound { diff --git a/server/shortcut.go b/server/shortcut.go index 7b88f418..11f4949b 100644 --- a/server/shortcut.go +++ b/server/shortcut.go @@ -128,7 +128,7 @@ func (s *Server) registerShortcutRoutes(g *echo.Group) { } shortcutDelete := &api.ShortcutDelete{ - ID: shortcutID, + ID: &shortcutID, } if err := s.Store.DeleteShortcut(ctx, shortcutDelete); err != nil { if common.ErrorCode(err) == common.NotFound { diff --git a/store/memo.go b/store/memo.go index a6c93161..1e44566b 100644 --- a/store/memo.go +++ b/store/memo.go @@ -221,13 +221,8 @@ func (s *Store) DeleteMemo(ctx context.Context, delete *api.MemoDelete) error { if err := deleteMemo(ctx, tx, delete); err != nil { return FormatError(err) } - - resourceDelete := &api.MemoResourceDelete{ - MemoID: delete.ID, - } - - if err := deleteMemoResource(ctx, tx, resourceDelete); err != nil { - return FormatError(err) + if err := vacuum(ctx, tx); err != nil { + return err } if err := tx.Commit(); err != nil { @@ -323,7 +318,7 @@ func findMemoRawList(ctx context.Context, tx *sql.Tx, find *api.MemoFind) ([]*me where, args = append(where, "row_status = ?"), append(args, *v) } if v := find.Pinned; v != nil { - where = append(where, "id in (SELECT memo_id FROM memo_organizer WHERE pinned = 1 AND user_id = memo.creator_id)") + where = append(where, "id IN (SELECT memo_id FROM memo_organizer WHERE pinned = 1 AND user_id = memo.creator_id)") } if v := find.ContentSearch; v != nil { where, args = append(where, "content LIKE ?"), append(args, "%"+*v+"%") @@ -382,16 +377,36 @@ func findMemoRawList(ctx context.Context, tx *sql.Tx, find *api.MemoFind) ([]*me } func deleteMemo(ctx context.Context, tx *sql.Tx, delete *api.MemoDelete) error { - result, err := tx.ExecContext(ctx, ` - DELETE FROM memo WHERE id = ? - `, delete.ID) + where, args := []string{"id = ?"}, []interface{}{delete.ID} + + stmt := `DELETE FROM memo WHERE ` + strings.Join(where, " AND ") + result, err := tx.ExecContext(ctx, stmt, args...) if err != nil { return FormatError(err) } rows, _ := result.RowsAffected() if rows == 0 { - return &common.Error{Code: common.NotFound, Err: fmt.Errorf("memo ID not found: %d", delete.ID)} + return &common.Error{Code: common.NotFound, Err: fmt.Errorf("memo not found")} + } + + return nil +} + +func vacuumMemo(ctx context.Context, tx *sql.Tx) error { + stmt := ` + DELETE FROM + memo + WHERE + creator_id NOT IN ( + SELECT + id + FROM + user + )` + _, err := tx.ExecContext(ctx, stmt) + if err != nil { + return FormatError(err) } return nil diff --git a/store/memo_organizer.go b/store/memo_organizer.go index fd657447..104dd2b7 100644 --- a/store/memo_organizer.go +++ b/store/memo_organizer.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "fmt" + "strings" "github.com/usememos/memos/api" "github.com/usememos/memos/common" @@ -65,6 +66,24 @@ func (s *Store) UpsertMemoOrganizer(ctx context.Context, upsert *api.MemoOrganiz return nil } +func (s *Store) DeleteMemoOrganizer(ctx context.Context, delete *api.MemoOrganizerDelete) error { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return FormatError(err) + } + defer tx.Rollback() + + if err := deleteMemoOrganizer(ctx, tx, delete); err != nil { + return err + } + + if err := tx.Commit(); err != nil { + return FormatError(err) + } + + return nil +} + func findMemoOrganizer(ctx context.Context, tx *sql.Tx, find *api.MemoOrganizerFind) (*memoOrganizerRaw, error) { query := ` SELECT @@ -127,3 +146,52 @@ func upsertMemoOrganizer(ctx context.Context, tx *sql.Tx, upsert *api.MemoOrgani return nil } + +func deleteMemoOrganizer(ctx context.Context, tx *sql.Tx, delete *api.MemoOrganizerDelete) error { + where, args := []string{}, []interface{}{} + + if v := delete.MemoID; v != nil { + where, args = append(where, "memo_id = ?"), append(args, *v) + } + if v := delete.UserID; v != nil { + where, args = append(where, "user_id = ?"), append(args, *v) + } + + stmt := `DELETE FROM memo_organizer WHERE ` + strings.Join(where, " AND ") + result, err := tx.ExecContext(ctx, stmt, args...) + if err != nil { + return FormatError(err) + } + + rows, _ := result.RowsAffected() + if rows == 0 { + return &common.Error{Code: common.NotFound, Err: fmt.Errorf("memo organizer not found")} + } + + return nil +} + +func vacuumMemoOrganizer(ctx context.Context, tx *sql.Tx) error { + stmt := ` + DELETE FROM + memo_organizer + WHERE + memo_id NOT IN ( + SELECT + id + FROM + memo + ) + OR user_id NOT IN ( + SELECT + id + FROM + user + )` + _, err := tx.ExecContext(ctx, stmt) + if err != nil { + return FormatError(err) + } + + return nil +} diff --git a/store/memo_resource.go b/store/memo_resource.go index b920eb65..5491ddc6 100644 --- a/store/memo_resource.go +++ b/store/memo_resource.go @@ -188,14 +188,17 @@ func upsertMemoResource(ctx context.Context, tx *sql.Tx, upsert *api.MemoResourc } func deleteMemoResource(ctx context.Context, tx *sql.Tx, delete *api.MemoResourceDelete) error { - where, args := []string{"memo_id = ?"}, []interface{}{delete.MemoID} + where, args := []string{}, []interface{}{} + if v := delete.MemoID; v != nil { + where, args = append(where, "memo_id = ?"), append(args, *v) + } if v := delete.ResourceID; v != nil { where, args = append(where, "resource_id = ?"), append(args, *v) } - result, err := tx.ExecContext(ctx, ` - DELETE FROM memo_resource WHERE `+strings.Join(where, " AND "), args...) + stmt := `DELETE FROM memo_resource WHERE ` + strings.Join(where, " AND ") + result, err := tx.ExecContext(ctx, stmt, args...) if err != nil { return FormatError(err) } @@ -207,3 +210,28 @@ func deleteMemoResource(ctx context.Context, tx *sql.Tx, delete *api.MemoResourc return nil } + +func vacuumMemoResource(ctx context.Context, tx *sql.Tx) error { + stmt := ` + DELETE FROM + memo_resource + WHERE + memo_id NOT IN ( + SELECT + id + FROM + memo + ) + OR resource_id NOT IN ( + SELECT + id + FROM + resource + )` + _, err := tx.ExecContext(ctx, stmt) + if err != nil { + return FormatError(err) + } + + return nil +} diff --git a/store/resource.go b/store/resource.go index 1bb3a356..eb8703b9 100644 --- a/store/resource.go +++ b/store/resource.go @@ -169,8 +169,10 @@ func (s *Store) DeleteResource(ctx context.Context, delete *api.ResourceDelete) } defer tx.Rollback() - err = deleteResource(ctx, tx, delete) - if err != nil { + if err := deleteResource(ctx, tx, delete); err != nil { + return err + } + if err := vacuum(ctx, tx); err != nil { return err } @@ -178,11 +180,6 @@ func (s *Store) DeleteResource(ctx context.Context, delete *api.ResourceDelete) return FormatError(err) } - // Vacuum sqlite database file size after deleting resource. - if _, err := s.db.Exec("VACUUM"); err != nil { - return err - } - s.cache.DeleteCache(api.ResourceCache, delete.ID) return nil @@ -340,16 +337,36 @@ func findResourceList(ctx context.Context, tx *sql.Tx, find *api.ResourceFind) ( } func deleteResource(ctx context.Context, tx *sql.Tx, delete *api.ResourceDelete) error { - result, err := tx.ExecContext(ctx, ` - DELETE FROM resource WHERE id = ? AND creator_id = ? - `, delete.ID, delete.CreatorID) + where, args := []string{"id = ?"}, []interface{}{delete.ID} + + stmt := `DELETE FROM resource WHERE ` + strings.Join(where, " AND ") + result, err := tx.ExecContext(ctx, stmt, args...) if err != nil { return FormatError(err) } rows, _ := result.RowsAffected() if rows == 0 { - return &common.Error{Code: common.NotFound, Err: fmt.Errorf("resource ID not found: %d", delete.ID)} + return &common.Error{Code: common.NotFound, Err: fmt.Errorf("resource not found")} + } + + return nil +} + +func vacuumResource(ctx context.Context, tx *sql.Tx) error { + stmt := ` + DELETE FROM + resource + WHERE + creator_id NOT IN ( + SELECT + id + FROM + user + )` + _, err := tx.ExecContext(ctx, stmt) + if err != nil { + return FormatError(err) } return nil diff --git a/store/shortcut.go b/store/shortcut.go index 43012bb2..05fa0d69 100644 --- a/store/shortcut.go +++ b/store/shortcut.go @@ -164,7 +164,7 @@ func (s *Store) DeleteShortcut(ctx context.Context, delete *api.ShortcutDelete) return FormatError(err) } - s.cache.DeleteCache(api.ShortcutCache, delete.ID) + s.cache.DeleteCache(api.ShortcutCache, *delete.ID) return nil } @@ -292,16 +292,43 @@ func findShortcutList(ctx context.Context, tx *sql.Tx, find *api.ShortcutFind) ( } func deleteShortcut(ctx context.Context, tx *sql.Tx, delete *api.ShortcutDelete) error { - result, err := tx.ExecContext(ctx, ` - DELETE FROM shortcut WHERE id = ? - `, delete.ID) + where, args := []string{}, []interface{}{} + + if v := delete.ID; v != nil { + where, args = append(where, "id = ?"), append(args, *v) + } + if v := delete.CreatorID; v != nil { + where, args = append(where, "creator_id = ?"), append(args, *v) + } + + stmt := `DELETE FROM shortcut WHERE ` + strings.Join(where, " AND ") + result, err := tx.ExecContext(ctx, stmt, args...) if err != nil { return FormatError(err) } rows, _ := result.RowsAffected() if rows == 0 { - return &common.Error{Code: common.NotFound, Err: fmt.Errorf("shortcut ID not found: %d", delete.ID)} + return &common.Error{Code: common.NotFound, Err: fmt.Errorf("shortcut not found")} + } + + return nil +} + +func vacuumShortcut(ctx context.Context, tx *sql.Tx) error { + stmt := ` + DELETE FROM + shortcut + WHERE + creator_id NOT IN ( + SELECT + id + FROM + user + )` + _, err := tx.ExecContext(ctx, stmt) + if err != nil { + return FormatError(err) } return nil diff --git a/store/store.go b/store/store.go index 736b295e..b3d6d585 100644 --- a/store/store.go +++ b/store/store.go @@ -1,6 +1,7 @@ package store import ( + "context" "database/sql" "github.com/usememos/memos/api" @@ -24,3 +25,51 @@ func New(db *sql.DB, profile *profile.Profile) *Store { cache: cacheService, } } + +func (s *Store) Vacuum(ctx context.Context) error { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return FormatError(err) + } + defer tx.Rollback() + + if err := vacuum(ctx, tx); err != nil { + return err + } + + if err := tx.Commit(); err != nil { + return FormatError(err) + } + + // Vacuum sqlite database file size after deleting resource. + if _, err := s.db.Exec("VACUUM"); err != nil { + return err + } + + return nil +} + +// Exec vacuum records in a transcation. +func vacuum(ctx context.Context, tx *sql.Tx) error { + if err := vacuumMemo(ctx, tx); err != nil { + return err + } + if err := vacuumResource(ctx, tx); err != nil { + return err + } + if err := vacuumShortcut(ctx, tx); err != nil { + return err + } + if err := vacuumUserSetting(ctx, tx); err != nil { + return err + } + if err := vacuumMemoOrganizer(ctx, tx); err != nil { + return err + } + if err := vacuumMemoResource(ctx, tx); err != nil { + // Prevent revive warning. + return err + } + + return nil +} diff --git a/store/user.go b/store/user.go index 3b4b5538..e18d656d 100644 --- a/store/user.go +++ b/store/user.go @@ -175,13 +175,15 @@ func (s *Store) DeleteUser(ctx context.Context, delete *api.UserDelete) error { } defer tx.Rollback() - err = deleteUser(ctx, tx, delete) - if err != nil { - return FormatError(err) + if err := deleteUser(ctx, tx, delete); err != nil { + return err + } + if err := vacuum(ctx, tx); err != nil { + return err } if err := tx.Commit(); err != nil { - return FormatError(err) + return err } s.cache.DeleteCache(api.UserCache, delete.ID) @@ -364,7 +366,7 @@ func deleteUser(ctx context.Context, tx *sql.Tx, delete *api.UserDelete) error { rows, _ := result.RowsAffected() if rows == 0 { - return &common.Error{Code: common.NotFound, Err: fmt.Errorf("user ID not found: %d", delete.ID)} + return &common.Error{Code: common.NotFound, Err: fmt.Errorf("user not found")} } return nil diff --git a/store/user_setting.go b/store/user_setting.go index cb8faadc..7e2a7f35 100644 --- a/store/user_setting.go +++ b/store/user_setting.go @@ -149,3 +149,22 @@ func findUserSettingList(ctx context.Context, tx *sql.Tx, find *api.UserSettingF return userSettingRawList, nil } + +func vacuumUserSetting(ctx context.Context, tx *sql.Tx) error { + stmt := ` + DELETE FROM + user_setting + WHERE + user_id NOT IN ( + SELECT + id + FROM + user + )` + _, err := tx.ExecContext(ctx, stmt) + if err != nil { + return FormatError(err) + } + + return nil +}