fix: patch memo visibility (#1695)

server/memo.go

@@ -163,6 +163,7 @@ func (s *Server) registerMemoRoutes(g *echo.Group) {
 			return echo.NewHTTPError(http.StatusInternalServerError, "Failed to compose memo").SetInternal(err)
 		}
 
+		if memoPatch.ResourceIDList != nil {
 			resourceIDList := make([]int, 0)
 			for _, resource := range memo.ResourceList {
 				resourceIDList = append(resourceIDList, resource.ID)
@@ -184,7 +185,9 @@ func (s *Server) registerMemoRoutes(g *echo.Group) {
 					return echo.NewHTTPError(http.StatusInternalServerError, "Failed to delete memo resource").SetInternal(err)
 				}
 			}
+		}
 
+		if memoPatch.RelationList != nil {
 			patchMemoRelationList := make([]*api.MemoRelation, 0)
 			for _, memoRelationUpsert := range memoPatch.RelationList {
 				patchMemoRelationList = append(patchMemoRelationList, &api.MemoRelation{
@@ -213,6 +216,7 @@ func (s *Server) registerMemoRoutes(g *echo.Group) {
 					return echo.NewHTTPError(http.StatusInternalServerError, "Failed to delete memo relation").SetInternal(err)
 				}
 			}
+		}
 
 		// After patching memo resources and relations, we need to re-compose it to get the latest data.
 		memo, err = s.Store.ComposeMemo(ctx, memo)