Merge pull request #3686 from pixelfed/staging

Staging

CHANGELOG.md

@@ -4,6 +4,8 @@
 
 ### Updates
 - Update ApiV1Controller, include self likes in favourited_by endpoint ([58b331d2](https://github.com/pixelfed/pixelfed/commit/58b331d2))
+- Update PublicApiController, remove expensive and unused relationships ([2ecc3144](https://github.com/pixelfed/pixelfed/commit/2ecc3144))
+- Update status deletion, fix database lock issues and side effects ([04e8c96a](https://github.com/pixelfed/pixelfed/commit/04e8c96a))
 -  ([](https://github.com/pixelfed/pixelfed/commit/))
 
 ## [v0.11.4 (2022-10-04)](https://github.com/pixelfed/pixelfed/compare/v0.11.3...v0.11.4)

app/Http/Controllers/Api/ApiV1Controller.php

@@ -3109,7 +3109,7 @@ class ApiV1Controller extends Controller
 		});
 
 		$ids = $ids->map(function($profile) {
-			return AccountService::getMastodon($profile->id, true);
+			return AccountService::get($profile->id, true);
 		})
 		->filter(function($profile) use($pid) {
 			return $profile && isset($profile['id']);

app/Http/Controllers/Api/BaseApiController.php

@@ -90,7 +90,7 @@ class BaseApiController extends Controller
 
         if(empty($res) && !Cache::has('pf:services:notifications:hasSynced:'.$pid)) {
         	Cache::put('pf:services:notifications:hasSynced:'.$pid, 1, 1209600);
-        	NotificationService::warmCache($pid, 400, true);
+        	NotificationService::warmCache($pid, 100, true);
         }
 
         return response()->json($res);

app/Http/Controllers/StatusController.php

@@ -225,7 +225,7 @@ class StatusController extends Controller
 		StatusService::del($status->id, true);
 		if ($status->profile_id == $user->profile->id || $user->is_admin == true) {
 			Cache::forget('profile:status_count:'.$status->profile_id);
-			StatusDelete::dispatch($status);
+			StatusDelete::dispatchNow($status);
 		}
 
 		if($request->wantsJson()) {

app/Jobs/DeletePipeline/DeleteRemoteStatusPipeline.php

@@ -0,0 +1,74 @@
+<?php
+
+namespace App\Jobs\DeletePipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use App\Bookmark;
+use App\DirectMessage;
+use App\Like;
+use App\Media;
+use App\MediaTag;
+use App\Mention;
+use App\Report;
+use App\Status;
+use App\StatusHashtag;
+use App\StatusView;
+use App\Notification;
+use App\Services\NetworkTimelineService;
+use App\Services\StatusService;
+use App\Jobs\ProfilePipeline\DecrementPostCount;
+use App\Jobs\MediaPipeline\MediaDeletePipeline;
+
+class DeleteRemoteStatusPipeline implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $status;
+
+    /**
+     * Create a new job instance.
+     *
+     * @return void
+     */
+    public function __construct(Status $status)
+    {
+        $this->status = $status->withoutRelations();
+    }
+
+    /**
+     * Execute the job.
+     *
+     * @return void
+     */
+    public function handle()
+    {
+        $status = $this->status;
+
+        NetworkTimelineService::del($status->id);
+        StatusService::del($status->id, true);
+        DecrementPostCount::dispatchNow($status->profile_id);
+        Bookmark::whereStatusId($status->id)->delete();
+        Notification::whereItemType('App\Status')
+            ->whereItemId($status->id)
+            ->forceDelete();
+        DirectMessage::whereStatusId($status->id)->delete();
+        Like::whereStatusId($status->id)->forceDelete();
+        MediaTag::whereStatusId($status->id)->delete();
+        Media::whereStatusId($status->id)
+            ->get()
+            ->each(function($media) {
+                MediaDeletePipeline::dispatchNow($media);
+            });
+        Mention::whereStatusId($status->id)->forceDelete();
+        Report::whereObjectType('App\Status')->whereObjectId($status->id)->delete();
+        StatusHashtag::whereStatusId($status->id)->delete();
+        StatusView::whereStatusId($status->id)->delete();
+        Status::whereReblogOfId($status->id)->forceDelete();
+        $status->delete();
+    }
+}

app/Jobs/StatusPipeline/StatusDelete.php

@@ -5,12 +5,19 @@ namespace App\Jobs\StatusPipeline;
 use DB, Storage;
 use App\{
 	AccountInterstitial,
+    Bookmark,
 	CollectionItem,
+    DirectMessage,
+    Like,
+    Media,
 	MediaTag,
+    Mention,
 	Notification,
 	Report,
 	Status,
+    StatusArchived,
 	StatusHashtag,
+    StatusView
 };
 use Illuminate\Bus\Queueable;
 use Illuminate\Contracts\Queue\ShouldQueue;
@@ -28,7 +35,7 @@ use GuzzleHttp\Promise;
 use App\Util\ActivityPub\HttpSignature;
 use App\Services\CollectionService;
 use App\Services\StatusService;
-use App\Services\MediaStorageService;
+use App\Jobs\MediaPipeline\MediaDeletePipeline;
 
 class StatusDelete implements ShouldQueue
 {
@@ -71,75 +78,65 @@ class StatusDelete implements ShouldQueue
 		}
 
 		if(config_cache('federation.activitypub.enabled') == true) {
-			$this->fanoutDelete($status);
+			return $this->fanoutDelete($status);
 		} else {
-			$this->unlinkRemoveMedia($status);
+			return $this->unlinkRemoveMedia($status);
 		}
-
 	}
 
 	public function unlinkRemoveMedia($status)
 	{
-		foreach ($status->media as $media) {
-			MediaStorageService::delete($media, true);
-		}
+        Media::whereStatusId($status->id)
+        ->get()
+        ->each(function($media) {
+            MediaDeletePipeline::dispatchNow($media);
+        });
 
 		if($status->in_reply_to_id) {
-			DB::transaction(function() use($status) {
-				$parent = Status::findOrFail($status->in_reply_to_id);
-				--$parent->reply_count;
-				$parent->save();
-			});
+			$parent = Status::findOrFail($status->in_reply_to_id);
+			--$parent->reply_count;
+			$parent->save();
 		}
 
-        DB::transaction(function() use($status) {
-            CollectionItem::whereObjectType('App\Status')
-                ->whereObjectId($status->id)
-                ->get()
-                ->each(function($col) {
-                    $id = $col->collection_id;
-                    $sid = $col->object_id;
-                    $col->delete();
-                    CollectionService::removeItem($id, $sid);
-                });
+        Bookmark::whereStatusId($status->id)->delete();
+
+        CollectionItem::whereObjectType('App\Status')
+            ->whereObjectId($status->id)
+            ->get()
+            ->each(function($col) {
+                CollectionService::removeItem($col->collection_id, $col->object_id);
+                $col->delete();
         });
 
-		DB::transaction(function() use($status) {
-			$comments = Status::where('in_reply_to_id', $status->id)->get();
-			foreach ($comments as $comment) {
-				$comment->in_reply_to_id = null;
-				$comment->save();
-				Notification::whereItemType('App\Status')
-					->whereItemId($comment->id)
-					->delete();
-			}
-			$status->likes()->delete();
-			Notification::whereItemType('App\Status')
-				->whereItemId($status->id)
-				->delete();
-			StatusHashtag::whereStatusId($status->id)->delete();
-			Report::whereObjectType('App\Status')
-				->whereObjectId($status->id)
-				->delete();
-			MediaTag::where('status_id', $status->id)
-				->cursor()
-				->each(function($tag) {
-					Notification::where('item_type', 'App\MediaTag')
-						->where('item_id', $tag->id)
-						->forceDelete();
-					$tag->delete();
-			});
-			AccountInterstitial::where('item_type', 'App\Status')
-				->where('item_id', $status->id)
-				->delete();
-
-			$status->forceDelete();
-		});
-
-		return true;
+        DirectMessage::whereStatusId($status->id)->delete();
+        Like::whereStatusId($status->id)->delete();
+
+		MediaTag::where('status_id', $status->id)->delete();
+        Mention::whereStatusId($status->id)->forceDelete();
+
+		Notification::whereItemType('App\Status')
+			->whereItemId($status->id)
+			->forceDelete();
+
+		Report::whereObjectType('App\Status')
+			->whereObjectId($status->id)
+			->delete();
+
+        StatusArchived::whereStatusId($status->id)->delete();
+        StatusHashtag::whereStatusId($status->id)->delete();
+        StatusView::whereStatusId($status->id)->delete();
+		Status::whereInReplyToId($status->id)->update(['in_reply_to_id' => null]);
+
+		AccountInterstitial::where('item_type', 'App\Status')
+			->where('item_id', $status->id)
+			->delete();
+
+		$status->forceDelete();
+
+		return 1;
 	}
 
-	protected function fanoutDelete($status)
+	public function fanoutDelete($status)
 	{
 		$audience = $status->profile->getAudienceInbox();
 		$profile = $status->profile;
@@ -189,5 +186,6 @@ class StatusDelete implements ShouldQueue
 
 		$promise->wait();
 
+        return 1;
 	}
 }

app/Util/ActivityPub/Inbox.php

@@ -24,6 +24,7 @@ use Illuminate\Support\Str;
 use App\Jobs\LikePipeline\LikePipeline;
 use App\Jobs\FollowPipeline\FollowPipeline;
 use App\Jobs\DeletePipeline\DeleteRemoteProfilePipeline;
+use App\Jobs\DeletePipeline\DeleteRemoteStatusPipeline;
 use App\Jobs\StoryPipeline\StoryExpire;
 use App\Jobs\StoryPipeline\StoryFetch;
 
@@ -622,7 +623,7 @@ class Inbox
 			if(!$profile || $profile->private_key != null) {
 				return;
 			}
-			DeleteRemoteProfilePipeline::dispatchNow($profile);
+			DeleteRemoteProfilePipeline::dispatch($profile)->onQueue('delete');
 			return;
 		} else {
 			if(!isset($obj['id'], $this->payload['object'], $this->payload['object']['id'])) {
@@ -643,7 +644,7 @@ class Inbox
 						if(!$profile || $profile->private_key != null) {
 							return;
 						}
-						DeleteRemoteProfilePipeline::dispatchNow($profile);
+						DeleteRemoteProfilePipeline::dispatch($profile)->onQueue('delete');
 						return;
 					break;
 
@@ -660,18 +661,7 @@ class Inbox
 						if(!$status) {
 							return;
 						}
-						NetworkTimelineService::del($status->id);
-						StatusService::del($status->id, true);
-						Notification::whereActorId($profile->id)
-							->whereItemType('App\Status')
-							->whereItemId($status->id)
-							->forceDelete();
-						$status->directMessage()->delete();
-						$status->media()->delete();
-						$status->likes()->delete();
-						$status->shares()->delete();
-						$status->delete();
-                        DecrementPostCount::dispatch($profile->id)->onQueue('low');
+						DeleteRemoteStatusPipeline::dispatch($status)->onQueue('delete');
 						return;
 					break;
 

database/migrations/2022_10_07_072311_add_status_id_index_to_bookmarks_table.php

@@ -0,0 +1,31 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+class AddStatusIdIndexToBookmarksTable extends Migration
+{
+    /**
+     * Run the migrations.
+     *
+     * @return void
+     */
+    public function up()
+    {
+        Schema::table('bookmarks', function (Blueprint $table) {
+            $table->index('status_id');
+            $table->index('profile_id');
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     *
+     * @return void
+     */
+    public function down()
+    {
+        //
+    }
+}

database/migrations/2022_10_07_072555_add_status_id_index_to_direct_messages_table.php

@@ -0,0 +1,31 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+class AddStatusIdIndexToDirectMessagesTable extends Migration
+{
+    /**
+     * Run the migrations.
+     *
+     * @return void
+     */
+    public function up()
+    {
+        Schema::table('direct_messages', function (Blueprint $table) {
+            $table->index('status_id');
+            $table->index('group_message');
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     *
+     * @return void
+     */
+    public function down()
+    {
+        //
+    }
+}

database/migrations/2022_10_07_072859_add_status_id_index_to_mentions_table.php

@@ -0,0 +1,31 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+class AddStatusIdIndexToMentionsTable extends Migration
+{
+    /**
+     * Run the migrations.
+     *
+     * @return void
+     */
+    public function up()
+    {
+        Schema::table('mentions', function (Blueprint $table) {
+            $table->index('status_id');
+            $table->index('profile_id');
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     *
+     * @return void
+     */
+    public function down()
+    {
+        //
+    }
+}

database/migrations/2022_10_07_073337_add_indexes_to_reports_table.php

@@ -0,0 +1,33 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+class AddIndexesToReportsTable extends Migration
+{
+    /**
+     * Run the migrations.
+     *
+     * @return void
+     */
+    public function up()
+    {
+        Schema::table('reports', function (Blueprint $table) {
+            $table->index('user_id');
+            $table->index('profile_id');
+            $table->index('object_id');
+            $table->index('object_type');
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     *
+     * @return void
+     */
+    public function down()
+    {
+        //
+    }
+}