From ddc768871b907ebd3f605ff9b0c8628671a9596b Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Tue, 16 Feb 2021 23:39:39 -0700 Subject: [PATCH 1/3] Update federation pipeline, add locks --- app/Jobs/InboxPipeline/InboxValidator.php | 9 +++ app/Jobs/InboxPipeline/InboxWorker.php | 9 +++ app/Util/ActivityPub/Helpers.php | 97 +++++++++++++---------- 3 files changed, 75 insertions(+), 40 deletions(-) diff --git a/app/Jobs/InboxPipeline/InboxValidator.php b/app/Jobs/InboxPipeline/InboxValidator.php index bfcb4d6d0..4b85a1bd0 100644 --- a/app/Jobs/InboxPipeline/InboxValidator.php +++ b/app/Jobs/InboxPipeline/InboxValidator.php @@ -53,6 +53,15 @@ class InboxValidator implements ShouldQueue $profile = Profile::whereNull('domain')->whereUsername($username)->first(); + if(isset($payload['id'])) { + $lockKey = hash('sha256', $payload['id']); + if(Cache::get($lockKey) !== null) { + // Job processed already + return 1; + } + Cache::put($lockKey, 1, 300); + } + if(!isset($headers['signature']) || !isset($headers['date'])) { return; } diff --git a/app/Jobs/InboxPipeline/InboxWorker.php b/app/Jobs/InboxPipeline/InboxWorker.php index acc72f16f..ad3a085b5 100644 --- a/app/Jobs/InboxPipeline/InboxWorker.php +++ b/app/Jobs/InboxPipeline/InboxWorker.php @@ -49,6 +49,15 @@ class InboxWorker implements ShouldQueue $headers = $this->headers; $payload = json_decode($this->payload, true, 8); + if(isset($payload['id'])) { + $lockKey = hash('sha256', $payload['id']); + if(Cache::get($lockKey) !== null) { + // Job processed already + return 1; + } + Cache::put($lockKey, 1, 300); + } + if(!isset($headers['signature']) || !isset($headers['date'])) { return; } diff --git a/app/Util/ActivityPub/Helpers.php b/app/Util/ActivityPub/Helpers.php index 1f127eb99..583cd8dc0 100644 --- a/app/Util/ActivityPub/Helpers.php +++ b/app/Util/ActivityPub/Helpers.php @@ -346,27 +346,41 @@ class Helpers { $reply_to = null; } $ts = is_array($res['published']) ? $res['published'][0] : $res['published']; - $status = DB::transaction(function() use($profile, $res, $url, $ts, $reply_to, $cw, $scope, $id) { - $status = new Status; - $status->profile_id = $profile->id; - $status->url = isset($res['url']) ? $res['url'] : $url; - $status->uri = isset($res['url']) ? $res['url'] : $url; - $status->object_url = $id; - $status->caption = strip_tags($res['content']); - $status->rendered = Purify::clean($res['content']); - $status->created_at = Carbon::parse($ts); - $status->in_reply_to_id = $reply_to; - $status->local = false; - $status->is_nsfw = $cw; - $status->scope = $scope; - $status->visibility = $scope; - $status->cw_summary = $cw == true && isset($res['summary']) ? - Purify::clean(strip_tags($res['summary'])) : null; - $status->save(); - if($reply_to == null) { - self::importNoteAttachment($res, $status); - } - return $status; + + $statusLockKey = 'helpers:status-lock:' . hash('sha256', $res['id']); + $status = Cache::lock($statusLockKey) + ->get(function () use( + $profile, + $res, + $url, + $ts, + $reply_to, + $cw, + $scope, + $id + ) { + return DB::transaction(function() use($profile, $res, $url, $ts, $reply_to, $cw, $scope, $id) { + $status = new Status; + $status->profile_id = $profile->id; + $status->url = isset($res['url']) ? $res['url'] : $url; + $status->uri = isset($res['url']) ? $res['url'] : $url; + $status->object_url = $id; + $status->caption = strip_tags($res['content']); + $status->rendered = Purify::clean($res['content']); + $status->created_at = Carbon::parse($ts); + $status->in_reply_to_id = $reply_to; + $status->local = false; + $status->is_nsfw = $cw; + $status->scope = $scope; + $status->visibility = $scope; + $status->cw_summary = $cw == true && isset($res['summary']) ? + Purify::clean(strip_tags($res['summary'])) : null; + $status->save(); + if($reply_to == null) { + self::importNoteAttachment($res, $status); + } + return $status; + }); }); return $status; @@ -458,25 +472,28 @@ class Helpers { $profile = Profile::whereRemoteUrl($res['id'])->first(); if(!$profile) { - $profile = DB::transaction(function() use($domain, $webfinger, $res, $runJobs) { - $profile = new Profile(); - $profile->domain = strtolower($domain); - $profile->username = strtolower(Purify::clean($webfinger)); - $profile->name = isset($res['name']) ? Purify::clean($res['name']) : 'user'; - $profile->bio = isset($res['summary']) ? Purify::clean($res['summary']) : null; - $profile->sharedInbox = isset($res['endpoints']) && isset($res['endpoints']['sharedInbox']) ? $res['endpoints']['sharedInbox'] : null; - $profile->inbox_url = strtolower($res['inbox']); - $profile->outbox_url = strtolower($res['outbox']); - $profile->remote_url = strtolower($res['id']); - $profile->public_key = $res['publicKey']['publicKeyPem']; - $profile->key_id = $res['publicKey']['id']; - $profile->webfinger = strtolower(Purify::clean($webfinger)); - $profile->last_fetched_at = now(); - $profile->save(); - if(config('pixelfed.cloud_storage') == true) { - RemoteAvatarFetch::dispatch($profile); - } - return $profile; + $profileLockKey = 'helpers:profile-lock:' . hash('sha256', $res['id']); + $profile = Cache::lock($profileLockKey)->get(function () use($domain, $webfinger, $res, $runJobs) { + return DB::transaction(function() use($domain, $webfinger, $res, $runJobs) { + $profile = new Profile(); + $profile->domain = strtolower($domain); + $profile->username = strtolower(Purify::clean($webfinger)); + $profile->name = isset($res['name']) ? Purify::clean($res['name']) : 'user'; + $profile->bio = isset($res['summary']) ? Purify::clean($res['summary']) : null; + $profile->sharedInbox = isset($res['endpoints']) && isset($res['endpoints']['sharedInbox']) ? $res['endpoints']['sharedInbox'] : null; + $profile->inbox_url = strtolower($res['inbox']); + $profile->outbox_url = strtolower($res['outbox']); + $profile->remote_url = strtolower($res['id']); + $profile->public_key = $res['publicKey']['publicKeyPem']; + $profile->key_id = $res['publicKey']['id']; + $profile->webfinger = strtolower(Purify::clean($webfinger)); + $profile->last_fetched_at = now(); + $profile->save(); + if(config('pixelfed.cloud_storage') == true) { + RemoteAvatarFetch::dispatch($profile); + } + return $profile; + }); }); } else { // Update info after 24 hours From acbca90c782f2e6fc0804d72e0b7041108dfc5b5 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Tue, 16 Feb 2021 23:56:54 -0700 Subject: [PATCH 2/3] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d4a9e2fc..3f9fb045a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,7 @@ - Updated InboxPipeline, fail earlier for invalid public keys. Fixes ([#2648](https://github.com/pixelfed/pixelfed/issues/2648)). ([d1c5e9b8](https://github.com/pixelfed/pixelfed/commit/d1c5e9b8)) - Updated Status model, refactor liked and shared methods to fix cache invalidation bug. ([f05c3b66](https://github.com/pixelfed/pixelfed/commit/f05c3b66)) - Updated Timeline component, add inline reports modal. ([e64b4bd3](https://github.com/pixelfed/pixelfed/commit/e64b4bd3)) +- Updated federation pipeline, add locks. ([ddc76887](https://github.com/pixelfed/pixelfed/commit/ddc76887)) - ([](https://github.com/pixelfed/pixelfed/commit/)) ## [v0.10.10 (2021-01-28)](https://github.com/pixelfed/pixelfed/compare/v0.10.9...v0.10.10) From 850e4499f143bfd6807c84364171977a28518a07 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Tue, 16 Feb 2021 23:59:10 -0700 Subject: [PATCH 3/3] Update web routes --- routes/web.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routes/web.php b/routes/web.php index e3a50260a..3606ae93d 100644 --- a/routes/web.php +++ b/routes/web.php @@ -244,13 +244,13 @@ Route::domain(config('pixelfed.domain.app'))->middleware(['validemail', 'twofact Route::group(['prefix' => 'i'], function () { Route::redirect('/', '/'); Route::get('compose', 'StatusController@compose')->name('compose'); - Route::post('comment', 'CommentController@store')->middleware('throttle:maxCommentsPerHour,60')->middleware('throttle:maxCommentsPerDay,1440'); + Route::post('comment', 'CommentController@store')->middleware('throttle:maxCommentsPerDay,1440'); Route::post('delete', 'StatusController@delete'); Route::post('mute', 'AccountController@mute'); Route::post('unmute', 'AccountController@unmute'); Route::post('block', 'AccountController@block'); Route::post('unblock', 'AccountController@unblock'); - Route::post('like', 'LikeController@store')->middleware('throttle:maxLikesPerHour,60')->middleware('throttle:maxLikesPerDay,1440'); + Route::post('like', 'LikeController@store')->middleware('throttle:maxLikesPerDay,1440'); Route::post('share', 'StatusController@storeShare')->middleware('throttle:maxSharesPerHour,60')->middleware('throttle:maxSharesPerDay,1440'); Route::post('follow', 'FollowerController@store'); Route::post('bookmark', 'BookmarkController@store');