Update Status storage, add SanitizerService to fix spacing in html stripped content

2 months ago · 3686c92122
parent f5f7b3e678
commit 3686c92122
9 changed files with 98 additions and 53 deletions
--- a/app/Http/Controllers/Api/ApiV1Controller.php
+++ b/app/Http/Controllers/Api/ApiV1Controller.php
@ -58,6 +58,7 @@ use App\Services\NotificationService;
 use App\Services\PublicTimelineService;
 use App\Services\ReblogService;
 use App\Services\RelationshipService;
+use App\Services\SanitizeService;
 use App\Services\SnowflakeService;
 use App\Services\StatusService;
 use App\Services\UserFilterService;
@ -87,7 +88,6 @@ use Illuminate\Support\Str;
 use Laravel\Passport\Passport;
 use League\Fractal;
 use League\Fractal\Serializer\ArraySerializer;
-use Purify;
 use Storage;

 class ApiV1Controller extends Controller
@ -1964,7 +1964,7 @@ class ApiV1Controller extends Controller
            'media:update:'.$user->id,
            10,
            function () use ($media, $request) {
-                $caption = Purify::clean($request->input('description'));
+                $caption = app(SanitizeService::class)->html($request->input('description'));

                if ($caption != $media->caption) {
                    $media->caption = $caption;
--- a/app/Http/Controllers/Api/ApiV1Dot1Controller.php
+++ b/app/Http/Controllers/Api/ApiV1Dot1Controller.php
@ -32,6 +32,7 @@ use App\Services\NotificationAppGatewayService;
 use App\Services\ProfileStatusService;
 use App\Services\PublicTimelineService;
 use App\Services\PushNotificationService;
+use App\Services\SanitizeService;
 use App\Services\StatusService;
 use App\Services\UserStorageService;
 use App\Status;
@ -50,7 +51,6 @@ use Jenssegers\Agent\Agent;
 use League\Fractal;
 use League\Fractal\Serializer\ArraySerializer;
 use Mail;
-use Purify;

 class ApiV1Dot1Controller extends Controller
 {
@ -1294,7 +1294,8 @@ class ApiV1Dot1Controller extends Controller
            return [];
        }
        $defaultCaption = '';
-        $content = $request->filled('status') ? strip_tags(Purify::clean($request->input('status'))) : $defaultCaption;
+        $cleanedStatus = app(SanitizeService::class)->html($request->input('status', ''));
+        $content = $request->filled('status') ? strip_tags($cleanedStatus) : $defaultCaption;
        $cw = $user->profile->cw == true ? true : $request->boolean('sensitive', false);
        $spoilerText = $cw && $request->filled('spoiler_text') ? $request->input('spoiler_text') : null;

--- a/app/Http/Controllers/RemoteAuthController.php
+++ b/app/Http/Controllers/RemoteAuthController.php
@ -3,9 +3,11 @@
 namespace App\Http\Controllers;

 use App\Models\RemoteAuth;
+use App\Rules\PixelfedUsername;
 use App\Services\Account\RemoteAuthService;
 use App\Services\EmailService;
 use App\Services\MediaStorageService;
+use App\Services\SanitizeService;
 use App\User;
 use App\Util\ActivityPub\Helpers;
 use App\Util\Lexer\RestrictedNames;
@ -14,7 +16,6 @@ use Illuminate\Http\Request;
 use Illuminate\Support\Facades\Auth;
 use Illuminate\Support\Facades\Hash;
 use Illuminate\Support\Str;
-use App\Rules\PixelfedUsername;
 use InvalidArgumentException;
 use Purify;

@ -360,7 +361,7 @@ class RemoteAuthController extends Controller
                'required',
                'min:2',
                'max:30',
-                new PixelfedUsername(),
+                new PixelfedUsername,
            ],
        ]);
        $username = strtolower($request->input('username'));
@ -544,7 +545,7 @@ class RemoteAuthController extends Controller
        ]);

        $profile = $request->user()->profile;
-        $profile->bio = Purify::clean($request->input('bio'));
+        $profile->bio = app(SanitizeService::class)->html($request->input('bio'));
        $profile->save();

        return [200];
--- a/app/Jobs/ProfilePipeline/HandleUpdateActivity.php
+++ b/app/Jobs/ProfilePipeline/HandleUpdateActivity.php
@ -2,19 +2,17 @@

 namespace App\Jobs\ProfilePipeline;

+use App\Jobs\AvatarPipeline\RemoteAvatarFetchFromUrl;
+use App\Profile;
+use App\Services\SanitizeService;
+use App\Util\Lexer\Autolink;
+use Cache;
 use Illuminate\Bus\Queueable;
-use Illuminate\Contracts\Queue\ShouldBeUnique;
 use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
-use App\Avatar;
-use App\Profile;
-use App\Util\ActivityPub\Helpers;
-use Cache;
 use Purify;
-use App\Jobs\AvatarPipeline\RemoteAvatarFetchFromUrl;
-use App\Util\Lexer\Autolink;

 class HandleUpdateActivity implements ShouldQueue
 {
@ -34,61 +32,58 @@ class HandleUpdateActivity implements ShouldQueue

    /**
     * Execute the job.
-     *
-     * @return void
     */
    public function handle(): void
    {
        $payload = $this->payload;

-        if(empty($payload) || !isset($payload['actor'])) {
+        if (empty($payload) || ! isset($payload['actor'])) {
            return;
        }

        $profile = Profile::whereRemoteUrl($payload['actor'])->first();

-        if(!$profile || $profile->domain === null || $profile->private_key) {
+        if (! $profile || $profile->domain === null || $profile->private_key) {
            return;
        }

-        if($profile->sharedInbox == null || $profile->sharedInbox != $payload['object']['endpoints']['sharedInbox']) {
+        if ($profile->sharedInbox == null || $profile->sharedInbox != $payload['object']['endpoints']['sharedInbox']) {
            $profile->sharedInbox = $payload['object']['endpoints']['sharedInbox'];
        }

-        if($profile->public_key !== $payload['object']['publicKey']['publicKeyPem']) {
+        if ($profile->public_key !== $payload['object']['publicKey']['publicKeyPem']) {
            $profile->public_key = $payload['object']['publicKey']['publicKeyPem'];
        }

-        if($profile->bio !== $payload['object']['summary']) {
+        if ($profile->bio !== $payload['object']['summary']) {
            $len = strlen(strip_tags($payload['object']['summary']));
-            if($len) {
-                if($len > 500) {
+            if ($len) {
+                if ($len > 500) {
                    $updated = strip_tags($payload['object']['summary']);
                    $updated = substr($updated, 0, config('pixelfed.max_bio_length'));
                    $profile->bio = Autolink::create()->autolink($updated);
                } else {
-                    $profile->bio = Purify::clean($payload['object']['summary']);
+                    $profile->bio = app(SanitizeService::class)->html($payload['object']['summary']);
                }
            } else {
                $profile->bio = null;
            }
        }

-        if($profile->name !== $payload['object']['name']) {
+        if ($profile->name !== $payload['object']['name']) {
            $profile->name = Purify::clean(substr($payload['object']['name'], 0, config('pixelfed.max_name_length')));
        }

-        if($profile->isDirty()) {
+        if ($profile->isDirty()) {
            $profile->save();
        }

-        if(isset($payload['object']['icon']) && isset($payload['object']['icon']['url'])) {
+        if (isset($payload['object']['icon']) && isset($payload['object']['icon']['url'])) {
            RemoteAvatarFetchFromUrl::dispatch($profile, $payload['object']['icon']['url'])->onQueue('low');
        } else {
            $profile->avatar->update(['remote_url' => null]);
-            Cache::forget('avatar:' . $profile->id);
+            Cache::forget('avatar:'.$profile->id);
        }

-        return;
    }
 }
--- a/app/Jobs/RemoteFollowPipeline/RemoteFollowPipeline.php
+++ b/app/Jobs/RemoteFollowPipeline/RemoteFollowPipeline.php
@ -3,7 +3,8 @@
 namespace App\Jobs\RemoteFollowPipeline;

 use App\Jobs\AvatarPipeline\CreateAvatar;
-use App\{Profile};
+use App\Profile;
+use App\Services\SanitizeService;
 use GuzzleHttp\Client;
 use HttpSignatures\Context;
 use HttpSignatures\GuzzleHttpSignatures;
@ -19,7 +20,9 @@ class RemoteFollowPipeline implements ShouldQueue
    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;

    protected $url;
+
    protected $follower;
+
    protected $response;

    /**
@ -55,15 +58,15 @@ class RemoteFollowPipeline implements ShouldQueue
    public function discover($url)
    {
        $context = new Context([
-            'keys'      => ['examplekey' => 'secret-key-here'],
+            'keys' => ['examplekey' => 'secret-key-here'],
            'algorithm' => 'hmac-sha256',
-            'headers'   => ['(request-target)', 'date'],
+            'headers' => ['(request-target)', 'date'],
        ]);

        $handlerStack = GuzzleHttpSignatures::defaultHandlerFromContext($context);
        $client = new Client(['handler' => $handlerStack]);
        $response = Zttp::withHeaders([
-            'Accept'     => 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"',
+            'Accept' => 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"',
            'User-Agent' => 'PixelfedBot v0.1 - https://pixelfed.org',
        ])->get($url);
        $this->response = $response->json();
@ -78,12 +81,12 @@ class RemoteFollowPipeline implements ShouldQueue
        $username = $res['preferredUsername'];
        $remoteUsername = "@{$username}@{$domain}";

-        $profile = new Profile();
+        $profile = new Profile;
        $profile->user_id = null;
        $profile->domain = $domain;
        $profile->username = $remoteUsername;
        $profile->name = $res['name'];
-        $profile->bio = Purify::clean($res['summary']);
+        $profile->bio = app(SanitizeService::class)->html($res['summary']);
        $profile->sharedInbox = $res['endpoints']['sharedInbox'];
        $profile->remote_url = $res['url'];
        $profile->save();
@ -98,7 +101,7 @@ class RemoteFollowPipeline implements ShouldQueue
        $url = $res['inbox'];

        $activity = Zttp::withHeaders(['Content-Type' => 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'])->post($url, [
-            'type'   => 'Follow',
+            'type' => 'Follow',
            'object' => $this->follower->url(),
        ]);
    }
--- a/app/Jobs/StatusPipeline/StatusRemoteUpdatePipeline.php
+++ b/app/Jobs/StatusPipeline/StatusRemoteUpdatePipeline.php
@ -6,6 +6,7 @@ use App\Media;
 use App\Models\StatusEdit;
 use App\ModLog;
 use App\Profile;
+use App\Services\SanitizeService;
 use App\Services\StatusService;
 use App\Status;
 use Illuminate\Bus\Queueable;
@ -120,7 +121,8 @@ class StatusRemoteUpdatePipeline implements ShouldQueue
    protected function updateImmediateAttributes($status, $activity)
    {
        if (isset($activity['content'])) {
-            $status->caption = strip_tags(Purify::clean($activity['content']));
+            $cleanedCaption = app(SanitizeService::class)->html($activity['content']);
+            $status->caption = strip_tags($cleanedCaption);
        }

        if (isset($activity['sensitive'])) {
@ -143,7 +145,7 @@ class StatusRemoteUpdatePipeline implements ShouldQueue
        }

        if (isset($activity['summary'])) {
-            $status->cw_summary = Purify::clean($activity['summary']);
+            $status->cw_summary = app(SanitizeService::class)->html($activity['summary']);
        } else {
            $status->cw_summary = null;
        }
@ -155,8 +157,8 @@ class StatusRemoteUpdatePipeline implements ShouldQueue

    protected function createEdit($status, $activity)
    {
-        $cleaned = isset($activity['content']) ? Purify::clean($activity['content']) : null;
-        $spoiler_text = isset($activity['summary']) ? Purify::clean($activity['summary']) : null;
+        $cleaned = isset($activity['content']) ? app(SanitizeService::class)->html($activity['content']) : null;
+        $spoiler_text = isset($activity['summary']) ? app(SanitizeService::class)->html($activity['summary']) : null;
        $sensitive = isset($activity['sensitive']) ? $activity['sensitive'] : null;
        $mids = $status->media()->count() ? $status->media()->orderBy('order')->pluck('id')->toArray() : null;
        StatusEdit::create([
--- a/app/Services/SanitizeService.php
+++ b/app/Services/SanitizeService.php
@ -0,0 +1,38 @@
+<?php
+
+namespace App\Services;
+
+use Stevebauman\Purify\Facades\Purify;
+
+class SanitizeService
+{
+    public function purify($html)
+    {
+        $cleaned = Purify::clean($html);
+
+        return $cleaned;
+    }
+
+    public function html($html)
+    {
+        return $this->cleanHtmlWithSpacing($html);
+    }
+
+    public function cleanHtmlWithSpacing($html)
+    {
+        $blockTags = ['p', 'img', 'div', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'li', 'blockquote', 'br'];
+
+        foreach ($blockTags as $tag) {
+            $html = preg_replace("/<\/{$tag}>/i", "</{$tag}> ", $html);
+        }
+
+        $html = preg_replace("/<br\s*\/?>/i", '<br /> ', $html);
+
+        $cleaned = Purify::clean($html);
+
+        $cleaned = preg_replace('/\s+/', ' ', $cleaned);
+        $cleaned = trim($cleaned);
+
+        return $cleaned;
+    }
+}
--- a/app/Util/ActivityPub/Helpers.php
+++ b/app/Util/ActivityPub/Helpers.php
@ -19,6 +19,7 @@ use App\Services\DomainService;
 use App\Services\InstanceService;
 use App\Services\MediaPathService;
 use App\Services\NetworkTimelineService;
+use App\Services\SanitizeService;
 use App\Services\UserFilterService;
 use App\Status;
 use App\Util\Media\License;
@ -175,7 +176,7 @@ class Helpers
                return false;
            }

-            if (!$disableDNSCheck && ! self::passesSecurityChecks($host, $disableDNSCheck, $forceBanCheck)) {
+            if (! $disableDNSCheck && ! self::passesSecurityChecks($host, $disableDNSCheck, $forceBanCheck)) {
                return false;
            }

@ -666,8 +667,11 @@ class Helpers
        bool $commentsDisabled
    ): Status {
        $caption = isset($activity['content']) ?
-            Purify::clean($activity['content']) :
+            app(SanitizeService::class)->html($activity['content']) :
            '';
+        $cwSummary = ($cw && isset($activity['summary'])) ?
+            app(SanitizeService::class)->html($activity['summary']) :
+            null;

        return Status::updateOrCreate(
            ['uri' => $url],
@ -683,9 +687,7 @@ class Helpers
                'is_nsfw' => $cw,
                'scope' => $scope,
                'visibility' => $scope,
-                'cw_summary' => ($cw && isset($activity['summary'])) ?
-                    Purify::clean(strip_tags($activity['summary'])) :
-                    null,
+                'cw_summary' => $cwSummary ? strip_tags($cwSummary) : null,
                'comments_disabled' => $commentsDisabled,
            ]
        );
@ -823,12 +825,15 @@ class Helpers
        })->toArray();

        $defaultCaption = '';
+        $cleanedCaption = ! empty($res['content']) ?
+            app(SanitizeService::class)->html($res['content']) :
+            null;
        $status = new Status;
        $status->profile_id = $profile->id;
        $status->url = isset($res['url']) ? $res['url'] : $url;
        $status->uri = isset($res['url']) ? $res['url'] : $url;
        $status->object_url = $id;
-        $status->caption = strip_tags(Purify::clean($res['content'])) ?? $defaultCaption;
+        $status->caption = $cleanedCaption ? strip_tags($cleanedCaption) : $defaultCaption;
        $status->rendered = Purify::clean($res['content'] ?? $defaultCaption);
        $status->created_at = Carbon::parse($ts)->tz('UTC');
        $status->in_reply_to_id = null;
@ -1261,7 +1266,7 @@ class Helpers
            'key_id' => $res['publicKey']['id'],
            'remote_url' => $res['id'],
            'name' => isset($res['name']) ? Purify::clean($res['name']) : 'user',
-            'bio' => isset($res['summary']) ? Purify::clean($res['summary']) : null,
+            'bio' => isset($res['summary']) ? app(SanitizeService::class)->html($res['summary']) : null,
            'sharedInbox' => $res['endpoints']['sharedInbox'] ?? null,
            'inbox_url' => $res['inbox'],
            'outbox_url' => $res['outbox'] ?? null,
--- a/app/Util/ActivityPub/Inbox.php
+++ b/app/Util/ActivityPub/Inbox.php
@ -33,6 +33,7 @@ use App\Services\PollService;
 use App\Services\PushNotificationService;
 use App\Services\ReblogService;
 use App\Services\RelationshipService;
+use App\Services\SanitizeService;
 use App\Services\StoryIndexService;
 use App\Services\UserFilterService;
 use App\Status;
@ -50,7 +51,6 @@ use Cache;
 use Illuminate\Support\Facades\Bus;
 use Illuminate\Support\Facades\Log;
 use Illuminate\Support\Str;
-use Purify;
 use Storage;
 use Throwable;

@ -423,7 +423,7 @@ class Inbox
            return;
        }

-        $msg = Purify::clean($activity['content']);
+        $msg = app(SanitizeService::class)->html($activity['content']);
        $msgText = strip_tags($msg);

        if (Str::startsWith($msgText, '@'.$profile->username)) {
@ -1064,7 +1064,7 @@ class Inbox
        $actor = $this->payload['actor'];
        $storyUrl = $this->payload['inReplyTo'];
        $to = $this->payload['to'];
-        $text = Purify::clean($this->payload['content']);
+        $text = app(SanitizeService::class)->html($this->payload['content']);

        if (parse_url($id, PHP_URL_HOST) !== parse_url($actor, PHP_URL_HOST)) {
            return;
@ -1184,7 +1184,7 @@ class Inbox
        $actor = $this->payload['actor'];
        $storyUrl = $this->payload['inReplyTo'];
        $to = $this->payload['to'];
-        $text = Purify::clean($this->payload['content']);
+        $text = app(SanitizeService::class)->html($this->payload['content']);

        if (parse_url($id, PHP_URL_HOST) !== parse_url($actor, PHP_URL_HOST)) {
            return;
@ -1310,9 +1310,9 @@ class Inbox
        $content = null;
        if (isset($this->payload['content'])) {
            if (strlen($this->payload['content']) > 5000) {
-                $content = Purify::clean(substr($this->payload['content'], 0, 5000).' ... (truncated message due to exceeding max length)');
+                $content = app(SanitizeService::class)->html(substr($this->payload['content'], 0, 5000).' ... (truncated message due to exceeding max length)');
            } else {
-                $content = Purify::clean($this->payload['content']);
+                $content = app(SanitizeService::class)->html($this->payload['content']);
            }
        }
        $object = $this->payload['object'];