Merge branch 'staging' into dev

.editorconfig

@@ -1,8 +1,8 @@
 root = true
 
 [*]
+indent_style = space
 indent_size = 4
-indent_style = tab
 end_of_line = lf
 charset = utf-8
 trim_trailing_whitespace = true

CHANGELOG.md

@@ -4,8 +4,13 @@
 
 ### Added
 - Resilient Media Storage ([#4665](https://github.com/pixelfed/pixelfed/pull/4665)) ([fb1deb6](https://github.com/pixelfed/pixelfed/commit/fb1deb6))
+- Video WebP2P ([#4713](https://github.com/pixelfed/pixelfed/pull/4713)) ([0405ef12](https://github.com/pixelfed/pixelfed/commit/0405ef12))
 - Added user:2fa command to easily disable 2FA for given account ([c6408fd7](https://github.com/pixelfed/pixelfed/commit/c6408fd7))
 - Added `avatar:storage-deep-clean` command to dispatch remote avatar storage cleanup jobs ([c37b7cde](https://github.com/pixelfed/pixelfed/commit/c37b7cde))
+- Added S3 command to rewrite media urls ([5b3a5610](https://github.com/pixelfed/pixelfed/commit/5b3a5610))
+- Experimental home feed ([#4752](https://github.com/pixelfed/pixelfed/pull/4752)) ([c39b9afb](https://github.com/pixelfed/pixelfed/commit/c39b9afb))
+- Added `app:hashtag-cached-count-update` command to update cached_count of hashtags and add to scheduler to run every 25 minutes past the hour ([1e31fee6](https://github.com/pixelfed/pixelfed/commit/1e31fee6))
+- Added `app:hashtag-related-generate` command to generate related hashtags ([176b4ed7](https://github.com/pixelfed/pixelfed/commit/176b4ed7))
 
 ### Federation
 - Update Privacy Settings, add support for Mastodon `indexable` search flag ([fc24630e](https://github.com/pixelfed/pixelfed/commit/fc24630e))
@@ -34,6 +39,30 @@
 - Update CreateAvatar job, add processing constraints and set `is_remote` attribute ([319ced40](https://github.com/pixelfed/pixelfed/commit/319ced40))
 - Update RemoteStatusDelete and DecrementPostCount pipelines ([edbcf3ed](https://github.com/pixelfed/pixelfed/commit/edbcf3ed))
 - Update lexer regex, fix mention regex and add more tests ([778e83d3](https://github.com/pixelfed/pixelfed/commit/778e83d3))
+- Update StatusTransformer, generate autolink on request ([dfe2379b](https://github.com/pixelfed/pixelfed/commit/dfe2379b))
+- Update ComposeModal component, fix multi filter bug and allow media re-ordering before upload/posting ([56e315f6](https://github.com/pixelfed/pixelfed/commit/56e315f6))
+- Update ApiV1Dot1Controller, allow iar rate limits to be configurable ([28a80803](https://github.com/pixelfed/pixelfed/commit/28a80803))
+- Update ApiV1Dot1Controller, add domain to iar redirect ([1f82d47c](https://github.com/pixelfed/pixelfed/commit/1f82d47c))
+- Update ApiV1Dot1Controller, add configurable app confirm rate limit ttl ([4c6a0719](https://github.com/pixelfed/pixelfed/commit/4c6a0719))
+- Update LikePipeline, dispatch to feed queue. Fixes ([#4723](https://github.com/pixelfed/pixelfed/issues/4723)) ([da510089](https://github.com/pixelfed/pixelfed/commit/da510089))
+- Update AccountImport ([5a2d7e3e](https://github.com/pixelfed/pixelfed/commit/5a2d7e3e))
+- Update ImportPostController, fix IG bug with missing spaces between hashtags ([9c24157a](https://github.com/pixelfed/pixelfed/commit/9c24157a))
+- Update ApiV1Controller, fix mutes in home feed ([ddc21714](https://github.com/pixelfed/pixelfed/commit/ddc21714))
+- Update AP helpers, improve preferredUsername validation ([21218c79](https://github.com/pixelfed/pixelfed/commit/21218c79))
+- Update delete pipelines, properly invoke StatusHashtag delete events ([ce54d29c](https://github.com/pixelfed/pixelfed/commit/ce54d29c))
+- Update mail config ([0e431271](https://github.com/pixelfed/pixelfed/commit/0e431271))
+- Update hashtag following ([015b1b80](https://github.com/pixelfed/pixelfed/commit/015b1b80))
+- Update IncrementPostCount job, prevent overlap ([b2c9cc23](https://github.com/pixelfed/pixelfed/commit/b2c9cc23))
+- Update HashtagFollowService, fix cache invalidation bug ([84f4e885](https://github.com/pixelfed/pixelfed/commit/84f4e885))
+- Update Experimental Home Feed, fix remote posts, shares and reblogs ([c6a6b3ae](https://github.com/pixelfed/pixelfed/commit/c6a6b3ae))
+- Update HashtagService, improve count perf ([3327a008](https://github.com/pixelfed/pixelfed/commit/3327a008))
+- Update StatusHashtagService, remove problematic cache layer ([e5401f85](https://github.com/pixelfed/pixelfed/commit/e5401f85))
+- Update HomeFeedPipeline, fix tag filtering ([f105f4e8](https://github.com/pixelfed/pixelfed/commit/f105f4e8))
+- Update HashtagService, reduce cached_count cache ttl ([15f29f7d](https://github.com/pixelfed/pixelfed/commit/15f29f7d))
+- Update ApiV1Controller, fix include_reblogs param on timelines/home endpoint, and improve limit pagination logic ([287f903b](https://github.com/pixelfed/pixelfed/commit/287f903b))
+- Update StoryApiV1Controller, add self-carousel endpoint. Fixes ([#4352](https://github.com/pixelfed/pixelfed/issues/4352)) ([bcb88d5b](https://github.com/pixelfed/pixelfed/commit/bcb88d5b))
+- Update FollowServiceWarmCache, use more efficient query ([fe9b4c5a](https://github.com/pixelfed/pixelfed/commit/fe9b4c5a))
+- Update HomeFeedPipeline, observe mutes/blocks during fanout ([8548294c](https://github.com/pixelfed/pixelfed/commit/8548294c))
 -  ([](https://github.com/pixelfed/pixelfed/commit/))
 
 ## [v0.11.9 (2023-08-21)](https://github.com/pixelfed/pixelfed/compare/v0.11.8...v0.11.9)

app/Console/Commands/HashtagCachedCountUpdate.php

@@ -0,0 +1,57 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+use App\Hashtag;
+use App\StatusHashtag;
+use DB;
+
+class HashtagCachedCountUpdate extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'app:hashtag-cached-count-update {--limit=100}';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Update cached counter of hashtags';
+
+    /**
+     * Execute the console command.
+     */
+    public function handle()
+    {
+        $limit = $this->option('limit');
+        $tags = Hashtag::whereNull('cached_count')->limit($limit)->get();
+        $count = count($tags);
+        if(!$count) {
+            return;
+        }
+
+        $bar = $this->output->createProgressBar($count);
+        $bar->start();
+
+        foreach($tags as $tag) {
+            $count = DB::table('status_hashtags')->whereHashtagId($tag->id)->count();
+            if(!$count) {
+                $tag->cached_count = 0;
+                $tag->saveQuietly();
+                $bar->advance();
+                continue;
+            }
+            $tag->cached_count = $count;
+            $tag->saveQuietly();
+            $bar->advance();
+        }
+        $bar->finish();
+        $this->line(' ');
+        return;
+    }
+}

app/Console/Commands/HashtagRelatedGenerate.php

@@ -0,0 +1,94 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+use App\Hashtag;
+use App\StatusHashtag;
+use App\Models\HashtagRelated;
+use App\Services\HashtagRelatedService;
+use Illuminate\Contracts\Console\PromptsForMissingInput;
+use function Laravel\Prompts\multiselect;
+use function Laravel\Prompts\confirm;
+
+class HashtagRelatedGenerate extends Command implements PromptsForMissingInput
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'app:hashtag-related-generate {tag}';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Command description';
+
+    /**
+     * Prompt for missing input arguments using the returned questions.
+     *
+     * @return array
+     */
+    protected function promptForMissingArgumentsUsing()
+    {
+        return [
+            'tag' => 'Which hashtag should we generate related tags for?',
+        ];
+    }
+
+    /**
+     * Execute the console command.
+     */
+    public function handle()
+    {
+        $tag = $this->argument('tag');
+        $hashtag = Hashtag::whereName($tag)->orWhere('slug', $tag)->first();
+        if(!$hashtag) {
+            $this->error('Hashtag not found, aborting...');
+            exit;
+        }
+
+        $exists = HashtagRelated::whereHashtagId($hashtag->id)->exists();
+
+        if($exists) {
+            $confirmed = confirm('Found existing related tags, do you want to regenerate them?');
+            if(!$confirmed) {
+                $this->error('Aborting...');
+                exit;
+            }
+        }
+
+        $this->info('Looking up #' . $tag . '...');
+
+        $tags = StatusHashtag::whereHashtagId($hashtag->id)->count();
+        if(!$tags || $tags < 100) {
+            $this->error('Not enough posts found to generate related hashtags!');
+            exit;
+        }
+
+        $this->info('Found ' . $tags . ' posts that use that hashtag');
+        $related = collect(HashtagRelatedService::fetchRelatedTags($tag));
+
+        $selected = multiselect(
+            label: 'Which tags do you want to generate?',
+            options: $related->pluck('name'),
+            required: true,
+        );
+
+        $filtered = $related->filter(fn($i) => in_array($i['name'], $selected))->all();
+        $agg_score = $related->filter(fn($i) => in_array($i['name'], $selected))->sum('related_count');
+
+        HashtagRelated::updateOrCreate([
+            'hashtag_id' => $hashtag->id,
+        ], [
+            'related_tags' => array_values($filtered),
+            'agg_score' => $agg_score,
+            'last_calculated_at' => now()
+        ]);
+
+        $this->info('Finished!');
+    }
+}

app/Console/Commands/MediaCloudUrlRewrite.php

@@ -0,0 +1,140 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+use App\Media;
+use Cache, Storage;
+use Illuminate\Contracts\Console\PromptsForMissingInput;
+
+class MediaCloudUrlRewrite extends Command implements PromptsForMissingInput
+{
+    /**
+    * The name and signature of the console command.
+    *
+    * @var string
+    */
+    protected $signature = 'media:cloud-url-rewrite {oldDomain} {newDomain}';
+
+    /**
+    * Prompt for missing input arguments using the returned questions.
+    *
+    * @return array
+    */
+    protected function promptForMissingArgumentsUsing()
+    {
+        return [
+            'oldDomain' => 'The old S3 domain',
+            'newDomain' => 'The new S3 domain'
+        ];
+    }
+    /**
+    * The console command description.
+    *
+    * @var string
+    */
+    protected $description = 'Rewrite S3 media urls from local users';
+
+    /**
+    * Execute the console command.
+    */
+    public function handle()
+    {
+        $this->preflightCheck();
+        $this->bootMessage();
+        $this->confirmCloudUrl();
+    }
+
+    protected function preflightCheck()
+    {
+        if(config_cache('pixelfed.cloud_storage') != true) {
+            $this->info('Error: Cloud storage is not enabled!');
+            $this->error('Aborting...');
+            exit;
+        }
+    }
+
+    protected function bootMessage()
+    {
+        $this->info('       ____  _           ______         __  ');
+        $this->info('      / __ \(_)  _____  / / __/__  ____/ /  ');
+        $this->info('     / /_/ / / |/_/ _ \/ / /_/ _ \/ __  /   ');
+        $this->info('    / ____/ />  </  __/ / __/  __/ /_/ /    ');
+        $this->info('   /_/   /_/_/|_|\___/_/_/  \___/\__,_/     ');
+        $this->info(' ');
+        $this->info('    Media Cloud Url Rewrite Tool');
+        $this->info('    ===');
+        $this->info('    Old S3: ' . trim($this->argument('oldDomain')));
+        $this->info('    New S3: ' . trim($this->argument('newDomain')));
+        $this->info(' ');
+    }
+
+    protected function confirmCloudUrl()
+    {
+        $disk = Storage::disk(config('filesystems.cloud'))->url('test');
+        $domain = parse_url($disk, PHP_URL_HOST);
+        if(trim($this->argument('newDomain')) !== $domain) {
+            $this->error('Error: The new S3 domain you entered is not currently configured');
+            exit;
+        }
+
+        if(!$this->confirm('Confirm this is correct')) {
+            $this->error('Aborting...');
+            exit;
+        }
+
+        $this->updateUrls();
+    }
+
+    protected function updateUrls()
+    {
+        $this->info('Updating urls...');
+        $oldDomain = trim($this->argument('oldDomain'));
+        $newDomain = trim($this->argument('newDomain'));
+        $disk = Storage::disk(config('filesystems.cloud'));
+        $count = Media::whereNotNull('cdn_url')->count();
+        $bar = $this->output->createProgressBar($count);
+        $counter = 0;
+        $bar->start();
+        foreach(Media::whereNotNull('cdn_url')->lazyById(1000, 'id') as $media) {
+            if(strncmp($media->media_path, 'http', 4) === 0) {
+                $bar->advance();
+                continue;
+            }
+            $cdnHost = parse_url($media->cdn_url, PHP_URL_HOST);
+            if($oldDomain != $cdnHost || $newDomain == $cdnHost) {
+                $bar->advance();
+                continue;
+            }
+
+            $media->cdn_url = str_replace($oldDomain, $newDomain, $media->cdn_url);
+
+            if($media->thumbnail_url != null) {
+                $thumbHost = parse_url($media->thumbnail_url, PHP_URL_HOST);
+                if($thumbHost == $oldDomain) {
+                    $thumbUrl = $disk->url($media->thumbnail_path);
+                    $media->thumbnail_url = $thumbUrl;
+                }
+            }
+
+            if($media->optimized_url != null) {
+                $optiHost = parse_url($media->optimized_url, PHP_URL_HOST);
+                if($optiHost == $oldDomain) {
+                    $optiUrl = str_replace($oldDomain, $newDomain, $media->optimized_url);
+                    $media->optimized_url = $optiUrl;
+                }
+            }
+
+            $media->save();
+            $counter++;
+            $bar->advance();
+        }
+
+        $bar->finish();
+
+        $this->line(' ');
+        $this->info('Finished! Updated ' . $counter . ' total records!');
+        $this->line(' ');
+        $this->info('Tip: Run `php artisan cache:clear` to purge cached urls');
+    }
+}

app/Console/Commands/NotificationEpochUpdate.php

@@ -0,0 +1,31 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+use App\Jobs\InternalPipeline\NotificationEpochUpdatePipeline;
+
+class NotificationEpochUpdate extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'app:notification-epoch-update';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Update notification epoch';
+
+    /**
+     * Execute the console command.
+     */
+    public function handle()
+    {
+        NotificationEpochUpdatePipeline::dispatch();
+    }
+}

app/Console/Kernel.php

@@ -43,6 +43,8 @@ class Kernel extends ConsoleKernel
             $schedule->command('app:import-remove-deleted-accounts')->hourlyAt(37);
             $schedule->command('app:import-upload-clean-storage')->twiceDailyAt(1, 13, 32);
         }
+        $schedule->command('app:notification-epoch-update')->weeklyOn(1, '2:21');
+        $schedule->command('app:hashtag-cached-count-update')->hourlyAt(25);
     }
 
     /**

app/Http/Controllers/Api/ApiV1Dot1Controller.php

@@ -11,6 +11,7 @@ use League\Fractal\Serializer\ArraySerializer;
 use League\Fractal\Pagination\IlluminatePaginatorAdapter;
 use App\AccountLog;
 use App\EmailVerification;
+use App\Follower;
 use App\Place;
 use App\Status;
 use App\Report;
@@ -21,6 +22,8 @@ use App\UserSetting;
 use App\Services\AccountService;
 use App\Services\StatusService;
 use App\Services\ProfileStatusService;
+use App\Services\LikeService;
+use App\Services\ReblogService;
 use App\Services\PublicTimelineService;
 use App\Services\NetworkTimelineService;
 use App\Util\Lexer\RestrictedNames;
@@ -470,7 +473,7 @@ class ApiV1Dot1Controller extends Controller
 			abort_if(BouncerService::checkIp($request->ip()), 404);
 		}
 
-		$rl = RateLimiter::attempt('pf:apiv1.1:iar:'.$request->ip(), 3, function(){}, 1800);
+		$rl = RateLimiter::attempt('pf:apiv1.1:iar:'.$request->ip(), config('pixelfed.app_registration_rate_limit_attempts', 3), function(){}, config('pixelfed.app_registration_rate_limit_decay', 1800));
 		abort_if(!$rl, 400, 'Too many requests');
 
 		$this->validate($request, [
@@ -543,10 +546,10 @@ class ApiV1Dot1Controller extends Controller
 		$user->password = Hash::make($password);
 		$user->register_source = 'app';
 		$user->app_register_ip = $request->ip();
-		$user->app_register_token = Str::random(32);
+		$user->app_register_token = Str::random(40);
 		$user->save();
 
-		$rtoken = Str::random(mt_rand(64, 70));
+		$rtoken = Str::random(64);
 
 		$verify = new EmailVerification();
 		$verify->user_id = $user->id;
@@ -555,7 +558,12 @@ class ApiV1Dot1Controller extends Controller
 		$verify->random_token = $rtoken;
 		$verify->save();
 
-		$appUrl = url('/api/v1.1/auth/iarer?ut=' . $user->app_register_token . '&rt=' . $rtoken);
+		$params = http_build_query([
+			'ut' => $user->app_register_token,
+			'rt' => $rtoken,
+			'ea' => base64_encode($user->email)
+		]);
+		$appUrl = url('/api/v1.1/auth/iarer?'. $params);
 
 		Mail::to($user->email)->send(new ConfirmAppEmail($verify, $appUrl));
 
@@ -568,14 +576,19 @@ class ApiV1Dot1Controller extends Controller
 	{
 		$this->validate($request, [
 			'ut' => 'required',
-			'rt' => 'required'
+			'rt' => 'required',
+			'ea' => 'required'
 		]);
-		if(config('pixelfed.bouncer.cloud_ips.ban_signups')) {
-			abort_if(BouncerService::checkIp($request->ip()), 404);
-		}
 		$ut = $request->input('ut');
 		$rt = $request->input('rt');
-		$url = 'pixelfed://confirm-account/'. $ut . '?rt=' . $rt;
+		$ea = $request->input('ea');
+		$params = http_build_query([
+			'ut' => $ut,
+			'rt' => $rt,
+			'domain' => config('pixelfed.domain.app'),
+			'ea' => $ea
+		]);
+		$url = 'pixelfed://confirm-account/'. $ut . '?' . $params;
 		return redirect()->away($url);
 	}
 
@@ -589,8 +602,8 @@ class ApiV1Dot1Controller extends Controller
 			abort_if(BouncerService::checkIp($request->ip()), 404);
 		}
 
-		$rl = RateLimiter::attempt('pf:apiv1.1:iarc:'.$request->ip(), 10, function(){}, 1800);
+		$rl = RateLimiter::attempt('pf:apiv1.1:iarc:'.$request->ip(), config('pixelfed.app_registration_confirm_rate_limit_attempts', 20), function(){}, config('pixelfed.app_registration_confirm_rate_limit_decay', 1800));
-		abort_if(!$rl, 400, 'Too many requests');
+		abort_if(!$rl, 429, 'Too many requests');
 
 		$this->validate($request, [
 			'user_token' => 'required',

app/Http/Controllers/Api/V1/TagsController.php

@@ -0,0 +1,207 @@
+<?php
+
+namespace App\Http\Controllers\Api\V1;
+
+use Illuminate\Http\Request;
+use App\Http\Controllers\Controller;
+use App\Hashtag;
+use App\HashtagFollow;
+use App\StatusHashtag;
+use App\Services\AccountService;
+use App\Services\HashtagService;
+use App\Services\HashtagFollowService;
+use App\Services\HashtagRelatedService;
+use App\Http\Resources\MastoApi\FollowedTagResource;
+use App\Jobs\HomeFeedPipeline\FeedWarmCachePipeline;
+use App\Jobs\HomeFeedPipeline\HashtagUnfollowPipeline;
+
+class TagsController extends Controller
+{
+    const PF_API_ENTITY_KEY = "_pe";
+
+    public function json($res, $code = 200, $headers = [])
+    {
+        return response()->json($res, $code, $headers, JSON_UNESCAPED_SLASHES);
+    }
+
+    /**
+    * GET /api/v1/tags/:id/related
+    *
+    *
+    * @return array
+    */
+    public function relatedTags(Request $request, $tag)
+    {
+        abort_unless($request->user(), 403);
+        $tag = Hashtag::whereSlug($tag)->firstOrFail();
+        return HashtagRelatedService::get($tag->id);
+    }
+
+    /**
+    * POST /api/v1/tags/:id/follow
+    *
+    *
+    * @return object
+    */
+    public function followHashtag(Request $request, $id)
+    {
+        abort_if(!$request->user(), 403);
+
+        $pid = $request->user()->profile_id;
+        $account = AccountService::get($pid);
+
+        $operator = config('database.default') == 'pgsql' ? 'ilike' : 'like';
+        $tag = Hashtag::where('name', $operator, $id)
+            ->orWhere('slug', $operator, $id)
+            ->first();
+
+        abort_if(!$tag, 422, 'Unknown hashtag');
+
+        abort_if(
+            HashtagFollow::whereProfileId($pid)->count() >= HashtagFollow::MAX_LIMIT,
+            422,
+            'You cannot follow more than ' . HashtagFollow::MAX_LIMIT . ' hashtags.'
+        );
+
+        $follows = HashtagFollow::updateOrCreate(
+            [
+                'profile_id' => $account['id'],
+                'hashtag_id' => $tag->id
+            ],
+            [
+                'user_id' => $request->user()->id
+            ]
+        );
+
+        HashtagService::follow($pid, $tag->id);
+        HashtagFollowService::add($tag->id, $pid);
+
+        return response()->json(FollowedTagResource::make($follows)->toArray($request));
+    }
+
+    /**
+    * POST /api/v1/tags/:id/unfollow
+    *
+    *
+    * @return object
+    */
+    public function unfollowHashtag(Request $request, $id)
+    {
+        abort_if(!$request->user(), 403);
+
+        $pid = $request->user()->profile_id;
+        $account = AccountService::get($pid);
+
+        $operator = config('database.default') == 'pgsql' ? 'ilike' : 'like';
+        $tag = Hashtag::where('name', $operator, $id)
+            ->orWhere('slug', $operator, $id)
+            ->first();
+
+        abort_if(!$tag, 422, 'Unknown hashtag');
+
+        $follows = HashtagFollow::whereProfileId($pid)
+            ->whereHashtagId($tag->id)
+            ->first();
+
+        if(!$follows) {
+            return [
+                'name' => $tag->name,
+                'url' => config('app.url') . '/i/web/hashtag/' . $tag->slug,
+                'history' => [],
+                'following' => false
+            ];
+        }
+
+        if($follows) {
+            HashtagService::unfollow($pid, $tag->id);
+            HashtagFollowService::unfollow($tag->id, $pid);
+            HashtagUnfollowPipeline::dispatch($tag->id, $pid, $tag->slug)->onQueue('feed');
+            $follows->delete();
+        }
+
+        $res = FollowedTagResource::make($follows)->toArray($request);
+        $res['following'] = false;
+        return response()->json($res);
+    }
+
+    /**
+    * GET /api/v1/tags/:id
+    *
+    *
+    * @return object
+    */
+    public function getHashtag(Request $request, $id)
+    {
+        abort_if(!$request->user(), 403);
+
+        $pid = $request->user()->profile_id;
+        $account = AccountService::get($pid);
+        $operator = config('database.default') == 'pgsql' ? 'ilike' : 'like';
+        $tag = Hashtag::where('name', $operator, $id)
+            ->orWhere('slug', $operator, $id)
+            ->first();
+
+        if(!$tag) {
+            return [
+                'name' => $id,
+                'url' => config('app.url') . '/i/web/hashtag/' . $id,
+                'history' => [],
+                'following' => false
+            ];
+        }
+
+        $res = [
+            'name' => $tag->name,
+            'url' => config('app.url') . '/i/web/hashtag/' . $tag->slug,
+            'history' => [],
+            'following' => HashtagService::isFollowing($pid, $tag->id)
+        ];
+
+        if($request->has(self::PF_API_ENTITY_KEY)) {
+            $res['count'] = HashtagService::count($tag->id);
+        }
+
+        return $this->json($res);
+    }
+
+    /**
+    * GET /api/v1/followed_tags
+    *
+    *
+    * @return array
+    */
+    public function getFollowedTags(Request $request)
+    {
+        abort_if(!$request->user(), 403);
+
+        $account = AccountService::get($request->user()->profile_id);
+
+        $this->validate($request, [
+            'cursor' => 'sometimes',
+            'limit' => 'sometimes|integer|min:1|max:200'
+        ]);
+        $limit = $request->input('limit', 100);
+
+        $res = HashtagFollow::whereProfileId($account['id'])
+            ->orderByDesc('id')
+            ->cursorPaginate($limit)
+            ->withQueryString();
+
+        $pagination = false;
+        $prevPage = $res->nextPageUrl();
+        $nextPage = $res->previousPageUrl();
+        if($nextPage && $prevPage) {
+            $pagination = '<' . $nextPage . '>; rel="next", <' . $prevPage . '>; rel="prev"';
+        } else if($nextPage && !$prevPage) {
+            $pagination = '<' . $nextPage . '>; rel="next"';
+        } else if(!$nextPage && $prevPage) {
+            $pagination = '<' . $prevPage . '>; rel="prev"';
+        }
+
+        if($pagination) {
+            return response()->json(FollowedTagResource::collection($res)->collection)
+                ->header('Link', $pagination);
+        }
+        return response()->json(FollowedTagResource::collection($res)->collection);
+    }
+}

app/Http/Controllers/ImportPostController.php

@@ -83,6 +83,17 @@ class ImportPostController extends Controller
         );
     }
 
+    public function formatHashtags($val = false)
+    {
+        if(!$val || !strlen($val)) {
+            return null;
+        }
+
+        $groupedHashtagRegex = '/#\w+(?=#)/';
+
+        return preg_replace($groupedHashtagRegex, '$0 ', $val);
+    }
+
     public function store(Request $request)
     {
         abort_unless(config('import.instagram.enabled'), 404);
@@ -128,11 +139,11 @@ class ImportPostController extends Controller
             $ip->media = $c->map(function($m) {
                 return [
                     'uri' => $m['uri'],
-                    'title' => $m['title'],
+                    'title' => $this->formatHashtags($m['title']),
                     'creation_timestamp' => $m['creation_timestamp']
                 ];
             })->toArray();
-            $ip->caption = $c->count() > 1 ? $file['title'] : $ip->media[0]['title'];
+            $ip->caption = $c->count() > 1 ? $this->formatHashtags($file['title']) : $this->formatHashtags($ip->media[0]['title']);
             $ip->filename = last(explode('/', $ip->media[0]['uri']));
             $ip->metadata = $c->map(function($m) {
                 return [

app/Http/Controllers/LikeController.php

@@ -25,8 +25,7 @@ class LikeController extends Controller
 			'item'    => 'required|integer|min:1',
 		]);
 
-		// API deprecated
+		abort(422, 'Deprecated API Endpoint');
-		return;
 
 		$user = Auth::user();
 		$profile = $user->profile;
@@ -34,7 +33,7 @@ class LikeController extends Controller
 
 		if (Like::whereStatusId($status->id)->whereProfileId($profile->id)->exists()) {
 			$like = Like::whereProfileId($profile->id)->whereStatusId($status->id)->firstOrFail();
-			UnlikePipeline::dispatch($like);
+			UnlikePipeline::dispatch($like)->onQueue('feed');
 		} else {
 			abort_if(
 				Like::whereProfileId($user->profile_id)
@@ -60,7 +59,7 @@ class LikeController extends Controller
 					]) == false;
 				$like->save();
 				$status->save();
-				LikePipeline::dispatch($like);
+				LikePipeline::dispatch($like)->onQueue('feed');
 			}
 		}
 

app/Http/Controllers/Stories/StoryApiV1Controller.php

@@ -24,358 +24,482 @@ use App\Http\Resources\StoryView as StoryViewResource;
 
 class StoryApiV1Controller extends Controller
 {
-	public function carousel(Request $request)
+    const RECENT_KEY = 'pf:stories:recent-by-id:';
-	{
+    const RECENT_TTL = 300;
-		abort_if(!config_cache('instance.stories.enabled') || !$request->user(), 404);
+
-		$pid = $request->user()->profile_id;
+    public function carousel(Request $request)
-
+    {
-		if(config('database.default') == 'pgsql') {
+        abort_if(!config_cache('instance.stories.enabled') || !$request->user(), 404);
-			$s = Story::select('stories.*', 'followers.following_id')
+        $pid = $request->user()->profile_id;
-				->leftJoin('followers', 'followers.following_id', 'stories.profile_id')
+
-				->where('followers.profile_id', $pid)
+        if(config('database.default') == 'pgsql') {
-				->where('stories.active', true)
+            $s = Cache::remember(self::RECENT_KEY . $pid, self::RECENT_TTL, function() use($pid) {
-				->get();
+                return Story::select('stories.*', 'followers.following_id')
-		} else {
+                    ->leftJoin('followers', 'followers.following_id', 'stories.profile_id')
-			$s = Story::select('stories.*', 'followers.following_id')
+                    ->where('followers.profile_id', $pid)
-				->leftJoin('followers', 'followers.following_id', 'stories.profile_id')
+                    ->where('stories.active', true)
-				->where('followers.profile_id', $pid)
+                    ->map(function($s) {
-				->where('stories.active', true)
+                        $r  = new \StdClass;
-				->orderBy('id')
+                        $r->id = $s->id;
-				->get();
+                        $r->profile_id = $s->profile_id;
-		}
+                        $r->type = $s->type;
-
+                        $r->path = $s->path;
-		$nodes = $s->map(function($s) use($pid) {
+                        return $r;
-			$profile = AccountService::get($s->profile_id, true);
+                    })
-			if(!$profile || !isset($profile['id'])) {
+                    ->unique('profile_id');
-				return false;
+            });
-			}
+        } else {
-
+            $s = Cache::remember(self::RECENT_KEY . $pid, self::RECENT_TTL, function() use($pid) {
-			return [
+                return Story::select('stories.*', 'followers.following_id')
-				'id' => (string) $s->id,
+                    ->leftJoin('followers', 'followers.following_id', 'stories.profile_id')
-				'pid' => (string) $s->profile_id,
+                    ->where('followers.profile_id', $pid)
-				'type' => $s->type,
+                    ->where('stories.active', true)
-				'src' => url(Storage::url($s->path)),
+                    ->orderBy('id')
-				'duration' => $s->duration ?? 3,
+                    ->get();
-				'seen' => StoryService::hasSeen($pid, $s->id),
+            });
-				'created_at' => $s->created_at->format('c')
+        }
-			];
+
-		})
+        $nodes = $s->map(function($s) use($pid) {
-		->filter()
+            $profile = AccountService::get($s->profile_id, true);
-		->groupBy('pid')
+            if(!$profile || !isset($profile['id'])) {
-		->map(function($item) use($pid) {
+                return false;
-			$profile = AccountService::get($item[0]['pid'], true);
+            }
-			$url = $profile['local'] ? url("/stories/{$profile['username']}") :
+
-				url("/i/rs/{$profile['id']}");
+            return [
-			return [
+                'id' => (string) $s->id,
-				'id' => 'pfs:' . $profile['id'],
+                'pid' => (string) $s->profile_id,
-				'user' => [
+                'type' => $s->type,
-					'id' => (string) $profile['id'],
+                'src' => url(Storage::url($s->path)),
-					'username' => $profile['username'],
+                'duration' => $s->duration ?? 3,
-					'username_acct' => $profile['acct'],
+                'seen' => StoryService::hasSeen($pid, $s->id),
-					'avatar' => $profile['avatar'],
+                'created_at' => $s->created_at->format('c')
-					'local' => $profile['local'],
+            ];
-					'is_author' => $profile['id'] == $pid
+        })
-				],
+        ->filter()
-				'nodes' => $item,
+        ->groupBy('pid')
-				'url' => $url,
+        ->map(function($item) use($pid) {
-				'seen' => StoryService::hasSeen($pid, StoryService::latest($profile['id'])),
+            $profile = AccountService::get($item[0]['pid'], true);
-			];
+            $url = $profile['local'] ? url("/stories/{$profile['username']}") :
-		})
+                url("/i/rs/{$profile['id']}");
-		->sortBy('seen')
+            return [
-		->values();
+                'id' => 'pfs:' . $profile['id'],
-
+                'user' => [
-		$res = [
+                    'id' => (string) $profile['id'],
-			'self' => [],
+                    'username' => $profile['username'],
-			'nodes' => $nodes,
+                    'username_acct' => $profile['acct'],
-		];
+                    'avatar' => $profile['avatar'],
-
+                    'local' => $profile['local'],
-		if(Story::whereProfileId($pid)->whereActive(true)->exists()) {
+                    'is_author' => $profile['id'] == $pid
-			$selfStories = Story::whereProfileId($pid)
+                ],
-				->whereActive(true)
+                'nodes' => $item,
-				->get()
+                'url' => $url,
-				->map(function($s) use($pid) {
+                'seen' => StoryService::hasSeen($pid, StoryService::latest($profile['id'])),
-					return [
+            ];
-						'id' => (string) $s->id,
+        })
-						'type' => $s->type,
+        ->sortBy('seen')
-						'src' => url(Storage::url($s->path)),
+        ->values();
-						'duration' => $s->duration,
+
-						'seen' => true,
+        $res = [
-						'created_at' => $s->created_at->format('c')
+            'self' => [],
-					];
+            'nodes' => $nodes,
-				})
+        ];
-				->sortBy('id')
+
-				->values();
+        if(Story::whereProfileId($pid)->whereActive(true)->exists()) {
-			$selfProfile = AccountService::get($pid, true);
+            $selfStories = Story::whereProfileId($pid)
-			$res['self'] = [
+                ->whereActive(true)
-				'user' => [
+                ->get()
-					'id' => (string) $selfProfile['id'],
+                ->map(function($s) use($pid) {
-					'username' => $selfProfile['acct'],
+                    return [
-					'avatar' => $selfProfile['avatar'],
+                        'id' => (string) $s->id,
-					'local' => $selfProfile['local'],
+                        'type' => $s->type,
-					'is_author' => true
+                        'src' => url(Storage::url($s->path)),
-				],
+                        'duration' => $s->duration,
-
+                        'seen' => true,
-				'nodes' => $selfStories,
+                        'created_at' => $s->created_at->format('c')
-			];
+                    ];
-		}
+                })
-		return response()->json($res, 200, [], JSON_PRETTY_PRINT|JSON_UNESCAPED_SLASHES);
+                ->sortBy('id')
-	}
+                ->values();
-
+            $selfProfile = AccountService::get($pid, true);
-	public function add(Request $request)
+            $res['self'] = [
-	{
+                'user' => [
-		abort_if(!config_cache('instance.stories.enabled') || !$request->user(), 404);
+                    'id' => (string) $selfProfile['id'],
-
+                    'username' => $selfProfile['acct'],
-		$this->validate($request, [
+                    'avatar' => $selfProfile['avatar'],
-			'file' => function() {
+                    'local' => $selfProfile['local'],
-				return [
+                    'is_author' => true
-					'required',
+                ],
-					'mimetypes:image/jpeg,image/png,video/mp4',
+
-					'max:' . config_cache('pixelfed.max_photo_size'),
+                'nodes' => $selfStories,
-				];
+            ];
-			},
+        }
-			'duration' => 'sometimes|integer|min:0|max:30'
+        return response()->json($res, 200, [], JSON_PRETTY_PRINT|JSON_UNESCAPED_SLASHES);
-		]);
+    }
-
+
-		$user = $request->user();
+    public function selfCarousel(Request $request)
-
+    {
-		$count = Story::whereProfileId($user->profile_id)
+        abort_if(!config_cache('instance.stories.enabled') || !$request->user(), 404);
-			->whereActive(true)
+        $pid = $request->user()->profile_id;
-			->where('expires_at', '>', now())
+
-			->count();
+        if(config('database.default') == 'pgsql') {
-
+            $s = Cache::remember(self::RECENT_KEY . $pid, self::RECENT_TTL, function() use($pid) {
-		if($count >= Story::MAX_PER_DAY) {
+                return Story::select('stories.*', 'followers.following_id')
-			abort(418, 'You have reached your limit for new Stories today.');
+                    ->leftJoin('followers', 'followers.following_id', 'stories.profile_id')
-		}
+                    ->where('followers.profile_id', $pid)
-
+                    ->where('stories.active', true)
-		$photo = $request->file('file');
+                    ->map(function($s) {
-		$path = $this->storeMedia($photo, $user);
+                        $r  = new \StdClass;
-
+                        $r->id = $s->id;
-		$story = new Story();
+                        $r->profile_id = $s->profile_id;
-		$story->duration = $request->input('duration', 3);
+                        $r->type = $s->type;
-		$story->profile_id = $user->profile_id;
+                        $r->path = $s->path;
-		$story->type = Str::endsWith($photo->getMimeType(), 'mp4') ? 'video' :'photo';
+                        return $r;
-		$story->mime = $photo->getMimeType();
+                    })
-		$story->path = $path;
+                    ->unique('profile_id');
-		$story->local = true;
+            });
-		$story->size = $photo->getSize();
+        } else {
-		$story->bearcap_token = str_random(64);
+            $s = Cache::remember(self::RECENT_KEY . $pid, self::RECENT_TTL, function() use($pid) {
-		$story->expires_at = now()->addMinutes(1440);
+                return Story::select('stories.*', 'followers.following_id')
-		$story->save();
+                    ->leftJoin('followers', 'followers.following_id', 'stories.profile_id')
-
+                    ->where('followers.profile_id', $pid)
-		$url = $story->path;
+                    ->where('stories.active', true)
-
+                    ->orderBy('id')
-		$res = [
+                    ->get();
-			'code' => 200,
+            });
-			'msg'  => 'Successfully added',
+        }
-			'media_id' => (string) $story->id,
+
-			'media_url' => url(Storage::url($url)) . '?v=' . time(),
+        $nodes = $s->map(function($s) use($pid) {
-			'media_type' => $story->type
+            $profile = AccountService::get($s->profile_id, true);
-		];
+            if(!$profile || !isset($profile['id'])) {
-
+                return false;
-		return $res;
+            }
-	}
+
-
+            return [
-	public function publish(Request $request)
+                'id' => (string) $s->id,
-	{
+                'pid' => (string) $s->profile_id,
-		abort_if(!config_cache('instance.stories.enabled') || !$request->user(), 404);
+                'type' => $s->type,
-
+                'src' => url(Storage::url($s->path)),
-		$this->validate($request, [
+                'duration' => $s->duration ?? 3,
-			'media_id' => 'required',
+                'seen' => StoryService::hasSeen($pid, $s->id),
-			'duration' => 'required|integer|min:0|max:30',
+                'created_at' => $s->created_at->format('c')
-			'can_reply' => 'required|boolean',
+            ];
-			'can_react' => 'required|boolean'
+        })
-		]);
+        ->filter()
-
+        ->groupBy('pid')
-		$id = $request->input('media_id');
+        ->map(function($item) use($pid) {
-		$user = $request->user();
+            $profile = AccountService::get($item[0]['pid'], true);
-		$story = Story::whereProfileId($user->profile_id)
+            $url = $profile['local'] ? url("/stories/{$profile['username']}") :
-			->findOrFail($id);
+                url("/i/rs/{$profile['id']}");
-
+            return [
-		$story->active = true;
+                'id' => 'pfs:' . $profile['id'],
-		$story->duration = $request->input('duration', 10);
+                'user' => [
-		$story->can_reply = $request->input('can_reply');
+                    'id' => (string) $profile['id'],
-		$story->can_react = $request->input('can_react');
+                    'username' => $profile['username'],
-		$story->save();
+                    'username_acct' => $profile['acct'],
-
+                    'avatar' => $profile['avatar'],
-		StoryService::delLatest($story->profile_id);
+                    'local' => $profile['local'],
-		StoryFanout::dispatch($story)->onQueue('story');
+                    'is_author' => $profile['id'] == $pid
-		StoryService::addRotateQueue($story->id);
+                ],
-
+                'nodes' => $item,
-		return [
+                'url' => $url,
-			'code' => 200,
+                'seen' => StoryService::hasSeen($pid, StoryService::latest($profile['id'])),
-			'msg'  => 'Successfully published',
+            ];
-		];
+        })
-	}
+        ->sortBy('seen')
-
+        ->values();
-	public function delete(Request $request, $id)
+
-	{
+        $selfProfile = AccountService::get($pid, true);
-		abort_if(!config_cache('instance.stories.enabled') || !$request->user(), 404);
+        $res = [
-
+            'self' => [
-		$user = $request->user();
+                'user' => [
-
+                    'id' => (string) $selfProfile['id'],
-		$story = Story::whereProfileId($user->profile_id)
+                    'username' => $selfProfile['acct'],
-			->findOrFail($id);
+                    'avatar' => $selfProfile['avatar'],
-		$story->active = false;
+                    'local' => $selfProfile['local'],
-		$story->save();
+                    'is_author' => true
-
+                ],
-		StoryDelete::dispatch($story)->onQueue('story');
+
-
+                'nodes' => [],
-		return [
+            ],
-			'code' => 200,
+            'nodes' => $nodes,
-			'msg'  => 'Successfully deleted'
+        ];
-		];
+
-	}
+        if(Story::whereProfileId($pid)->whereActive(true)->exists()) {
-
+            $selfStories = Story::whereProfileId($pid)
-	public function viewed(Request $request)
+                ->whereActive(true)
-	{
+                ->get()
-		abort_if(!config_cache('instance.stories.enabled') || !$request->user(), 404);
+                ->map(function($s) use($pid) {
-
+                    return [
-		$this->validate($request, [
+                        'id' => (string) $s->id,
-			'id'	=> 'required|min:1',
+                        'type' => $s->type,
-		]);
+                        'src' => url(Storage::url($s->path)),
-		$id = $request->input('id');
+                        'duration' => $s->duration,
-
+                        'seen' => true,
-		$authed = $request->user()->profile;
+                        'created_at' => $s->created_at->format('c')
-
+                    ];
-		$story = Story::with('profile')
+                })
-			->findOrFail($id);
+                ->sortBy('id')
-		$exp = $story->expires_at;
+                ->values();
-
+            $res['self']['nodes'] = $selfStories;
-		$profile = $story->profile;
+        }
-
+        return response()->json($res, 200, [], JSON_PRETTY_PRINT|JSON_UNESCAPED_SLASHES);
-		if($story->profile_id == $authed->id) {
+    }
-			return [];
+
-		}
+    public function add(Request $request)
-
+    {
-		$publicOnly = (bool) $profile->followedBy($authed);
+        abort_if(!config_cache('instance.stories.enabled') || !$request->user(), 404);
-		abort_if(!$publicOnly, 403);
+
-
+        $this->validate($request, [
-		$v = StoryView::firstOrCreate([
+            'file' => function() {
-			'story_id' => $id,
+                return [
-			'profile_id' => $authed->id
+                    'required',
-		]);
+                    'mimetypes:image/jpeg,image/png,video/mp4',
-
+                    'max:' . config_cache('pixelfed.max_photo_size'),
-		if($v->wasRecentlyCreated) {
+                ];
-			Story::findOrFail($story->id)->increment('view_count');
+            },
-
+            'duration' => 'sometimes|integer|min:0|max:30'
-			if($story->local == false) {
+        ]);
-				StoryViewDeliver::dispatch($story, $authed)->onQueue('story');
+
-			}
+        $user = $request->user();
-		}
+
-
+        $count = Story::whereProfileId($user->profile_id)
-		Cache::forget('stories:recent:by_id:' . $authed->id);
+            ->whereActive(true)
-		StoryService::addSeen($authed->id, $story->id);
+            ->where('expires_at', '>', now())
-		return ['code' => 200];
+            ->count();
-	}
+
-
+        if($count >= Story::MAX_PER_DAY) {
-	public function comment(Request $request)
+            abort(418, 'You have reached your limit for new Stories today.');
-	{
+        }
-		abort_if(!config_cache('instance.stories.enabled') || !$request->user(), 404);
+
-		$this->validate($request, [
+        $photo = $request->file('file');
-			'sid' => 'required',
+        $path = $this->storeMedia($photo, $user);
-			'caption' => 'required|string'
+
-		]);
+        $story = new Story();
-		$pid = $request->user()->profile_id;
+        $story->duration = $request->input('duration', 3);
-		$text = $request->input('caption');
+        $story->profile_id = $user->profile_id;
-
+        $story->type = Str::endsWith($photo->getMimeType(), 'mp4') ? 'video' :'photo';
-		$story = Story::findOrFail($request->input('sid'));
+        $story->mime = $photo->getMimeType();
-
+        $story->path = $path;
-		abort_if(!$story->can_reply, 422);
+        $story->local = true;
-
+        $story->size = $photo->getSize();
-		$status = new Status;
+        $story->bearcap_token = str_random(64);
-		$status->type = 'story:reply';
+        $story->expires_at = now()->addMinutes(1440);
-		$status->profile_id = $pid;
+        $story->save();
-		$status->caption = $text;
+
-		$status->rendered = $text;
+        $url = $story->path;
-		$status->scope = 'direct';
+
-		$status->visibility = 'direct';
+        $res = [
-		$status->in_reply_to_profile_id = $story->profile_id;
+            'code' => 200,
-		$status->entities = json_encode([
+            'msg'  => 'Successfully added',
-			'story_id' => $story->id
+            'media_id' => (string) $story->id,
-		]);
+            'media_url' => url(Storage::url($url)) . '?v=' . time(),
-		$status->save();
+            'media_type' => $story->type
-
+        ];
-		$dm = new DirectMessage;
+
-		$dm->to_id = $story->profile_id;
+        return $res;
-		$dm->from_id = $pid;
+    }
-		$dm->type = 'story:comment';
+
-		$dm->status_id = $status->id;
+    public function publish(Request $request)
-		$dm->meta = json_encode([
+    {
-			'story_username' => $story->profile->username,
+        abort_if(!config_cache('instance.stories.enabled') || !$request->user(), 404);
-			'story_actor_username' => $request->user()->username,
+
-			'story_id' => $story->id,
+        $this->validate($request, [
-			'story_media_url' => url(Storage::url($story->path)),
+            'media_id' => 'required',
-			'caption' => $text
+            'duration' => 'required|integer|min:0|max:30',
-		]);
+            'can_reply' => 'required|boolean',
-		$dm->save();
+            'can_react' => 'required|boolean'
-
+        ]);
-		Conversation::updateOrInsert(
+
-			[
+        $id = $request->input('media_id');
-				'to_id' => $story->profile_id,
+        $user = $request->user();
-				'from_id' => $pid
+        $story = Story::whereProfileId($user->profile_id)
-			],
+            ->findOrFail($id);
-			[
+
-				'type' => 'story:comment',
+        $story->active = true;
-				'status_id' => $status->id,
+        $story->duration = $request->input('duration', 10);
-				'dm_id' => $dm->id,
+        $story->can_reply = $request->input('can_reply');
-				'is_hidden' => false
+        $story->can_react = $request->input('can_react');
-			]
+        $story->save();
-		);
+
-
+        StoryService::delLatest($story->profile_id);
-		if($story->local) {
+        StoryFanout::dispatch($story)->onQueue('story');
-			$n = new Notification;
+        StoryService::addRotateQueue($story->id);
-			$n->profile_id = $dm->to_id;
+
-			$n->actor_id = $dm->from_id;
+        return [
-			$n->item_id = $dm->id;
+            'code' => 200,
-			$n->item_type = 'App\DirectMessage';
+            'msg'  => 'Successfully published',
-			$n->action = 'story:comment';
+        ];
-			$n->save();
+    }
-		} else {
+
-			StoryReplyDeliver::dispatch($story, $status)->onQueue('story');
+    public function delete(Request $request, $id)
-		}
+    {
-
+        abort_if(!config_cache('instance.stories.enabled') || !$request->user(), 404);
-		return [
+
-			'code' => 200,
+        $user = $request->user();
-			'msg'  => 'Sent!'
+
-		];
+        $story = Story::whereProfileId($user->profile_id)
-	}
+            ->findOrFail($id);
-
+        $story->active = false;
-	protected function storeMedia($photo, $user)
+        $story->save();
-	{
+
-		$mimes = explode(',', config_cache('pixelfed.media_types'));
+        StoryDelete::dispatch($story)->onQueue('story');
-		if(in_array($photo->getMimeType(), [
+
-			'image/jpeg',
+        return [
-			'image/png',
+            'code' => 200,
-			'video/mp4'
+            'msg'  => 'Successfully deleted'
-		]) == false) {
+        ];
-			abort(400, 'Invalid media type');
+    }
-			return;
+
-		}
+    public function viewed(Request $request)
-
+    {
-		$storagePath = MediaPathService::story($user->profile);
+        abort_if(!config_cache('instance.stories.enabled') || !$request->user(), 404);
-		$path = $photo->storePubliclyAs($storagePath, Str::random(random_int(2, 12)) . '_' . Str::random(random_int(32, 35)) . '_' . Str::random(random_int(1, 14)) . '.' . $photo->extension());
+
-		return $path;
+        $this->validate($request, [
-	}
+            'id'    => 'required|min:1',
-
+        ]);
-	public function viewers(Request $request)
+        $id = $request->input('id');
-	{
+
-		abort_if(!config_cache('instance.stories.enabled') || !$request->user(), 404);
+        $authed = $request->user()->profile;
-
+
-		$this->validate($request, [
+        $story = Story::with('profile')
-			'sid' => 'required|string|min:1|max:50'
+            ->findOrFail($id);
-		]);
+        $exp = $story->expires_at;
-
+
-		$pid = $request->user()->profile_id;
+        $profile = $story->profile;
-		$sid = $request->input('sid');
+
-
+        if($story->profile_id == $authed->id) {
-		$story = Story::whereProfileId($pid)
+            return [];
-			->whereActive(true)
+        }
-			->findOrFail($sid);
+
-
+        $publicOnly = (bool) $profile->followedBy($authed);
-		$viewers = StoryView::whereStoryId($story->id)
+        abort_if(!$publicOnly, 403);
+
+        $v = StoryView::firstOrCreate([
+            'story_id' => $id,
+            'profile_id' => $authed->id
+        ]);
+
+        if($v->wasRecentlyCreated) {
+            Story::findOrFail($story->id)->increment('view_count');
+
+            if($story->local == false) {
+                StoryViewDeliver::dispatch($story, $authed)->onQueue('story');
+            }
+        }
+
+        Cache::forget('stories:recent:by_id:' . $authed->id);
+        StoryService::addSeen($authed->id, $story->id);
+        return ['code' => 200];
+    }
+
+    public function comment(Request $request)
+    {
+        abort_if(!config_cache('instance.stories.enabled') || !$request->user(), 404);
+        $this->validate($request, [
+            'sid' => 'required',
+            'caption' => 'required|string'
+        ]);
+        $pid = $request->user()->profile_id;
+        $text = $request->input('caption');
+
+        $story = Story::findOrFail($request->input('sid'));
+
+        abort_if(!$story->can_reply, 422);
+
+        $status = new Status;
+        $status->type = 'story:reply';
+        $status->profile_id = $pid;
+        $status->caption = $text;
+        $status->rendered = $text;
+        $status->scope = 'direct';
+        $status->visibility = 'direct';
+        $status->in_reply_to_profile_id = $story->profile_id;
+        $status->entities = json_encode([
+            'story_id' => $story->id
+        ]);
+        $status->save();
+
+        $dm = new DirectMessage;
+        $dm->to_id = $story->profile_id;
+        $dm->from_id = $pid;
+        $dm->type = 'story:comment';
+        $dm->status_id = $status->id;
+        $dm->meta = json_encode([
+            'story_username' => $story->profile->username,
+            'story_actor_username' => $request->user()->username,
+            'story_id' => $story->id,
+            'story_media_url' => url(Storage::url($story->path)),
+            'caption' => $text
+        ]);
+        $dm->save();
+
+        Conversation::updateOrInsert(
+            [
+                'to_id' => $story->profile_id,
+                'from_id' => $pid
+            ],
+            [
+                'type' => 'story:comment',
+                'status_id' => $status->id,
+                'dm_id' => $dm->id,
+                'is_hidden' => false
+            ]
+        );
+
+        if($story->local) {
+            $n = new Notification;
+            $n->profile_id = $dm->to_id;
+            $n->actor_id = $dm->from_id;
+            $n->item_id = $dm->id;
+            $n->item_type = 'App\DirectMessage';
+            $n->action = 'story:comment';
+            $n->save();
+        } else {
+            StoryReplyDeliver::dispatch($story, $status)->onQueue('story');
+        }
+
+        return [
+            'code' => 200,
+            'msg'  => 'Sent!'
+        ];
+    }
+
+    protected function storeMedia($photo, $user)
+    {
+        $mimes = explode(',', config_cache('pixelfed.media_types'));
+        if(in_array($photo->getMimeType(), [
+            'image/jpeg',
+            'image/png',
+            'video/mp4'
+        ]) == false) {
+            abort(400, 'Invalid media type');
+            return;
+        }
+
+        $storagePath = MediaPathService::story($user->profile);
+        $path = $photo->storePubliclyAs($storagePath, Str::random(random_int(2, 12)) . '_' . Str::random(random_int(32, 35)) . '_' . Str::random(random_int(1, 14)) . '.' . $photo->extension());
+        return $path;
+    }
+
+    public function viewers(Request $request)
+    {
+        abort_if(!config_cache('instance.stories.enabled') || !$request->user(), 404);
+
+        $this->validate($request, [
+            'sid' => 'required|string|min:1|max:50'
+        ]);
+
+        $pid = $request->user()->profile_id;
+        $sid = $request->input('sid');
+
+        $story = Story::whereProfileId($pid)
+            ->whereActive(true)
+            ->findOrFail($sid);
+
+        $viewers = StoryView::whereStoryId($story->id)
             ->orderByDesc('id')
-			->cursorPaginate(10);
+            ->cursorPaginate(10);
 
-		return StoryViewResource::collection($viewers);
+        return StoryViewResource::collection($viewers);
-	}
+    }
 }

app/Jobs/DeletePipeline/DeleteRemoteStatusPipeline.php

@@ -57,7 +57,7 @@ class DeleteRemoteStatusPipeline implements ShouldQueue
         $status = $this->status;
 
         if(AccountService::get($status->profile_id, true)) {
-            DecrementPostCount::dispatch($status->profile_id)->onQueue('feed');
+            DecrementPostCount::dispatch($status->profile_id)->onQueue('low');
         }
 
         NetworkTimelineService::del($status->id);
@@ -76,7 +76,10 @@ class DeleteRemoteStatusPipeline implements ShouldQueue
             });
         Mention::whereStatusId($status->id)->forceDelete();
         Report::whereObjectType('App\Status')->whereObjectId($status->id)->delete();
-        StatusHashtag::whereStatusId($status->id)->delete();
+        $statusHashtags = StatusHashtag::whereStatusId($status->id)->get();
+        foreach($statusHashtags as $stag) {
+        	$stag->delete();
+        }
         StatusView::whereStatusId($status->id)->delete();
         Status::whereReblogOfId($status->id)->forceDelete();
         $status->forceDelete();

app/Jobs/FollowPipeline/FollowServiceWarmCache.php

@@ -73,7 +73,7 @@ class FollowServiceWarmCache implements ShouldQueue
         if(Follower::whereProfileId($id)->orWhere('following_id', $id)->count()) {
             $following = [];
             $followers = [];
-    		foreach(Follower::lazy() as $follow) {
+    		foreach(Follower::where('following_id', $id)->orWhere('profile_id', $id)->lazyById(500) as $follow) {
                 if($follow->following_id != $id && $follow->profile_id != $id) {
                     continue;
                 }

app/Jobs/FollowPipeline/UnfollowPipeline.php

@@ -17,6 +17,7 @@ use Illuminate\Support\Facades\Redis;
 use App\Services\AccountService;
 use App\Services\FollowerService;
 use App\Services\NotificationService;
+use App\Jobs\HomeFeedPipeline\FeedUnfollowPipeline;
 
 class UnfollowPipeline implements ShouldQueue
 {
@@ -55,6 +56,8 @@ class UnfollowPipeline implements ShouldQueue
 			return;
 		}
 
+		FeedUnfollowPipeline::dispatch($actor, $target)->onQueue('follow');
+
 		FollowerService::remove($actor, $target);
 
 		$actorProfileSync = Cache::get(FollowerService::FOLLOWING_SYNC_KEY . $actor);

app/Jobs/HomeFeedPipeline/FeedFollowPipeline.php

@@ -0,0 +1,87 @@
+<?php
+
+namespace App\Jobs\HomeFeedPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+use App\Services\AccountService;
+use App\Services\HomeTimelineService;
+use App\Services\SnowflakeService;
+use App\Status;
+
+class FeedFollowPipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $actorId;
+    protected $followingId;
+
+    public $timeout = 900;
+    public $tries = 3;
+    public $maxExceptions = 1;
+    public $failOnTimeout = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'hts:feed:insert:follows:aid:' . $this->actorId . ':fid:' . $this->followingId;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("hts:feed:insert:follows:aid:{$this->actorId}:fid:{$this->followingId}"))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct($actorId, $followingId)
+    {
+        $this->actorId = $actorId;
+        $this->followingId = $followingId;
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        $actorId = $this->actorId;
+        $followingId = $this->followingId;
+
+        $minId = SnowflakeService::byDate(now()->subWeeks(6));
+
+        $ids = Status::where('id', '>', $minId)
+            ->where('profile_id', $followingId)
+            ->whereNull(['in_reply_to_id', 'reblog_of_id'])
+            ->whereIn('type', ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])
+            ->whereIn('visibility',['public', 'unlisted', 'private'])
+            ->orderByDesc('id')
+            ->limit(HomeTimelineService::FOLLOWER_FEED_POST_LIMIT)
+            ->pluck('id');
+
+        foreach($ids as $id) {
+            HomeTimelineService::add($actorId, $id);
+        }
+    }
+}

app/Jobs/HomeFeedPipeline/FeedInsertPipeline.php

@@ -0,0 +1,97 @@
+<?php
+
+namespace App\Jobs\HomeFeedPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+use App\UserFilter;
+use App\Services\FollowerService;
+use App\Services\HomeTimelineService;
+use App\Services\StatusService;
+
+class FeedInsertPipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $sid;
+    protected $pid;
+
+    public $timeout = 900;
+    public $tries = 3;
+    public $maxExceptions = 1;
+    public $failOnTimeout = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'hts:feed:insert:sid:' . $this->sid;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("hts:feed:insert:sid:{$this->sid}"))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct($sid, $pid)
+    {
+        $this->sid = $sid;
+        $this->pid = $pid;
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        $sid = $this->sid;
+        $status = StatusService::get($sid, false);
+
+        if(!$status) {
+            return;
+        }
+
+        if(!in_array($status['pf_type'], ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])) {
+            return;
+        }
+
+        HomeTimelineService::add($this->pid, $this->sid);
+
+
+        $ids = FollowerService::localFollowerIds($this->pid);
+
+        if(!$ids || !count($ids)) {
+            return;
+        }
+
+        $skipIds = UserFilter::whereFilterableType('App\Profile')->whereFilterableId($status['account']['id'])->whereIn('filter_type', ['mute', 'block'])->pluck('user_id')->toArray();
+
+        foreach($ids as $id) {
+            if(!in_array($id, $skipIds)) {
+                HomeTimelineService::add($id, $this->sid);
+            }
+        }
+    }
+}

app/Jobs/HomeFeedPipeline/FeedInsertRemotePipeline.php

@@ -0,0 +1,94 @@
+<?php
+
+namespace App\Jobs\HomeFeedPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+use App\UserFilter;
+use App\Services\FollowerService;
+use App\Services\HomeTimelineService;
+use App\Services\StatusService;
+
+class FeedInsertRemotePipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $sid;
+    protected $pid;
+
+    public $timeout = 900;
+    public $tries = 3;
+    public $maxExceptions = 1;
+    public $failOnTimeout = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'hts:feed:insert:remote:sid:' . $this->sid;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("hts:feed:insert:remote:sid:{$this->sid}"))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct($sid, $pid)
+    {
+        $this->sid = $sid;
+        $this->pid = $pid;
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        $sid = $this->sid;
+        $status = StatusService::get($sid, false);
+
+        if(!$status) {
+            return;
+        }
+
+        if(!in_array($status['pf_type'], ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])) {
+            return;
+        }
+
+        $ids = FollowerService::localFollowerIds($this->pid);
+
+        if(!$ids || !count($ids)) {
+            return;
+        }
+
+        $skipIds = UserFilter::whereFilterableType('App\Profile')->whereFilterableId($status['account']['id'])->whereIn('filter_type', ['mute', 'block'])->pluck('user_id')->toArray();
+
+        foreach($ids as $id) {
+            if(!in_array($id, $skipIds)) {
+                HomeTimelineService::add($id, $this->sid);
+            }
+        }
+    }
+}

app/Jobs/HomeFeedPipeline/FeedRemovePipeline.php

@@ -0,0 +1,76 @@
+<?php
+
+namespace App\Jobs\HomeFeedPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+use App\Services\FollowerService;
+use App\Services\StatusService;
+use App\Services\HomeTimelineService;
+
+class FeedRemovePipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $sid;
+    protected $pid;
+
+    public $timeout = 900;
+    public $tries = 3;
+    public $maxExceptions = 1;
+    public $failOnTimeout = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'hts:feed:remove:sid:' . $this->sid;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("hts:feed:remove:sid:{$this->sid}"))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct($sid, $pid)
+    {
+        $this->sid = $sid;
+        $this->pid = $pid;
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        $ids = FollowerService::localFollowerIds($this->pid);
+
+        HomeTimelineService::rem($this->pid, $this->sid);
+
+        foreach($ids as $id) {
+            HomeTimelineService::rem($id, $this->sid);
+        }
+    }
+}

app/Jobs/HomeFeedPipeline/FeedRemoveRemotePipeline.php

@@ -0,0 +1,74 @@
+<?php
+
+namespace App\Jobs\HomeFeedPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+use App\Services\FollowerService;
+use App\Services\StatusService;
+use App\Services\HomeTimelineService;
+
+class FeedRemoveRemotePipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $sid;
+    protected $pid;
+
+    public $timeout = 900;
+    public $tries = 3;
+    public $maxExceptions = 1;
+    public $failOnTimeout = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'hts:feed:remove:remote:sid:' . $this->sid;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("hts:feed:remove:remote:sid:{$this->sid}"))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct($sid, $pid)
+    {
+        $this->sid = $sid;
+        $this->pid = $pid;
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        $ids = FollowerService::localFollowerIds($this->pid);
+
+        foreach($ids as $id) {
+            HomeTimelineService::rem($id, $this->sid);
+        }
+    }
+}

app/Jobs/HomeFeedPipeline/FeedUnfollowPipeline.php

@@ -0,0 +1,81 @@
+<?php
+
+namespace App\Jobs\HomeFeedPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+use App\Services\AccountService;
+use App\Services\StatusService;
+use App\Services\HomeTimelineService;
+
+class FeedUnfollowPipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $actorId;
+    protected $followingId;
+
+    public $timeout = 900;
+    public $tries = 3;
+    public $maxExceptions = 1;
+    public $failOnTimeout = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'hts:feed:remove:follows:aid:' . $this->actorId . ':fid:' . $this->followingId;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("hts:feed:remove:follows:aid:{$this->actorId}:fid:{$this->followingId}"))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct($actorId, $followingId)
+    {
+        $this->actorId = $actorId;
+        $this->followingId = $followingId;
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        $actorId = $this->actorId;
+        $followingId = $this->followingId;
+
+        $ids = HomeTimelineService::get($actorId, 0, -1);
+        foreach($ids as $id) {
+            $status = StatusService::get($id, false);
+            if($status && isset($status['account'], $status['account']['id'])) {
+                if($status['account']['id'] == $followingId) {
+                    HomeTimelineService::rem($actorId, $id);
+                }
+            }
+        }
+    }
+}

app/Jobs/HomeFeedPipeline/FeedWarmCachePipeline.php

@@ -0,0 +1,67 @@
+<?php
+
+namespace App\Jobs\HomeFeedPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use App\Services\HomeTimelineService;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+
+class FeedWarmCachePipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $pid;
+
+    public $timeout = 900;
+    public $tries = 3;
+    public $maxExceptions = 1;
+    public $failOnTimeout = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'hfp:warm-cache:pid:' . $this->pid;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("hfp:warm-cache:pid:{$this->pid}"))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct($pid)
+    {
+        $this->pid = $pid;
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        $pid = $this->pid;
+        HomeTimelineService::warmCache($pid, true, 400, true);
+    }
+}

app/Jobs/HomeFeedPipeline/HashtagInsertFanoutPipeline.php

@@ -0,0 +1,102 @@
+<?php
+
+namespace App\Jobs\HomeFeedPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use App\Hashtag;
+use App\StatusHashtag;
+use App\UserFilter;
+use App\Services\HashtagFollowService;
+use App\Services\HomeTimelineService;
+use App\Services\StatusService;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+
+class HashtagInsertFanoutPipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $hashtag;
+
+    public $timeout = 900;
+    public $tries = 3;
+    public $maxExceptions = 1;
+    public $failOnTimeout = true;
+
+    /**
+     * Delete the job if its models no longer exist.
+     *
+     * @var bool
+     */
+    public $deleteWhenMissingModels = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'hfp:hashtag:fanout:insert:' . $this->hashtag->id;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("hfp:hashtag:fanout:insert:{$this->hashtag->id}"))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct(StatusHashtag $hashtag)
+    {
+        $this->hashtag = $hashtag;
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        $hashtag = $this->hashtag;
+        $sid = $hashtag->status_id;
+        $status = StatusService::get($sid, false);
+
+        if(!$status) {
+            return;
+        }
+
+        if(!in_array($status['pf_type'], ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])) {
+            return;
+        }
+
+        $skipIds = UserFilter::whereFilterableType('App\Profile')->whereFilterableId($status['account']['id'])->whereIn('filter_type', ['mute', 'block'])->pluck('user_id')->toArray();
+
+        $ids = HashtagFollowService::getPidByHid($hashtag->hashtag_id);
+
+        if(!$ids || !count($ids)) {
+            return;
+        }
+
+        foreach($ids as $id) {
+            if(!in_array($id, $skipIds)) {
+                HomeTimelineService::add($id, $hashtag->status_id);
+            }
+        }
+    }
+}

app/Jobs/HomeFeedPipeline/HashtagRemoveFanoutPipeline.php

@@ -0,0 +1,92 @@
+<?php
+
+namespace App\Jobs\HomeFeedPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use App\Hashtag;
+use App\StatusHashtag;
+use App\Services\HashtagFollowService;
+use App\Services\HomeTimelineService;
+use App\Services\StatusService;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+
+class HashtagRemoveFanoutPipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $sid;
+    protected $hid;
+
+    public $timeout = 900;
+    public $tries = 3;
+    public $maxExceptions = 1;
+    public $failOnTimeout = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'hfp:hashtag:fanout:remove:' . $this->hid . ':' . $this->sid;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("hfp:hashtag:fanout:remove:{$this->hid}:{$this->sid}"))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct($sid, $hid)
+    {
+        $this->sid = $sid;
+        $this->hid = $hid;
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        $sid = $this->sid;
+        $hid = $this->hid;
+        $status = StatusService::get($sid, false);
+
+        if(!$status) {
+            return;
+        }
+
+        if(!in_array($status['pf_type'], ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])) {
+            return;
+        }
+
+        $ids = HashtagFollowService::getPidByHid($hid);
+
+        if(!$ids || !count($ids)) {
+            return;
+        }
+
+        foreach($ids as $id) {
+            HomeTimelineService::rem($id, $sid);
+        }
+    }
+}

app/Jobs/HomeFeedPipeline/HashtagUnfollowPipeline.php

@@ -0,0 +1,80 @@
+<?php
+
+namespace App\Jobs\HomeFeedPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+use Illuminate\Support\Facades\Cache;
+use App\Follower;
+use App\Hashtag;
+use App\StatusHashtag;
+use App\Services\HashtagFollowService;
+use App\Services\StatusService;
+use App\Services\HomeTimelineService;
+
+class HashtagUnfollowPipeline implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $pid;
+    protected $hid;
+    protected $slug;
+
+    public $timeout = 900;
+    public $tries = 3;
+    public $maxExceptions = 1;
+    public $failOnTimeout = true;
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct($hid, $pid, $slug)
+    {
+        $this->hid = $hid;
+        $this->pid = $pid;
+        $this->slug = $slug;
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        $hid = $this->hid;
+        $pid = $this->pid;
+        $slug = strtolower($this->slug);
+
+        $statusIds = HomeTimelineService::get($pid, 0, -1);
+
+        $followingIds = Cache::remember('profile:following:'.$pid, 1209600, function() use($pid) {
+            $following = Follower::whereProfileId($pid)->pluck('following_id');
+            return $following->push($pid)->toArray();
+        });
+
+        foreach($statusIds as $id) {
+            $status = StatusService::get($id, false);
+            if(!$status || empty($status['tags'])) {
+                HomeTimelineService::rem($pid, $id);
+                continue;
+            }
+            $following = in_array((int) $status['account']['id'], $followingIds);
+            if($following === true) {
+                continue;
+            }
+
+            $tags = collect($status['tags'])->map(function($tag) {
+                return strtolower($tag['name']);
+            })->filter()->values()->toArray();
+
+            if(in_array($slug, $tags)) {
+                HomeTimelineService::rem($pid, $id);
+            }
+        }
+    }
+}

app/Jobs/InboxPipeline/InboxValidator.php

@@ -193,7 +193,7 @@ class InboxValidator implements ShouldQueue
 		}
 
 		try {
-			$res = Http::timeout(20)->withHeaders([
+			$res = Http::withOptions(['allow_redirects' => false])->timeout(20)->withHeaders([
 			  'Accept'     => 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"',
 			  'User-Agent' => 'PixelfedBot v0.1 - https://pixelfed.org',
 			])->get($actor->remote_url);

app/Jobs/InboxPipeline/InboxWorker.php

@@ -173,7 +173,7 @@ class InboxWorker implements ShouldQueue
 		}
 
 		try {
-			$res = Http::timeout(20)->withHeaders([
+			$res = Http::withOptions(['allow_redirects' => false])->timeout(20)->withHeaders([
 			  'Accept'     => 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"',
 			  'User-Agent' => 'PixelfedBot v0.1 - https://pixelfed.org',
 			])->get($actor->remote_url);

app/Jobs/InternalPipeline/NotificationEpochUpdatePipeline.php

@@ -0,0 +1,71 @@
+<?php
+
+namespace App\Jobs\InternalPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+use App\Notification;
+use Cache;
+use App\Services\NotificationService;
+
+class NotificationEpochUpdatePipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    public $timeout = 1500;
+    public $tries = 3;
+    public $maxExceptions = 1;
+    public $failOnTimeout = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'ip:notification-epoch-update';
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping('ip:notification-epoch-update'))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct()
+    {
+        //
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        $rec = Notification::where('created_at', '>', now()->subMonths(6))->first();
+        $id = 1;
+        if($rec) {
+            $id = $rec->id;
+        }
+        Cache::put(NotificationService::EPOCH_CACHE_KEY . '6', $id, 1209600);
+    }
+}

app/Jobs/MediaPipeline/MediaDeletePipeline.php

@@ -10,8 +10,11 @@ use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
 use Illuminate\Support\Facades\Redis;
 use Illuminate\Support\Facades\Storage;
+use App\Services\Media\MediaHlsService;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
 
-class MediaDeletePipeline implements ShouldQueue
+class MediaDeletePipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
 {
 	use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 
@@ -20,8 +23,34 @@ class MediaDeletePipeline implements ShouldQueue
     public $timeout = 300;
     public $tries = 3;
     public $maxExceptions = 1;
+    public $failOnTimeout = true;
     public $deleteWhenMissingModels = true;
 
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'media:purge-job:id-' . $this->media->id;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("media:purge-job:id-{$this->media->id}"))->shared()->dontRelease()];
+    }
+
 	public function __construct(Media $media)
 	{
 		$this->media = $media;
@@ -63,9 +92,17 @@ class MediaDeletePipeline implements ShouldQueue
 			$disk->delete($thumb);
 		}
 
+		if($media->hls_path != null) {
+            $files = MediaHlsService::allFiles($media);
+            if($files && count($files)) {
+                foreach($files as $file) {
+                    $disk->delete($file);
+                }
+            }
+		}
+
 		$media->delete();
 
 		return 1;
 	}
-
 }

app/Jobs/ProfilePipeline/IncrementPostCount.php

@@ -8,16 +8,48 @@ use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
 use App\Profile;
 use App\Status;
 use App\Services\AccountService;
 
-class IncrementPostCount implements ShouldQueue
+class IncrementPostCount implements ShouldQueue, ShouldBeUniqueUntilProcessing
 {
 	use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 
 	public $id;
 
+	public $timeout = 900;
+	public $tries = 3;
+	public $maxExceptions = 1;
+	public $failOnTimeout = true;
+
+	/**
+	 * The number of seconds after which the job's unique lock will be released.
+	 *
+	 * @var int
+	 */
+	public $uniqueFor = 3600;
+
+	/**
+	 * Get the unique ID for the job.
+	 */
+	public function uniqueId(): string
+	{
+		return 'propipe:ipc:' . $this->id;
+	}
+
+	/**
+	 * Get the middleware the job should pass through.
+	 *
+	 * @return array<int, object>
+	 */
+	public function middleware(): array
+	{
+		return [(new WithoutOverlapping("propipe:ipc:{$this->id}"))->shared()->dontRelease()];
+	}
+
 	/**
 	 * Create a new job instance.
 	 *
@@ -47,6 +79,7 @@ class IncrementPostCount implements ShouldQueue
 		$profile->last_status_at = now();
 		$profile->save();
 		AccountService::del($id);
+		AccountService::get($id);
 
 		return 1;
 	}

app/Jobs/SharePipeline/SharePipeline.php

@@ -17,6 +17,7 @@ use GuzzleHttp\{Pool, Client, Promise};
 use App\Util\ActivityPub\HttpSignature;
 use App\Services\ReblogService;
 use App\Services\StatusService;
+use App\Jobs\HomeFeedPipeline\FeedInsertPipeline;
 
 class SharePipeline implements ShouldQueue
 {
@@ -82,6 +83,8 @@ class SharePipeline implements ShouldQueue
 			]
 		);
 
+		FeedInsertPipeline::dispatch($status->id, $status->profile_id)->onQueue('feed');
+
 		return $this->remoteAnnounceDeliver();
 	}
 

app/Jobs/SharePipeline/UndoSharePipeline.php

@@ -17,6 +17,7 @@ use GuzzleHttp\{Pool, Client, Promise};
 use App\Util\ActivityPub\HttpSignature;
 use App\Services\ReblogService;
 use App\Services\StatusService;
+use App\Jobs\HomeFeedPipeline\FeedRemovePipeline;
 
 class UndoSharePipeline implements ShouldQueue
 {
@@ -35,6 +36,8 @@ class UndoSharePipeline implements ShouldQueue
 		$actor = $status->profile;
 		$parent = Status::find($status->reblog_of_id);
 
+		FeedRemovePipeline::dispatch($status->id, $status->profile_id)->onQueue('feed');
+
 		if($parent) {
 			$target = $parent->profile_id;
 			ReblogService::removePostReblog($parent->profile_id, $status->id);

app/Jobs/StatusPipeline/RemoteStatusDelete.php

@@ -153,7 +153,10 @@ class RemoteStatusDelete implements ShouldQueue, ShouldBeUniqueUntilProcessing
             ->whereObjectId($status->id)
             ->delete();
         StatusArchived::whereStatusId($status->id)->delete();
-        StatusHashtag::whereStatusId($status->id)->delete();
+        $statusHashtags = StatusHashtag::whereStatusId($status->id)->get();
+        foreach($statusHashtags as $stag) {
+        	$stag->delete();
+        }
         StatusView::whereStatusId($status->id)->delete();
         Status::whereInReplyToId($status->id)->update(['in_reply_to_id' => null]);
 

app/Jobs/StatusPipeline/StatusDelete.php

@@ -130,7 +130,10 @@ class StatusDelete implements ShouldQueue
 			->delete();
 
         StatusArchived::whereStatusId($status->id)->delete();
-        StatusHashtag::whereStatusId($status->id)->delete();
+        $statusHashtags = StatusHashtag::whereStatusId($status->id)->get();
+        foreach($statusHashtags as $stag) {
+        	$stag->delete();
+        }
         StatusView::whereStatusId($status->id)->delete();
 		Status::whereInReplyToId($status->id)->update(['in_reply_to_id' => null]);
 

app/Jobs/StatusPipeline/StatusEntityLexer.php

@@ -19,171 +19,189 @@ use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
+use App\Services\StatusService;
 use App\Services\UserFilterService;
 use App\Services\AdminShadowFilterService;
+use App\Jobs\HomeFeedPipeline\FeedInsertPipeline;
+use App\Jobs\HomeFeedPipeline\HashtagInsertFanoutPipeline;
 
 class StatusEntityLexer implements ShouldQueue
 {
-	use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
-
+
-	protected $status;
+    protected $status;
-	protected $entities;
+    protected $entities;
-	protected $autolink;
+    protected $autolink;
-
+
-	/**
+    /**
-	 * Delete the job if its models no longer exist.
+     * Delete the job if its models no longer exist.
-	 *
+     *
-	 * @var bool
+     * @var bool
-	 */
+     */
-	public $deleteWhenMissingModels = true;
+    public $deleteWhenMissingModels = true;
-
+
-	/**
+    /**
-	 * Create a new job instance.
+     * Create a new job instance.
-	 *
+     *
-	 * @return void
+     * @return void
-	 */
+     */
-	public function __construct(Status $status)
+    public function __construct(Status $status)
-	{
+    {
-		$this->status = $status;
+        $this->status = $status;
-	}
+    }
-
+
-	/**
+    /**
-	 * Execute the job.
+     * Execute the job.
-	 *
+     *
-	 * @return void
+     * @return void
-	 */
+     */
-	public function handle()
+    public function handle()
-	{
+    {
-		$profile = $this->status->profile;
+        $profile = $this->status->profile;
-		$status = $this->status;
+        $status = $this->status;
-
+
-		if(in_array($status->type, ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])) {
+        if(in_array($status->type, ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])) {
-			$profile->status_count = $profile->status_count + 1;
+            $profile->status_count = $profile->status_count + 1;
-			$profile->save();
+            $profile->save();
-		}
+        }
-
+
-		if($profile->no_autolink == false) {
+        if($profile->no_autolink == false) {
-			$this->parseEntities();
+            $this->parseEntities();
-		}
+        }
-	}
+    }
-
+
-	public function parseEntities()
+    public function parseEntities()
-	{
+    {
-		$this->extractEntities();
+        $this->extractEntities();
-	}
+    }
-
+
-	public function extractEntities()
+    public function extractEntities()
-	{
+    {
-		$this->entities = Extractor::create()->extract($this->status->caption);
+        $this->entities = Extractor::create()->extract($this->status->caption);
-		$this->autolinkStatus();
+        $this->autolinkStatus();
-	}
+    }
-
+
-	public function autolinkStatus()
+    public function autolinkStatus()
-	{
+    {
-		$this->autolink = Autolink::create()->autolink($this->status->caption);
+        $this->autolink = Autolink::create()->autolink($this->status->caption);
-		$this->storeEntities();
+        $this->storeEntities();
-	}
+    }
-
+
-	public function storeEntities()
+    public function storeEntities()
-	{
+    {
-		$this->storeHashtags();
+        $this->storeHashtags();
-		DB::transaction(function () {
+        DB::transaction(function () {
-			$status = $this->status;
+            $status = $this->status;
-			$status->rendered = nl2br($this->autolink);
+            $status->rendered = nl2br($this->autolink);
-			$status->save();
+            $status->save();
-		});
+        });
-	}
+    }
-
+
-	public function storeHashtags()
+    public function storeHashtags()
-	{
+    {
-		$tags = array_unique($this->entities['hashtags']);
+        $tags = array_unique($this->entities['hashtags']);
-		$status = $this->status;
+        $status = $this->status;
-
+
-		foreach ($tags as $tag) {
+        foreach ($tags as $tag) {
-			if(mb_strlen($tag) > 124) {
+            if(mb_strlen($tag) > 124) {
-				continue;
+                continue;
-			}
+            }
-			DB::transaction(function () use ($status, $tag) {
+            DB::transaction(function () use ($status, $tag) {
-				$slug = str_slug($tag, '-', false);
+                $slug = str_slug($tag, '-', false);
-				$hashtag = Hashtag::where('slug', $slug)->first();
+
-				if (!$hashtag) {
+                $hashtag = Hashtag::firstOrCreate([
-					$hashtag = Hashtag::create(
+                    'slug' => $slug
-						['name' => $tag, 'slug' => $slug]
+                ], [
-					);
+                    'name' => $tag
-				}
+                ]);
-
+
-				StatusHashtag::firstOrCreate(
+                StatusHashtag::firstOrCreate(
-					[
+                    [
-						'status_id' => $status->id,
+                        'status_id' => $status->id,
-						'hashtag_id' => $hashtag->id,
+                        'hashtag_id' => $hashtag->id,
-						'profile_id' => $status->profile_id,
+                        'profile_id' => $status->profile_id,
-						'status_visibility' => $status->visibility,
+                        'status_visibility' => $status->visibility,
-					]
+                    ]
-				);
+                );
-			});
+            });
-		}
+        }
-		$this->storeMentions();
+        $this->storeMentions();
-	}
+    }
-
+
-	public function storeMentions()
+    public function storeMentions()
-	{
+    {
-		$mentions = array_unique($this->entities['mentions']);
+        $mentions = array_unique($this->entities['mentions']);
-		$status = $this->status;
+        $status = $this->status;
-
+
-		foreach ($mentions as $mention) {
+        foreach ($mentions as $mention) {
-			$mentioned = Profile::whereUsername($mention)->first();
+            $mentioned = Profile::whereUsername($mention)->first();
-
+
-			if (empty($mentioned) || !isset($mentioned->id)) {
+            if (empty($mentioned) || !isset($mentioned->id)) {
-				continue;
+                continue;
-			}
+            }
             $blocks = UserFilterService::blocks($mentioned->id);
             if($blocks && in_array($status->profile_id, $blocks)) {
                 continue;
             }
 
-			DB::transaction(function () use ($status, $mentioned) {
+            DB::transaction(function () use ($status, $mentioned) {
-				$m = new Mention();
+                $m = new Mention();
-				$m->status_id = $status->id;
+                $m->status_id = $status->id;
-				$m->profile_id = $mentioned->id;
+                $m->profile_id = $mentioned->id;
-				$m->save();
+                $m->save();
-
+
-				MentionPipeline::dispatch($status, $m);
+                MentionPipeline::dispatch($status, $m);
-			});
+            });
-		}
+        }
-		$this->deliver();
+        $this->fanout();
-	}
+    }
-
+
-	public function deliver()
+    public function fanout()
-	{
+    {
-		$status = $this->status;
+        $status = $this->status;
-		$types = [
+        StatusService::refresh($status->id);
-			'photo',
+
-			'photo:album',
+        if(config('exp.cached_home_timeline')) {
-			'video',
+            if( $status->in_reply_to_id === null &&
-			'video:album',
+                in_array($status->scope, ['public', 'unlisted', 'private'])
-			'photo:video:album'
+            ) {
-		];
+                FeedInsertPipeline::dispatch($status->id, $status->profile_id)->onQueue('feed');
-
+            }
-		if(config_cache('pixelfed.bouncer.enabled')) {
+        }
-			Bouncer::get($status);
+        $this->deliver();
-		}
+    }
-
+
-		Cache::forget('pf:atom:user-feed:by-id:' . $status->profile_id);
+    public function deliver()
-		$hideNsfw = config('instance.hide_nsfw_on_public_feeds');
+    {
-		if( $status->uri == null &&
+        $status = $this->status;
-			$status->scope == 'public' &&
+        $types = [
-			in_array($status->type, $types) &&
+            'photo',
-			$status->in_reply_to_id === null &&
+            'photo:album',
-			$status->reblog_of_id === null &&
+            'video',
-			($hideNsfw ? $status->is_nsfw == false : true)
+            'video:album',
-		) {
+            'photo:video:album'
+        ];
+
+        if(config_cache('pixelfed.bouncer.enabled')) {
+            Bouncer::get($status);
+        }
+
+        Cache::forget('pf:atom:user-feed:by-id:' . $status->profile_id);
+        $hideNsfw = config('instance.hide_nsfw_on_public_feeds');
+        if( $status->uri == null &&
+            $status->scope == 'public' &&
+            in_array($status->type, $types) &&
+            $status->in_reply_to_id === null &&
+            $status->reblog_of_id === null &&
+            ($hideNsfw ? $status->is_nsfw == false : true)
+        ) {
             if(AdminShadowFilterService::canAddToPublicFeedByProfileId($status->profile_id)) {
-    			PublicTimelineService::add($status->id);
+                PublicTimelineService::add($status->id);
             }
-		}
+        }
 
-		if(config_cache('federation.activitypub.enabled') == true && config('app.env') == 'production') {
+        if(config_cache('federation.activitypub.enabled') == true && config('app.env') == 'production') {
-			StatusActivityPubDeliver::dispatch($status);
+            StatusActivityPubDeliver::dispatch($status);
-		}
+        }
-	}
+    }
 }

app/Jobs/StatusPipeline/StatusRemoteUpdatePipeline.php

@@ -90,7 +90,7 @@ class StatusRemoteUpdatePipeline implements ShouldQueue
 			]);
 
 		$nm->each(function($n, $key) use($status) {
-			$res = Http::retry(3, 100, throw: false)->head($n['url']);
+			$res = Http::withOptions(['allow_redirects' => false])->retry(3, 100, throw: false)->head($n['url']);
 
 			if(!$res->successful()) {
 				return;

app/Jobs/StatusPipeline/StatusTagsPipeline.php

@@ -20,117 +20,119 @@ use App\Util\ActivityPub\Helpers;
 
 class StatusTagsPipeline implements ShouldQueue
 {
-	use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
-
+
-	protected $activity;
+    protected $activity;
-	protected $status;
+    protected $status;
-
+
-	/**
+    /**
-	 * Create a new job instance.
+     * Create a new job instance.
-	 *
+     *
-	 * @return void
+     * @return void
-	 */
+     */
-	public function __construct($activity, $status)
+    public function __construct($activity, $status)
-	{
+    {
-		$this->activity = $activity;
+        $this->activity = $activity;
-		$this->status = $status;
+        $this->status = $status;
-	}
+    }
-
+
-	/**
+    /**
-	 * Execute the job.
+     * Execute the job.
-	 *
+     *
-	 * @return void
+     * @return void
-	 */
+     */
-	public function handle()
+    public function handle()
-	{
+    {
-		$res = $this->activity;
+        $res = $this->activity;
-		$status = $this->status;
+        $status = $this->status;
 
         if(isset($res['tag']['type'], $res['tag']['name'])) {
             $res['tag'] = [$res['tag']];
         }
 
-		$tags = collect($res['tag']);
+        $tags = collect($res['tag']);
 
-		// Emoji
+        // Emoji
-		$tags->filter(function($tag) {
+        $tags->filter(function($tag) {
-			return $tag && isset($tag['id'], $tag['icon'], $tag['name'], $tag['type']) && $tag['type'] == 'Emoji';
+            return $tag && isset($tag['id'], $tag['icon'], $tag['name'], $tag['type']) && $tag['type'] == 'Emoji';
-		})
+        })
-		->map(function($tag) {
+        ->map(function($tag) {
-			CustomEmojiService::import($tag['id'], $this->status->id);
+            CustomEmojiService::import($tag['id'], $this->status->id);
-		});
+        });
 
-		// Hashtags
+        // Hashtags
-		$tags->filter(function($tag) {
+        $tags->filter(function($tag) {
-			return $tag && $tag['type'] == 'Hashtag' && isset($tag['href'], $tag['name']);
+            return $tag && $tag['type'] == 'Hashtag' && isset($tag['href'], $tag['name']);
-		})
+        })
-		->map(function($tag) use($status) {
+        ->map(function($tag) use($status) {
-			$name = substr($tag['name'], 0, 1) == '#' ?
+            $name = substr($tag['name'], 0, 1) == '#' ?
-				substr($tag['name'], 1) : $tag['name'];
+                substr($tag['name'], 1) : $tag['name'];
 
-			$banned = TrendingHashtagService::getBannedHashtagNames();
+            $banned = TrendingHashtagService::getBannedHashtagNames();
 
-			if(count($banned)) {
+            if(count($banned)) {
                 if(in_array(strtolower($name), array_map('strtolower', $banned))) {
-                   	return;
+                    return;
                 }
             }
 
             if(config('database.default') === 'pgsql') {
-            	$hashtag = Hashtag::where('name', 'ilike', $name)
+                $hashtag = Hashtag::where('name', 'ilike', $name)
-            		->orWhere('slug', 'ilike', str_slug($name, '-', false))
+                    ->orWhere('slug', 'ilike', str_slug($name, '-', false))
-            		->first();
+                    ->first();
-
+
-				if(!$hashtag) {
+                if(!$hashtag) {
-					$hashtag = Hashtag::updateOrCreate([
+                    $hashtag = Hashtag::updateOrCreate([
-						'slug' => str_slug($name, '-', false),
+                        'slug' => str_slug($name, '-', false),
-						'name' => $name
+                        'name' => $name
-					]);
+                    ]);
-				}
+                }
             } else {
-				$hashtag = Hashtag::updateOrCreate([
+                $hashtag = Hashtag::updateOrCreate([
-					'slug' => str_slug($name, '-', false),
+                    'slug' => str_slug($name, '-', false),
-					'name' => $name
+                    'name' => $name
-				]);
+                ]);
             }
 
-			StatusHashtag::firstOrCreate([
+            StatusHashtag::firstOrCreate([
-				'status_id' => $status->id,
+                'status_id' => $status->id,
-				'hashtag_id' => $hashtag->id,
+                'hashtag_id' => $hashtag->id,
-				'profile_id' => $status->profile_id,
+                'profile_id' => $status->profile_id,
-				'status_visibility' => $status->scope
+                'status_visibility' => $status->scope
-			]);
+            ]);
-		});
+        });
-
+
-		// Mentions
+        // Mentions
-		$tags->filter(function($tag) {
+        $tags->filter(function($tag) {
-			return $tag &&
+            return $tag &&
-				$tag['type'] == 'Mention' &&
+                $tag['type'] == 'Mention' &&
-				isset($tag['href']) &&
+                isset($tag['href']) &&
-				substr($tag['href'], 0, 8) === 'https://';
+                substr($tag['href'], 0, 8) === 'https://';
-		})
+        })
-		->map(function($tag) use($status) {
+        ->map(function($tag) use($status) {
-			if(Helpers::validateLocalUrl($tag['href'])) {
+            if(Helpers::validateLocalUrl($tag['href'])) {
-				$parts = explode('/', $tag['href']);
+                $parts = explode('/', $tag['href']);
-				if(!$parts) {
+                if(!$parts) {
-					return;
+                    return;
-				}
+                }
-				$pid = AccountService::usernameToId(end($parts));
+                $pid = AccountService::usernameToId(end($parts));
-				if(!$pid) {
+                if(!$pid) {
-					return;
+                    return;
-				}
+                }
-			} else {
+            } else {
-				$acct = Helpers::profileFetch($tag['href']);
+                $acct = Helpers::profileFetch($tag['href']);
-				if(!$acct) {
+                if(!$acct) {
-					return;
+                    return;
-				}
+                }
-				$pid = $acct->id;
+                $pid = $acct->id;
-			}
+            }
-			$mention = new Mention;
+            $mention = new Mention;
-			$mention->status_id = $status->id;
+            $mention->status_id = $status->id;
-			$mention->profile_id = $pid;
+            $mention->profile_id = $pid;
-			$mention->save();
+            $mention->save();
-			MentionPipeline::dispatch($status, $mention);
+            MentionPipeline::dispatch($status, $mention);
-		});
+        });
-	}
+
+        StatusService::refresh($status->id);
+    }
 }

app/Jobs/VideoPipeline/VideoHlsPipeline.php

@@ -0,0 +1,109 @@
+<?php
+
+namespace App\Jobs\VideoPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use FFMpeg\Format\Video\X264;
+use FFMpeg;
+use Cache;
+use App\Services\MediaService;
+use App\Services\StatusService;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+
+class VideoHlsPipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $media;
+
+    public $timeout = 900;
+    public $tries = 3;
+    public $maxExceptions = 1;
+    public $failOnTimeout = true;
+    public $deleteWhenMissingModels = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'media:video-hls:id-' . $this->media->id;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("media:video-hls:id-{$this->media->id}"))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct($media)
+    {
+        $this->media = $media;
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        $depCheck = Cache::rememberForever('video-pipeline:hls:depcheck', function() {
+            $bin = config('laravel-ffmpeg.ffmpeg.binaries');
+            $output = shell_exec($bin . ' -version');
+            if($output && preg_match('/ffmpeg version ([^\s]+)/', $output, $matches)) {
+                $version = $matches[1];
+                return (version_compare($version, config('laravel-ffmpeg.min_hls_version')) >= 0) ? 'ok' : false;
+            } else {
+                return false;
+            }
+        });
+
+        if(!$depCheck || $depCheck !== 'ok') {
+            return;
+        }
+
+        $media = $this->media;
+
+        $bitrate = (new X264)->setKiloBitrate(config('media.hls.bitrate') ?? 1000);
+
+        $mp4 = $media->media_path;
+        $man = str_replace('.mp4', '.m3u8', $mp4);
+
+        FFMpeg::fromDisk('local')
+            ->open($mp4)
+            ->exportForHLS()
+            ->setSegmentLength(16)
+            ->setKeyFrameInterval(48)
+            ->addFormat($bitrate)
+            ->save($man);
+
+        $media->hls_path = $man;
+        $media->hls_transcoded_at = now();
+        $media->save();
+
+        MediaService::del($media->status_id);
+        usleep(50000);
+        StatusService::del($media->status_id);
+
+        return;
+    }
+}

app/Jobs/VideoPipeline/VideoThumbnail.php

@@ -16,13 +16,46 @@ use App\Jobs\MediaPipeline\MediaStoragePipeline;
 use App\Util\Media\Blurhash;
 use App\Services\MediaService;
 use App\Services\StatusService;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
 
-class VideoThumbnail implements ShouldQueue
+class VideoThumbnail implements ShouldQueue, ShouldBeUniqueUntilProcessing
 {
     use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 
     protected $media;
 
+    public $timeout = 900;
+    public $tries = 3;
+    public $maxExceptions = 1;
+    public $failOnTimeout = true;
+    public $deleteWhenMissingModels = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'media:video-thumb:id-' . $this->media->id;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("media:video-thumb:id-{$this->media->id}"))->shared()->dontRelease()];
+    }
+
     /**
      * Create a new job instance.
      *
@@ -54,7 +87,7 @@ class VideoThumbnail implements ShouldQueue
             $path[$i] = $t;
             $save = implode('/', $path);
             $video = FFMpeg::open($base)
-            ->getFrameFromSeconds(0)
+            ->getFrameFromSeconds(1)
             ->export()
             ->toDisk('local')
             ->save($save);
@@ -68,6 +101,9 @@ class VideoThumbnail implements ShouldQueue
                 $media->save();
             }
 
+            if(config('media.hls.enabled')) {
+                VideoHlsPipeline::dispatch($media)->onQueue('mmo');
+            }
         } catch (Exception $e) {
             
         }

app/Models/HashtagRelated.php

@@ -0,0 +1,24 @@
+<?php
+
+namespace App\Models;
+
+use Illuminate\Database\Eloquent\Factories\HasFactory;
+use Illuminate\Database\Eloquent\Model;
+
+class HashtagRelated extends Model
+{
+    use HasFactory;
+
+    protected $guarded = [];
+
+    /**
+     * The attributes that should be mutated to dates and other custom formats.
+     *
+     * @var array
+     */
+    protected $casts = [
+        'related_tags' => 'array',
+        'last_calculated_at' => 'datetime',
+        'last_moderated_at' => 'datetime',
+    ];
+}

app/Observers/FollowerObserver.php

@@ -5,6 +5,8 @@ namespace App\Observers;
 use App\Follower;
 use App\Services\FollowerService;
 use Cache;
+use App\Jobs\HomeFeedPipeline\FeedFollowPipeline;
+use App\Jobs\HomeFeedPipeline\FeedUnfollowPipeline;
 
 class FollowerObserver
 {
@@ -21,6 +23,7 @@ class FollowerObserver
         }
 
         FollowerService::add($follower->profile_id, $follower->following_id);
+        FeedFollowPipeline::dispatch($follower->profile_id, $follower->following_id)->onQueue('follow');
     }
 
     /**

app/Observers/HashtagFollowObserver.php

@@ -0,0 +1,51 @@
+<?php
+
+namespace App\Observers;
+
+use App\HashtagFollow;
+use App\Services\HashtagFollowService;
+use App\Jobs\HomeFeedPipeline\HashtagUnfollowPipeline;
+use Illuminate\Contracts\Events\ShouldHandleEventsAfterCommit;
+
+class HashtagFollowObserver implements ShouldHandleEventsAfterCommit
+{
+    /**
+     * Handle the HashtagFollow "created" event.
+     */
+    public function created(HashtagFollow $hashtagFollow): void
+    {
+        HashtagFollowService::add($hashtagFollow->hashtag_id, $hashtagFollow->profile_id);
+    }
+
+    /**
+     * Handle the HashtagFollow "updated" event.
+     */
+    public function updated(HashtagFollow $hashtagFollow): void
+    {
+    	//
+    }
+
+    /**
+     * Handle the HashtagFollow "deleting" event.
+     */
+    public function deleting(HashtagFollow $hashtagFollow): void
+    {
+        HashtagFollowService::unfollow($hashtagFollow->hashtag_id, $hashtagFollow->profile_id);
+    }
+
+    /**
+     * Handle the HashtagFollow "restored" event.
+     */
+    public function restored(HashtagFollow $hashtagFollow): void
+    {
+        //
+    }
+
+    /**
+     * Handle the HashtagFollow "force deleted" event.
+     */
+    public function forceDeleted(HashtagFollow $hashtagFollow): void
+    {
+        HashtagFollowService::unfollow($hashtagFollow->hashtag_id, $hashtagFollow->profile_id);
+    }
+}

app/Observers/StatusHashtagObserver.php

@@ -5,32 +5,31 @@ namespace App\Observers;
 use DB;
 use App\StatusHashtag;
 use App\Services\StatusHashtagService;
+use App\Jobs\HomeFeedPipeline\HashtagInsertFanoutPipeline;
+use App\Jobs\HomeFeedPipeline\HashtagRemoveFanoutPipeline;
+use Illuminate\Contracts\Events\ShouldHandleEventsAfterCommit;
 
-class StatusHashtagObserver
+class StatusHashtagObserver implements ShouldHandleEventsAfterCommit
 {
-    /**
-     * Handle events after all transactions are committed.
-     *
-     * @var bool
-     */
-    public $afterCommit = true;
-
     /**
      * Handle the notification "created" event.
      *
-     * @param  \App\Notification  $notification
+     * @param  \App\StatusHashtag  $hashtag
      * @return void
      */
     public function created(StatusHashtag $hashtag)
     {
         StatusHashtagService::set($hashtag->hashtag_id, $hashtag->status_id);
         DB::table('hashtags')->where('id', $hashtag->hashtag_id)->increment('cached_count');
+        if($hashtag->status_visibility && $hashtag->status_visibility === 'public') {
+            HashtagInsertFanoutPipeline::dispatch($hashtag)->onQueue('feed');
+        }
     }
 
     /**
      * Handle the notification "updated" event.
      *
-     * @param  \App\Notification  $notification
+     * @param  \App\StatusHashtag  $hashtag
      * @return void
      */
     public function updated(StatusHashtag $hashtag)
@@ -41,19 +40,22 @@ class StatusHashtagObserver
     /**
      * Handle the notification "deleted" event.
      *
-     * @param  \App\Notification  $notification
+     * @param  \App\StatusHashtag  $hashtag
      * @return void
      */
     public function deleted(StatusHashtag $hashtag)
     {
         StatusHashtagService::del($hashtag->hashtag_id, $hashtag->status_id);
         DB::table('hashtags')->where('id', $hashtag->hashtag_id)->decrement('cached_count');
+        if($hashtag->status_visibility && $hashtag->status_visibility === 'public') {
+            HashtagRemoveFanoutPipeline::dispatch($hashtag->status_id, $hashtag->hashtag_id)->onQueue('feed');
+        }
     }
 
     /**
      * Handle the notification "restored" event.
      *
-     * @param  \App\Notification  $notification
+     * @param  \App\StatusHashtag  $hashtag
      * @return void
      */
     public function restored(StatusHashtag $hashtag)
@@ -64,7 +66,7 @@ class StatusHashtagObserver
     /**
      * Handle the notification "force deleted" event.
      *
-     * @param  \App\Notification  $notification
+     * @param  \App\StatusHashtag  $hashtag
      * @return void
      */
     public function forceDeleted(StatusHashtag $hashtag)

app/Observers/StatusObserver.php

@@ -7,6 +7,8 @@ use App\Services\ProfileStatusService;
 use Cache;
 use App\Models\ImportPost;
 use App\Services\ImportService;
+use App\Jobs\HomeFeedPipeline\FeedRemovePipeline;
+use App\Jobs\HomeFeedPipeline\FeedRemoveRemotePipeline;
 
 class StatusObserver
 {
@@ -63,6 +65,14 @@ class StatusObserver
             ImportPost::whereProfileId($status->profile_id)->whereStatusId($status->id)->delete();
             ImportService::clearImportedFiles($status->profile_id);
         }
+
+        if(config('exp.cached_home_timeline')) {
+        	if($status->uri) {
+        		FeedRemoveRemotePipeline::dispatch($status->id, $status->profile_id)->onQueue('feed');
+        	} else {
+        		FeedRemovePipeline::dispatch($status->id, $status->profile_id)->onQueue('feed');
+        	}
+        }
     }
 
     /**

app/Observers/UserFilterObserver.php

@@ -4,6 +4,8 @@ namespace App\Observers;
 
 use App\UserFilter;
 use App\Services\UserFilterService;
+use App\Jobs\HomeFeedPipeline\FeedFollowPipeline;
+use App\Jobs\HomeFeedPipeline\FeedUnfollowPipeline;
 
 class UserFilterObserver
 {
@@ -78,10 +80,12 @@ class UserFilterObserver
 		switch ($userFilter->filter_type) {
 			case 'mute':
 				UserFilterService::mute($userFilter->user_id, $userFilter->filterable_id);
+				FeedUnfollowPipeline::dispatch($userFilter->user_id, $userFilter->filterable_id)->onQueue('feed');
 				break;
 				
 			case 'block':
 				UserFilterService::block($userFilter->user_id, $userFilter->filterable_id);
+				FeedUnfollowPipeline::dispatch($userFilter->user_id, $userFilter->filterable_id)->onQueue('feed');
 				break;
 		}
 	}
@@ -96,10 +100,12 @@ class UserFilterObserver
 		switch ($userFilter->filter_type) {
 			case 'mute':
 				UserFilterService::unmute($userFilter->user_id, $userFilter->filterable_id);
+				FeedFollowPipeline::dispatch($userFilter->user_id, $userFilter->filterable_id)->onQueue('feed');
 				break;
 				
 			case 'block':
 				UserFilterService::unblock($userFilter->user_id, $userFilter->filterable_id);
+				FeedFollowPipeline::dispatch($userFilter->user_id, $userFilter->filterable_id)->onQueue('feed');
 				break;
 		}
 	}

app/Services/ActivityPubFetchService.php

@@ -28,7 +28,7 @@ class ActivityPubFetchService
 		$headers['User-Agent'] = 'PixelFedBot/1.0.0 (Pixelfed/'.config('pixelfed.version').'; +'.config('app.url').')';
 
 		try {
-			$res = Http::withHeaders($headers)
+			$res = Http::withOptions(['allow_redirects' => false])->withHeaders($headers)
 				->timeout(30)
 				->connectTimeout(5)
 				->retry(3, 500)

app/Services/FollowerService.php

@@ -19,6 +19,7 @@ class FollowerService
 	const FOLLOWING_SYNC_KEY = 'pf:services:followers:sync-following:';
 	const FOLLOWING_KEY = 'pf:services:follow:following:id:';
 	const FOLLOWERS_KEY = 'pf:services:follow:followers:id:';
+	const FOLLOWERS_LOCAL_KEY = 'pf:services:follow:local-follower-ids:';
 
 	public static function add($actor, $target, $refresh = true)
 	{
@@ -212,4 +213,15 @@ class FollowerService
 		Cache::forget(self::FOLLOWERS_SYNC_KEY . $id);
 		Cache::forget(self::FOLLOWING_SYNC_KEY . $id);
 	}
+
+	public static function localFollowerIds($pid, $limit = 0)
+	{
+		$key = self::FOLLOWERS_LOCAL_KEY . $pid;
+		$res = Cache::remember($key, 7200, function() use($pid) {
+			return DB::table('followers')->whereFollowingId($pid)->whereLocalProfile(true)->pluck('profile_id')->sort();
+		});
+		return $limit ?
+			$res->take($limit)->values()->toArray() :
+			$res->values()->toArray();
+	}
 }

app/Services/HashtagFollowService.php

@@ -0,0 +1,72 @@
+<?php
+
+namespace App\Services;
+
+use Illuminate\Support\Facades\Cache;
+use Illuminate\Support\Facades\Redis;
+use App\Hashtag;
+use App\StatusHashtag;
+use App\HashtagFollow;
+
+class HashtagFollowService
+{
+	const FOLLOW_KEY = 'pf:services:hashtag-follows:v1:';
+	const CACHE_KEY = 'pf:services:hfs:byHid:';
+	const CACHE_WARMED = 'pf:services:hfs:wc:byHid';
+
+	public static function getPidByHid($hid)
+	{
+		if(!self::isWarm($hid)) {
+			return self::warmCache($hid);
+		}
+		return self::get($hid);
+	}
+
+	public static function unfollow($hid, $pid)
+	{
+		return Redis::zrem(self::CACHE_KEY . $hid, $pid);
+	}
+
+	public static function add($hid, $pid)
+	{
+		return Redis::zadd(self::CACHE_KEY . $hid, $pid, $pid);
+	}
+
+	public static function rem($hid, $pid)
+	{
+		return Redis::zrem(self::CACHE_KEY . $hid, $pid);
+	}
+
+	public static function get($hid)
+	{
+		return Redis::zrange(self::CACHE_KEY . $hid, 0, -1);
+	}
+
+	public static function count($hid)
+	{
+		return Redis::zcard(self::CACHE_KEY . $hid);
+	}
+
+	public static function warmCache($hid)
+	{
+		foreach(HashtagFollow::whereHashtagId($hid)->lazyById(20, 'id') as $h) {
+			if($h) {
+				self::add($h->hashtag_id, $h->profile_id);
+			}
+		}
+
+		self::setWarm($hid);
+
+		return self::get($hid);
+	}
+
+	public static function isWarm($hid)
+	{
+		return Redis::zcount(self::CACHE_KEY . $hid, 0, -1) ?? Redis::zscore(self::CACHE_WARMED, $hid) != null;
+	}
+
+	public static function setWarm($hid)
+	{
+		return Redis::zadd(self::CACHE_WARMED, $hid, $hid);
+	}
+}

app/Services/HashtagRelatedService.php

@@ -0,0 +1,38 @@
+<?php
+
+namespace App\Services;
+
+use DB;
+use App\StatusHashtag;
+use App\Models\HashtagRelated;
+
+class HashtagRelatedService
+{
+    public static function get($id)
+    {
+        $tag = HashtagRelated::whereHashtagId($id)->first();
+        if(!$tag) {
+            return [];
+        }
+        return $tag->related_tags;
+    }
+
+    public static function fetchRelatedTags($tag)
+    {
+        $res = StatusHashtag::query()
+            ->select('h2.name', DB::raw('COUNT(*) as related_count'))
+            ->join('status_hashtags as hs2', function ($join) {
+                $join->on('status_hashtags.status_id', '=', 'hs2.status_id')
+                     ->whereRaw('status_hashtags.hashtag_id != hs2.hashtag_id');
+            })
+            ->join('hashtags as h1', 'status_hashtags.hashtag_id', '=', 'h1.id')
+            ->join('hashtags as h2', 'hs2.hashtag_id', '=', 'h2.id')
+            ->where('h1.name', '=', $tag)
+            ->groupBy('h2.name')
+            ->orderBy('related_count', 'desc')
+            ->limit(30)
+            ->get();
+
+        return $res;
+    }
+}

app/Services/HashtagService.php

@@ -8,65 +8,80 @@ use App\Hashtag;
 use App\StatusHashtag;
 use App\HashtagFollow;
 
-class HashtagService {
+class HashtagService
+{
+    const FOLLOW_KEY = 'pf:services:hashtag:following:v1:';
+    const FOLLOW_PIDS_KEY = 'pf:services:hashtag-follows:v1:';
 
-	const FOLLOW_KEY = 'pf:services:hashtag:following:';
+    public static function get($id)
+    {
+        return Cache::remember('services:hashtag:by_id:' . $id, 3600, function() use($id) {
+            $tag = Hashtag::find($id);
+            if(!$tag) {
+                return [];
+            }
+            return [
+                'name' => $tag->name,
+                'slug' => $tag->slug,
+            ];
+        });
+    }
 
-	public static function get($id)
+    public static function count($id)
-	{
+    {
-		return Cache::remember('services:hashtag:by_id:' . $id, 3600, function() use($id) {
+        return Cache::remember('services:hashtag:total-count:by_id:' . $id, 300, function() use($id) {
-			$tag = Hashtag::find($id);
+            $tag = Hashtag::find($id);
-			if(!$tag) {
+            return $tag ? $tag->cached_count ?? 0 : 0;
-				return [];
+        });
-			}
+    }
-			return [
-				'name' => $tag->name,
-				'slug' => $tag->slug,
-			];
-		});
-	}
 
-	public static function count($id)
+    public static function isFollowing($pid, $hid)
-	{
+    {
-		return Cache::remember('services:hashtag:public-count:by_id:' . $id, 86400, function() use($id) {
+        $res = Redis::zscore(self::FOLLOW_KEY . $hid, $pid);
-			return StatusHashtag::whereHashtagId($id)->whereStatusVisibility('public')->count();
+        if($res) {
-		});
+            return true;
-	}
+        }
 
-	public static function isFollowing($pid, $hid)
+        $synced = Cache::get(self::FOLLOW_KEY . 'acct:' . $pid . ':synced');
-	{
+        if(!$synced) {
-		$res = Redis::zscore(self::FOLLOW_KEY . $pid, $hid);
+            $tags = HashtagFollow::whereProfileId($pid)
-		if($res) {
+                ->get()
-			return true;
+                ->each(function($tag) use($pid) {
-		}
+                    self::follow($pid, $tag->hashtag_id);
+                });
+            Cache::set(self::FOLLOW_KEY . 'acct:' . $pid . ':synced', true, 1209600);
 
-		$synced = Cache::get(self::FOLLOW_KEY . $pid . ':synced');
+            return (bool) Redis::zscore(self::FOLLOW_KEY . $hid, $pid) >= 1;
-		if(!$synced) {
+        }
-			$tags = HashtagFollow::whereProfileId($pid)
-				->get()
-				->each(function($tag) use($pid) {
-					self::follow($pid, $tag->hashtag_id);
-				});
-			Cache::set(self::FOLLOW_KEY . $pid . ':synced', true, 1209600);
 
-			return (bool) Redis::zscore(self::FOLLOW_KEY . $pid, $hid) > 1;
+        return false;
-		}
+    }
 
-		return false;
+    public static function follow($pid, $hid)
-	}
+    {
+    	Cache::forget(self::FOLLOW_PIDS_KEY . $hid);
+        return Redis::zadd(self::FOLLOW_KEY . $hid, $pid, $pid);
+    }
 
-	public static function follow($pid, $hid)
+    public static function unfollow($pid, $hid)
-	{
+    {
-		return Redis::zadd(self::FOLLOW_KEY . $pid, $hid, $hid);
+    	Cache::forget(self::FOLLOW_PIDS_KEY . $hid);
-	}
+        return Redis::zrem(self::FOLLOW_KEY . $hid, $pid);
+    }
 
-	public static function unfollow($pid, $hid)
+    public static function following($hid, $start = 0, $limit = 10)
-	{
+    {
-		return Redis::zrem(self::FOLLOW_KEY . $pid, $hid);
+        $synced = Cache::get(self::FOLLOW_KEY . 'acct-following:' . $hid . ':synced');
-	}
+        if(!$synced) {
+            $tags = HashtagFollow::whereHashtagId($hid)
+                ->get()
+                ->each(function($tag) use($hid) {
+                    self::follow($tag->profile_id, $hid);
+                });
+            Cache::set(self::FOLLOW_KEY . 'acct-following:' . $hid . ':synced', true, 1209600);
 
-	public static function following($pid, $start = 0, $limit = 10)
+            return Redis::zrevrange(self::FOLLOW_KEY . $hid, $start, $limit);
-	{
+        }
-		return Redis::zrevrange(self::FOLLOW_KEY . $pid, $start, $limit);
+        return Redis::zrevrange(self::FOLLOW_KEY . $hid, $start, $limit);
-	}
+    }
 }

app/Services/HomeTimelineService.php

@@ -0,0 +1,101 @@
+<?php
+
+namespace App\Services;
+
+use Illuminate\Support\Facades\Cache;
+use Illuminate\Support\Facades\Redis;
+use App\Follower;
+use App\Status;
+
+class HomeTimelineService
+{
+    const CACHE_KEY = 'pf:services:timeline:home:';
+    const FOLLOWER_FEED_POST_LIMIT = 10;
+
+    public static function get($id, $start = 0, $stop = 10)
+    {
+        if($stop > 100) {
+            $stop = 100;
+        }
+
+        return Redis::zrevrange(self::CACHE_KEY . $id, $start, $stop);
+    }
+
+    public static function getRankedMaxId($id, $start = null, $limit = 10)
+    {
+        if(!$start) {
+            return [];
+        }
+
+        return array_keys(Redis::zrevrangebyscore(self::CACHE_KEY . $id, $start, '-inf', [
+            'withscores' => true,
+            'limit' => [1, $limit - 1]
+        ]));
+    }
+
+    public static function getRankedMinId($id, $end = null, $limit = 10)
+    {
+        if(!$end) {
+            return [];
+        }
+
+        return array_keys(Redis::zrevrangebyscore(self::CACHE_KEY . $id, '+inf', $end, [
+            'withscores' => true,
+            'limit' => [0, $limit]
+        ]));
+    }
+
+    public static function add($id, $val)
+    {
+        if(self::count($id) >= 400) {
+            Redis::zpopmin(self::CACHE_KEY . $id);
+        }
+
+        return Redis::zadd(self::CACHE_KEY .$id, $val, $val);
+    }
+
+    public static function rem($id, $val)
+    {
+        return Redis::zrem(self::CACHE_KEY . $id, $val);
+    }
+
+    public static function count($id)
+    {
+        return Redis::zcard(self::CACHE_KEY . $id);
+    }
+
+    public static function warmCache($id, $force = false, $limit = 100, $returnIds = false)
+    {
+        if(self::count($id) == 0 || $force == true) {
+            Redis::del(self::CACHE_KEY . $id);
+            $following = Cache::remember('profile:following:'.$id, 1209600, function() use($id) {
+                $following = Follower::whereProfileId($id)->pluck('following_id');
+                return $following->push($id)->toArray();
+            });
+
+            $minId = SnowflakeService::byDate(now()->subMonths(6));
+
+            $filters = UserFilterService::filters($id);
+
+            if($filters && count($filters)) {
+                $following = array_diff($following, $filters);
+            }
+
+            $ids = Status::where('id', '>', $minId)
+                ->whereIn('profile_id', $following)
+                ->whereNull(['in_reply_to_id', 'reblog_of_id'])
+                ->whereIn('type', ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])
+                ->whereIn('visibility',['public', 'unlisted', 'private'])
+                ->orderByDesc('id')
+                ->limit($limit)
+                ->pluck('id');
+
+            foreach($ids as $pid) {
+                self::add($id, $pid);
+            }
+
+            return $returnIds ? $ids : 1;
+        }
+        return 0;
+    }
+}

app/Services/Media/MediaHlsService.php

@@ -0,0 +1,27 @@
+<?php
+
+namespace App\Services\Media;
+
+use Storage;
+
+class MediaHlsService
+{
+    public static function allFiles($media)
+    {
+        $path = $media->media_path;
+        if(!$path) { return; }
+        $parts = explode('/', $path);
+        $filename = array_pop($parts);
+        $dir = implode('/', $parts);
+        [$name, $ext] = explode('.', $filename);
+
+        $files = Storage::files($dir);
+
+        return collect($files)
+            ->filter(function($p) use($dir, $name) {
+                return str_starts_with($p, $dir . '/' . $name);
+            })
+            ->values()
+            ->toArray();
+    }
+}

app/Services/MediaService.php

@@ -18,7 +18,7 @@ class MediaService
 
 	public static function get($statusId)
 	{
-		return Cache::remember(self::CACHE_KEY.$statusId, 86400, function() use($statusId) {
+		return Cache::remember(self::CACHE_KEY.$statusId, 21600, function() use($statusId) {
 			$media = Media::whereStatusId($statusId)->orderBy('order')->get();
 			if(!$media) {
 				return [];
@@ -46,7 +46,8 @@ class MediaService
 				$media['orientation'],
 				$media['filter_name'],
 				$media['filter_class'],
-				$media['mime']
+				$media['mime'],
+				$media['hls_manifest']
 			);
 
 			$media['type'] = $mime ? strtolower($mime[0]) : 'unknown';

app/Services/NotificationService.php

@@ -12,6 +12,7 @@ use App\Transformer\Api\NotificationTransformer;
 use League\Fractal;
 use League\Fractal\Serializer\ArraySerializer;
 use League\Fractal\Pagination\IlluminatePaginatorAdapter;
+use App\Jobs\InternalPipeline\NotificationEpochUpdatePipeline;
 
 class NotificationService {
 
@@ -48,12 +49,12 @@ class NotificationService {
 
 	public static function getEpochId($months = 6)
 	{
-		return Cache::remember(self::EPOCH_CACHE_KEY . $months, 1209600, function() use($months) {
+		$epoch = Cache::get(self::EPOCH_CACHE_KEY . $months);
-            if(Notification::count() === 0) {
+		if(!$epoch) {
-                return 0;
+			NotificationEpochUpdatePipeline::dispatch();
-            }
+			return 1;
-			return Notification::where('created_at', '>', now()->subMonths($months))->first()->id;
+		}
-		});
+		return $epoch;
 	}
 
 	public static function coldGet($id, $start = 0, $stop = 400)

app/Services/StatusHashtagService.php

@@ -84,18 +84,14 @@ class StatusHashtagService {
 
 	public static function statusTags($statusId)
 	{
-		$key = 'pf:services:sh:id:' . $statusId;
+		$status = Status::with('hashtags')->find($statusId);
-
+		if(!$status) {
-		return Cache::remember($key, 604800, function() use($statusId) {
+			return [];
-			$status = Status::find($statusId);
+		}
-			if(!$status) {
-				return [];
-			}
 
-			$fractal = new Fractal\Manager();
+		$fractal = new Fractal\Manager();
-			$fractal->setSerializer(new ArraySerializer());
+		$fractal->setSerializer(new ArraySerializer());
-			$resource = new Fractal\Resource\Collection($status->hashtags, new HashtagTransformer());
+		$resource = new Fractal\Resource\Collection($status->hashtags, new HashtagTransformer());
-			return $fractal->createData($resource)->toArray();
+		return $fractal->createData($resource)->toArray();
-		});
 	}
 }

app/Transformer/Api/MediaTransformer.php

@@ -4,6 +4,7 @@ namespace App\Transformer\Api;
 
 use App\Media;
 use League\Fractal;
+use Storage;
 
 class MediaTransformer extends Fractal\TransformerAbstract
 {
@@ -28,6 +29,10 @@ class MediaTransformer extends Fractal\TransformerAbstract
             'blurhash'      => $media->blurhash ?? 'U4Rfzst8?bt7ogayj[j[~pfQ9Goe%Mj[WBay'
         ];
 
+        if(config('media.hls.enabled') && $media->hls_transcoded_at != null && $media->hls_path) {
+            $res['hls_manifest'] = url(Storage::url($media->hls_path));
+        }
+
         if($media->width && $media->height) {
             $res['meta'] = [
                 'focus' => [

app/Util/ActivityPub/Helpers.php

@@ -35,6 +35,7 @@ use App\Services\MediaStorageService;
 use App\Services\NetworkTimelineService;
 use App\Jobs\MediaPipeline\MediaStoragePipeline;
 use App\Jobs\AvatarPipeline\RemoteAvatarFetch;
+use App\Jobs\HomeFeedPipeline\FeedInsertRemotePipeline;
 use App\Util\Media\License;
 use App\Models\Poll;
 use Illuminate\Contracts\Cache\LockTimeoutException;
@@ -537,6 +538,12 @@ class Helpers {
 
         IncrementPostCount::dispatch($pid)->onQueue('low');
 
+        if( $status->in_reply_to_id === null &&
+            in_array($status->type, ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])
+        ) {
+            FeedInsertRemotePipeline::dispatch($status->id, $pid)->onQueue('feed');
+        }
+
         return $status;
     }
 
@@ -760,6 +767,13 @@ class Helpers {
         if(!isset($res['preferredUsername']) && !isset($res['nickname'])) {
             return;
         }
+        // skip invalid usernames
+        if(!ctype_alnum($res['preferredUsername'])) {
+            $tmpUsername = str_replace(['_', '.', '-'], '', $res['preferredUsername']);
+            if(!ctype_alnum($tmpUsername)) {
+                return;
+            }
+        }
         $username = (string) Purify::clean($res['preferredUsername'] ?? $res['nickname']);
         if(empty($username)) {
             return;

app/Util/ActivityPub/Inbox.php

@@ -49,6 +49,7 @@ use App\Models\Conversation;
 use App\Models\RemoteReport;
 use App\Jobs\ProfilePipeline\IncrementPostCount;
 use App\Jobs\ProfilePipeline\DecrementPostCount;
+use App\Jobs\HomeFeedPipeline\FeedRemoveRemotePipeline;
 
 class Inbox
 {
@@ -707,6 +708,7 @@ class Inbox
 						if(!$status) {
 							return;
 						}
+						FeedRemoveRemotePipeline::dispatch($status->id, $status->profile_id)->onQueue('feed');
 						RemoteStatusDelete::dispatch($status)->onQueue('high');
 						return;
 					break;
@@ -803,6 +805,7 @@ class Inbox
 				if(!$status) {
 					return;
 				}
+				FeedRemoveRemotePipeline::dispatch($status->id, $status->profile_id)->onQueue('feed');
 				Status::whereProfileId($profile->id)
 					->whereReblogOfId($status->id)
 					->delete();

app/Util/Site/Config.php

@@ -7,86 +7,100 @@ use Illuminate\Support\Str;
 
 class Config {
 
-	const CACHE_KEY = 'api:site:configuration:_v0.8';
+    const CACHE_KEY = 'api:site:configuration:_v0.8';
 
-	public static function get() {
+    public static function get() {
-		return Cache::remember(self::CACHE_KEY, 900, function() {
+        return Cache::remember(self::CACHE_KEY, 900, function() {
-			return [
+            $hls = [
-				'version' => config('pixelfed.version'),
+                'enabled' => config('media.hls.enabled'),
-				'open_registration' => (bool) config_cache('pixelfed.open_registration'),
+            ];
-				'uploader' => [
+            if(config('media.hls.enabled')) {
-					'max_photo_size' => (int) config('pixelfed.max_photo_size'),
+                $hls = [
-					'max_caption_length' => (int) config('pixelfed.max_caption_length'),
+                    'enabled' => true,
-					'max_altext_length' => (int) config('pixelfed.max_altext_length', 150),
+                    'debug' => (bool) config('media.hls.debug'),
-					'album_limit' => (int) config_cache('pixelfed.max_album_length'),
+                    'p2p' => (bool) config('media.hls.p2p'),
-					'image_quality' => (int) config_cache('pixelfed.image_quality'),
+                    'p2p_debug' => (bool) config('media.hls.p2p_debug'),
+                    'tracker' => config('media.hls.tracker'),
+                    'ice' => config('media.hls.ice')
+                ];
+            }
+            return [
+                'version' => config('pixelfed.version'),
+                'open_registration' => (bool) config_cache('pixelfed.open_registration'),
+                'uploader' => [
+                    'max_photo_size' => (int) config('pixelfed.max_photo_size'),
+                    'max_caption_length' => (int) config('pixelfed.max_caption_length'),
+                    'max_altext_length' => (int) config('pixelfed.max_altext_length', 150),
+                    'album_limit' => (int) config_cache('pixelfed.max_album_length'),
+                    'image_quality' => (int) config_cache('pixelfed.image_quality'),
 
-					'max_collection_length' => (int) config('pixelfed.max_collection_length', 18),
+                    'max_collection_length' => (int) config('pixelfed.max_collection_length', 18),
 
-					'optimize_image' => (bool) config('pixelfed.optimize_image'),
+                    'optimize_image' => (bool) config('pixelfed.optimize_image'),
-					'optimize_video' => (bool) config('pixelfed.optimize_video'),
+                    'optimize_video' => (bool) config('pixelfed.optimize_video'),
 
-					'media_types' => config_cache('pixelfed.media_types'),
+                    'media_types' => config_cache('pixelfed.media_types'),
-					'mime_types' => config_cache('pixelfed.media_types') ? explode(',', config_cache('pixelfed.media_types')) : [],
+                    'mime_types' => config_cache('pixelfed.media_types') ? explode(',', config_cache('pixelfed.media_types')) : [],
-					'enforce_account_limit' => (bool) config_cache('pixelfed.enforce_account_limit')
+                    'enforce_account_limit' => (bool) config_cache('pixelfed.enforce_account_limit')
-				],
+                ],
 
-				'activitypub' => [
+                'activitypub' => [
-					'enabled' => (bool) config_cache('federation.activitypub.enabled'),
+                    'enabled' => (bool) config_cache('federation.activitypub.enabled'),
-					'remote_follow' => config('federation.activitypub.remoteFollow')
+                    'remote_follow' => config('federation.activitypub.remoteFollow')
-				],
+                ],
 
-				'ab' => config('exp'),
+                'ab' => config('exp'),
 
-				'site' => [
+                'site' => [
-					'name' => config_cache('app.name'),
+                    'name' => config_cache('app.name'),
-					'domain' => config('pixelfed.domain.app'),
+                    'domain' => config('pixelfed.domain.app'),
-					'url'    => config('app.url'),
+                    'url'    => config('app.url'),
-					'description' => config_cache('app.short_description')
+                    'description' => config_cache('app.short_description')
-				],
+                ],
 
-				'account' => [
+                'account' => [
-					'max_avatar_size' => config('pixelfed.max_avatar_size'),
+                    'max_avatar_size' => config('pixelfed.max_avatar_size'),
-					'max_bio_length' => config('pixelfed.max_bio_length'),
+                    'max_bio_length' => config('pixelfed.max_bio_length'),
-					'max_name_length' => config('pixelfed.max_name_length'),
+                    'max_name_length' => config('pixelfed.max_name_length'),
-					'min_password_length' => config('pixelfed.min_password_length'),
+                    'min_password_length' => config('pixelfed.min_password_length'),
-					'max_account_size' => config('pixelfed.max_account_size')
+                    'max_account_size' => config('pixelfed.max_account_size')
-				],
+                ],
 
-				'username' => [
+                'username' => [
-					'remote' => [
+                    'remote' => [
-						'formats' => config('instance.username.remote.formats'),
+                        'formats' => config('instance.username.remote.formats'),
-						'format' => config('instance.username.remote.format'),
+                        'format' => config('instance.username.remote.format'),
-						'custom' => config('instance.username.remote.custom')
+                        'custom' => config('instance.username.remote.custom')
-					]
+                    ]
-				],
+                ],
 
-				'features' => [
+                'features' => [
-					'timelines' => [
+                    'timelines' => [
-						'local' => true,
+                        'local' => true,
-						'network' => (bool) config('federation.network_timeline'),
+                        'network' => (bool) config('federation.network_timeline'),
-					],
+                    ],
-					'mobile_apis' => (bool) config_cache('pixelfed.oauth_enabled'),
+                    'mobile_apis' => (bool) config_cache('pixelfed.oauth_enabled'),
-					'stories' => (bool) config_cache('instance.stories.enabled'),
+                    'stories' => (bool) config_cache('instance.stories.enabled'),
-					'video'	=> Str::contains(config_cache('pixelfed.media_types'), 'video/mp4'),
+                    'video' => Str::contains(config_cache('pixelfed.media_types'), 'video/mp4'),
-					'import' => [
+                    'import' => [
-						'instagram' => (bool) config_cache('pixelfed.import.instagram.enabled'),
+                        'instagram' => (bool) config_cache('pixelfed.import.instagram.enabled'),
-						'mastodon' => false,
+                        'mastodon' => false,
-						'pixelfed' => false
+                        'pixelfed' => false
-					],
+                    ],
-					'label' => [
+                    'label' => [
-						'covid' => [
+                        'covid' => [
-							'enabled' => (bool) config('instance.label.covid.enabled'),
+                            'enabled' => (bool) config('instance.label.covid.enabled'),
-							'org' => config('instance.label.covid.org'),
+                            'org' => config('instance.label.covid.org'),
-							'url' => config('instance.label.covid.url'),
+                            'url' => config('instance.label.covid.url'),
-						]
+                        ]
-					]
+                    ],
-				]
+                    'hls' => $hls
-			];
+                ]
-		});
+            ];
-	}
+        });
+    }
 
-	public static function json() {
+    public static function json() {
-		return json_encode(self::get(), JSON_FORCE_OBJECT);
+        return json_encode(self::get(), JSON_FORCE_OBJECT);
-	}
+    }
 }

composer.json

@@ -20,6 +20,7 @@
 		"doctrine/dbal": "^3.0",
 		"intervention/image": "^2.4",
 		"jenssegers/agent": "^2.6",
+		"laravel-notification-channels/webpush": "^7.1",
 		"laravel/framework": "^10.0",
 		"laravel/helpers": "^1.1",
 		"laravel/horizon": "^5.0",

config/laravel-ffmpeg.php

@@ -3,8 +3,7 @@
 return [
     'ffmpeg' => [
         'binaries' => env('FFMPEG_BINARIES', 'ffmpeg'),
-
+        'threads' => env('FFMPEG_THREADS', false),
-        'threads' => 12,   // set to false to disable the default 'threads' filter
     ],
 
     'ffprobe' => [
@@ -18,4 +17,6 @@ return [
     'temporary_files_root' => env('FFMPEG_TEMPORARY_FILES_ROOT', sys_get_temp_dir()),
 
     'temporary_files_encrypted_hls' => env('FFMPEG_TEMPORARY_ENCRYPTED_HLS', env('FFMPEG_TEMPORARY_FILES_ROOT', sys_get_temp_dir())),
+
+    'min_hls_version' => env('FFMPEG_MIN_HLS_VERSION', '4.3.0'),
 ];

config/mail.php

@@ -4,45 +4,89 @@ return [
 
     /*
     |--------------------------------------------------------------------------
-    | Mail Driver
+    | Default Mailer
     |--------------------------------------------------------------------------
     |
-    | Laravel supports both SMTP and PHP's "mail" function as drivers for the
+    | This option controls the default mailer that is used to send any email
-    | sending of e-mail. You may specify which one you're using throughout
+    | messages sent by your application. Alternative mailers may be setup
-    | your application here. By default, Laravel is setup for SMTP mail.
+    | and used as needed; however, this mailer will be used by default.
-    |
-    | Supported: "smtp", "sendmail", "mailgun", "mandrill", "ses",
-    |            "sparkpost", "log", "array"
     |
     */
 
-    'driver' => env('MAIL_DRIVER', 'smtp'),
+    'default' => env('MAIL_DRIVER', 'smtp'),
 
     /*
     |--------------------------------------------------------------------------
-    | SMTP Host Address
+    | Mailer Configurations
     |--------------------------------------------------------------------------
     |
-    | Here you may provide the host address of the SMTP server used by your
+    | Here you may configure all of the mailers used by your application plus
-    | applications. A default option is provided that is compatible with
+    | their respective settings. Several examples have been configured for
-    | the Mailgun mail service which will provide reliable deliveries.
+    | you and you are free to add your own as your application requires.
     |
-    */
+    | Laravel supports a variety of mail "transport" drivers to be used while
-
+    | sending an e-mail. You will specify which one you are using for your
-    'host' => env('MAIL_HOST', 'smtp.mailgun.org'),
+    | mailers below. You are free to add additional mailers as required.
-
-    /*
-    |--------------------------------------------------------------------------
-    | SMTP Host Port
-    |--------------------------------------------------------------------------
     |
-    | This is the SMTP port used by your application to deliver e-mails to
+    | Supported: "smtp", "sendmail", "mailgun", "ses", "ses-v2",
-    | users of the application. Like the host we have set this value to
+    |            "postmark", "log", "array", "failover"
-    | stay compatible with the Mailgun e-mail application by default.
     |
     */
 
-    'port' => env('MAIL_PORT', 587),
+    'mailers' => [
+        'smtp' => [
+            'transport' => 'smtp',
+            'url' => env('MAIL_URL'),
+            'host' => env('MAIL_HOST', 'smtp.mailgun.org'),
+            'port' => env('MAIL_PORT', 587),
+            'encryption' => env('MAIL_ENCRYPTION', 'tls'),
+            'username' => env('MAIL_USERNAME'),
+            'password' => env('MAIL_PASSWORD'),
+            'timeout' => null,
+            'local_domain' => env('MAIL_EHLO_DOMAIN'),
+            'verify_peer' => env('MAIL_SMTP_VERIFY_PEER', true),
+        ],
+
+        'ses' => [
+            'transport' => 'ses',
+        ],
+
+        'mailgun' => [
+            'transport' => 'mailgun',
+            // 'client' => [
+            //     'timeout' => 5,
+            // ],
+        ],
+
+        'postmark' => [
+            'transport' => 'postmark',
+            // 'client' => [
+            //     'timeout' => 5,
+            // ],
+        ],
+
+        'sendmail' => [
+            'transport' => 'sendmail',
+            'path' => env('MAIL_SENDMAIL_PATH', '/usr/sbin/sendmail -bs -i'),
+        ],
+
+        'log' => [
+            'transport' => 'log',
+            'channel' => env('MAIL_LOG_CHANNEL'),
+        ],
+
+        'array' => [
+            'transport' => 'array',
+        ],
+
+        'failover' => [
+            'transport' => 'failover',
+            'mailers' => [
+                'smtp',
+                'log',
+            ],
+        ],
+    ],
 
     /*
     |--------------------------------------------------------------------------
@@ -57,63 +101,9 @@ return [
 
     'from' => [
         'address' => env('MAIL_FROM_ADDRESS', 'hello@example.com'),
-        'name'    => env('MAIL_FROM_NAME', 'Example'),
+        'name' => env('MAIL_FROM_NAME', 'Example'),
     ],
 
-    /*
-    |--------------------------------------------------------------------------
-    | E-Mail Encryption Protocol
-    |--------------------------------------------------------------------------
-    |
-    | Here you may specify the encryption protocol that should be used when
-    | the application send e-mail messages. A sensible default using the
-    | transport layer security protocol should provide great security.
-    |
-    */
-
-    'encryption' => env('MAIL_ENCRYPTION', 'tls'),
-
-    /*
-    |--------------------------------------------------------------------------
-    | SMTP Server Username
-    |--------------------------------------------------------------------------
-    |
-    | If your SMTP server requires a username for authentication, you should
-    | set it here. This will get used to authenticate with your server on
-    | connection. You may also set the "password" value below this one.
-    |
-    */
-
-    'username' => env('MAIL_USERNAME'),
-    'password' => env('MAIL_PASSWORD'),
-    
-    
-    /*
-    |--------------------------------------------------------------------------
-    | SMTP EHLO Domain
-    |--------------------------------------------------------------------------
-    |
-    | Some SMTP servers require to present a known domain in order to 
-    | allow sending through its relay. (ie: Google Workspace)
-    | This will use the MAIL_SMTP_EHLO env variable to avoid the 421 error
-    | if not present by authenticating the sender domain instead the host.
-    |
-    */
-    'local_domain' => env('MAIL_EHLO_DOMAIN'),
-
-    /*
-    |--------------------------------------------------------------------------
-    | Sendmail System Path
-    |--------------------------------------------------------------------------
-    |
-    | When using the "sendmail" driver to send e-mails, we will need to know
-    | the path to where Sendmail lives on this server. A default path has
-    | been provided here, which will work well on most of your systems.
-    |
-    */
-
-    'sendmail' => '/usr/sbin/sendmail -bs',
-
     /*
     |--------------------------------------------------------------------------
     | Markdown Mail Settings

config/media.php

@@ -22,5 +22,39 @@ return [
 
             'resilient_mode' => env('ALT_PRI_ENABLED', false) || env('ALT_SEC_ENABLED', false),
         ],
+    ],
+
+    'hls' => [
+        /*
+        |--------------------------------------------------------------------------
+        | Enable HLS
+        |--------------------------------------------------------------------------
+        |
+        | Enable optional HLS support, required for video p2p support. Requires FFMPEG
+        | Disabled by default.
+        |
+        */
+        'enabled' => env('MEDIA_HLS_ENABLED', false),
+
+        'debug' => env('MEDIA_HLS_DEBUG', false),
+
+        /*
+        |--------------------------------------------------------------------------
+        | Enable Video P2P support
+        |--------------------------------------------------------------------------
+        |
+        | Enable optional video p2p support. Requires FFMPEG + HLS
+        | Disabled by default.
+        |
+        */
+        'p2p' => env('MEDIA_HLS_P2P', false),
+
+        'p2p_debug' => env('MEDIA_HLS_P2P_DEBUG', false),
+
+        'bitrate' => env('MEDIA_HLS_BITRATE', 1000),
+
+        'tracker' => env('MEDIA_HLS_P2P_TRACKER', 'wss://tracker.webtorrent.dev'),
+
+        'ice' => env('MEDIA_HLS_P2P_ICE_SERVER', 'stun:stun.l.google.com:19302'),
     ]
 ];

config/pixelfed.php

@@ -286,4 +286,9 @@ return [
 	'max_altext_length' => env('PF_MEDIA_MAX_ALTTEXT_LENGTH', 1000),
 
 	'allow_app_registration' => env('PF_ALLOW_APP_REGISTRATION', true),
+
+    'app_registration_rate_limit_attempts' => env('PF_IAR_RL_ATTEMPTS', 3),
+    'app_registration_rate_limit_decay' => env('PF_IAR_RL_DECAY', 1800),
+    'app_registration_confirm_rate_limit_attempts' => env('PF_IARC_RL_ATTEMPTS', 20),
+    'app_registration_confirm_rate_limit_decay' => env('PF_IARC_RL_ATTEMPTS', 1800),
 ];

config/webpush.php

@@ -0,0 +1,48 @@
+<?php
+
+return [
+
+    /**
+     * These are the keys for authentication (VAPID).
+     * These keys must be safely stored and should not change.
+     */
+    'vapid' => [
+        'subject' => env('VAPID_SUBJECT'),
+        'public_key' => env('VAPID_PUBLIC_KEY'),
+        'private_key' => env('VAPID_PRIVATE_KEY'),
+        'pem_file' => env('VAPID_PEM_FILE'),
+    ],
+
+    /**
+     * This is model that will be used to for push subscriptions.
+     */
+    'model' => \NotificationChannels\WebPush\PushSubscription::class,
+
+    /**
+     * This is the name of the table that will be created by the migration and
+     * used by the PushSubscription model shipped with this package.
+     */
+    'table_name' => env('WEBPUSH_DB_TABLE', 'push_subscriptions'),
+
+    /**
+     * This is the database connection that will be used by the migration and
+     * the PushSubscription model shipped with this package.
+     */
+    'database_connection' => env('WEBPUSH_DB_CONNECTION', env('DB_CONNECTION', 'mysql')),
+
+    /**
+     * The Guzzle client options used by Minishlink\WebPush.
+     */
+    'client_options' => [],
+
+    /**
+     * Google Cloud Messaging.
+     *
+     * @deprecated
+     */
+    'gcm' => [
+        'key' => env('GCM_KEY'),
+        'sender_id' => env('GCM_SENDER_ID'),
+    ],
+
+];

database/migrations/2023_11_13_062429_add_followers_count_index_to_profiles_table.php

@@ -0,0 +1,34 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+    /**
+     * Run the migrations.
+     */
+    public function up(): void
+    {
+        Schema::table('profiles', function (Blueprint $table) {
+           $table->index('followers_count', 'profiles_followers_count_index');
+           $table->index('following_count', 'profiles_following_count_index');
+           $table->index('status_count', 'profiles_status_count_index');
+           $table->index('is_private', 'profiles_is_private_index');
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     */
+    public function down(): void
+    {
+        Schema::table('profiles', function (Blueprint $table) {
+            $table->dropIndex('profiles_followers_count_index');
+            $table->dropIndex('profiles_following_count_index');
+            $table->dropIndex('profiles_status_count_index');
+            $table->dropIndex('profiles_is_private_index');
+        });
+    }
+};

database/migrations/2023_11_16_124107_create_hashtag_related_table.php

@@ -0,0 +1,33 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+    /**
+     * Run the migrations.
+     */
+    public function up(): void
+    {
+        Schema::create('hashtag_related', function (Blueprint $table) {
+            $table->bigIncrements('id');
+            $table->bigInteger('hashtag_id')->unsigned()->unique()->index();
+            $table->json('related_tags')->nullable();
+            $table->bigInteger('agg_score')->unsigned()->nullable()->index();
+            $table->timestamp('last_calculated_at')->nullable()->index();
+            $table->timestamp('last_moderated_at')->nullable()->index();
+            $table->boolean('skip_refresh')->default(false)->index();
+            $table->timestamps();
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     */
+    public function down(): void
+    {
+        Schema::dropIfExists('hashtag_related');
+    }
+};

database/migrations/2023_11_26_082439_add_state_and_score_to_places_table.php

@@ -0,0 +1,34 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+    /**
+     * Run the migrations.
+     */
+    public function up(): void
+    {
+        Schema::table('places', function (Blueprint $table) {
+            $table->string('state')->nullable()->index()->after('name');
+            $table->tinyInteger('score')->default(0)->index()->after('long');
+            $table->unsignedBigInteger('cached_post_count')->nullable();
+            $table->timestamp('last_checked_at')->nullable()->index();
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     */
+    public function down(): void
+    {
+        Schema::table('places', function (Blueprint $table) {
+            $table->dropColumn('state');
+            $table->dropColumn('score');
+            $table->dropColumn('cached_post_count');
+            $table->dropColumn('last_checked_at');
+        });
+    }
+};

database/migrations/2023_12_04_041631_create_push_subscriptions_table.php

@@ -0,0 +1,36 @@
+<?php
+
+use Illuminate\Support\Facades\Schema;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Database\Migrations\Migration;
+
+class CreatePushSubscriptionsTable extends Migration
+{
+    /**
+     * Run the migrations.
+     *
+     * @return void
+     */
+    public function up()
+    {
+        Schema::connection(config('webpush.database_connection'))->create(config('webpush.table_name'), function (Blueprint $table) {
+            $table->bigIncrements('id');
+            $table->morphs('subscribable');
+            $table->string('endpoint', 500)->unique();
+            $table->string('public_key')->nullable();
+            $table->string('auth_token')->nullable();
+            $table->string('content_encoding')->nullable();
+            $table->timestamps();
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     *
+     * @return void
+     */
+    public function down()
+    {
+        Schema::connection(config('webpush.database_connection'))->dropIfExists(config('webpush.table_name'));
+    }
+}

package.json

@@ -34,9 +34,12 @@
 	},
 	"dependencies": {
 		"@fancyapps/fancybox": "^3.5.7",
+		"@hcaptcha/vue-hcaptcha": "^1.3.0",
+		"@peertube/p2p-media-loader-core": "^1.0.14",
+		"@peertube/p2p-media-loader-hlsjs": "^1.0.14",
 		"@trevoreyre/autocomplete-vue": "^2.2.0",
 		"@web3-storage/parse-link-header": "^3.1.0",
-		"@zip.js/zip.js": "^2.7.14",
+		"@zip.js/zip.js": "^2.7.24",
 		"animate.css": "^4.1.0",
 		"bigpicture": "^2.6.2",
 		"blurhash": "^1.1.3",