aiden
/
pixelfed
mirror of https://github.com/pixelfed/pixelfed
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove cached timeline feature flags, make caching always-on

Browse Source 
Remove EXP_CPT and EXP_CHT config entries from exp.php and eliminate
all conditional branches. The Redis-cached timeline paths are now the
only code paths:

- ApiV1Controller: Remove ~150 lines of DB-query home timeline fallback
- PublicApiController: Remove ~70 lines of DB-query public timeline fallback
- StatusObserver: Always dispatch feed removal on status delete
- StatusEntityLexer: Always dispatch feed insert on new status
- FeedRemoveDomainPipeline: Remove early-return guard
- Diagnostics: Remove EXP_CPT row

The cached timeline code is mature with proper cold-boot handling
and Redis is already a hard dependency for Pixelfed.
pull/6581/head
Your Name 3 months ago
parent b6ad52b522
commit 5fbe5682f2
7 changed files with 86 additions and 382 deletions
Show Stats Download Patch File Download Diff File

289
app/Http/Controllers/Api/ApiV1Controller.php
Unescape Escape View File

@ -2644,258 +2644,69 @@ class ApiV1Controller extends Controller
return in_array('home', $filter->context);
});
if (config('exp.cached_home_timeline')) {
$paddedLimit = $includeReblogs ? $limit + 10 : $limit + 50;
if ($min || $max) {
if ($request->has('min_id')) {
$res = HomeTimelineService::getRankedMinId($pid, $min ?? 0, $paddedLimit);
} else {
$res = HomeTimelineService::getRankedMaxId($pid, $max ?? 0, $paddedLimit);
}
$paddedLimit = $includeReblogs ? $limit + 10 : $limit + 50;
if ($min || $max) {
if ($request->has('min_id')) {
$res = HomeTimelineService::getRankedMinId($pid, $min ?? 0, $paddedLimit);
} else {
$res = HomeTimelineService::get($pid, 0, $paddedLimit);
$res = HomeTimelineService::getRankedMaxId($pid, $max ?? 0, $paddedLimit);
}
} else {
$res = HomeTimelineService::get($pid, 0, $paddedLimit);
}
if (! $res) {
$res = Cache::has('pf:services:apiv1:home:cached:coldbootcheck:'.$pid);
if (! $res) {
$res = Cache::has('pf:services:apiv1:home:cached:coldbootcheck:'.$pid);
if (! $res) {
Cache::set('pf:services:apiv1:home:cached:coldbootcheck:'.$pid, 1, 86400);
FeedWarmCachePipeline::dispatchSync($pid);
return response()->json([], 206);
} else {
Cache::set('pf:services:apiv1:home:cached:coldbootcheck:'.$pid, 1, 86400);
return response()->json([], 206);
}
}
$res = collect($res)
->map(function ($id) use ($napi) {
return $napi ? StatusService::get($id, false) : StatusService::getMastodon($id, false);
})
->filter(function ($res) {
return $res && isset($res['account']);
})
->filter(function ($s) use ($includeReblogs) {
return $includeReblogs ? true : $s['reblog'] == null;
})
->map(function ($status) use ($homeFilters) {
$filterResults = CustomFilter::applyCachedFilters($homeFilters, $status);
if (! empty($filterResults)) {
$status['filtered'] = $filterResults;
$shouldHide = collect($filterResults)->contains(function ($result) {
return $result['filter']['filter_action'] === 'hide';
});
if ($shouldHide) {
return null;
}
}
return $status;
})
->filter()
->take($limit)
->map(function ($status) use ($pid) {
if ($pid) {
$status['favourited'] = (bool) LikeService::liked($pid, $status['id']);
$status['reblogged'] = (bool) ReblogService::get($pid, $status['id']);
$status['bookmarked'] = (bool) BookmarkService::get($pid, $status['id']);
}
return $status;
})
->values();
$baseUrl = config('app.url').'/api/v1/timelines/home?limit='.$limit.'&';
$minId = $res->map(function ($s) {
return ['id' => $s['id']];
})->min('id');
$maxId = $res->map(function ($s) {
return ['id' => $s['id']];
})->max('id');
if ($minId == $maxId) {
$minId = null;
}
if ($maxId && $res->count() >= $limit) {
$link = '<'.$baseUrl.'max_id='.$minId.'>; rel="next"';
}
Cache::set('pf:services:apiv1:home:cached:coldbootcheck:'.$pid, 1, 86400);
FeedWarmCachePipeline::dispatchSync($pid);
if ($minId) {
$link = '<'.$baseUrl.'min_id='.$maxId.'>; rel="prev"';
}
return response()->json([], 206);
} else {
Cache::set('pf:services:apiv1:home:cached:coldbootcheck:'.$pid, 1, 86400);
if ($maxId && $minId) {
$link = '<'.$baseUrl.'max_id='.$minId.'>; rel="next",<'.$baseUrl.'min_id='.$maxId.'>; rel="prev"';
return response()->json([], 206);
}
$headers = isset($link) ? ['Link' => $link] : [];
return $this->json($res->toArray(), 200, $headers);
}
$following = Cache::remember('profile:following:'.$pid, 1209600, function () use ($pid) {
$following = Follower::whereProfileId($pid)->pluck('following_id');
return $following->push($pid)->toArray();
});
$muted = UserFilterService::mutes($pid);
if ($muted && count($muted)) {
$following = array_diff($following, $muted);
}
if ($min || $max) {
$dir = $min ? '>' : '<';
$id = $min ?? $max;
$res = Status::select(
'id',
'profile_id',
'type',
'visibility',
'in_reply_to_id',
'reblog_of_id'
)
->where('id', $dir, $id)
->whereNull($nullFields)
->whereIntegerInRaw('profile_id', $following)
->whereIn('type', $inTypes)
->whereIn('visibility', ['public', 'unlisted', 'private'])
->orderByDesc('id')
->take(($limit * 2))
->get()
->map(function ($s) use ($pid, $napi) {
try {
$account = $napi ? AccountService::get($s['profile_id'], true) : AccountService::getMastodon($s['profile_id'], true);
if (! $account) {
return false;
}
$status = $napi ? StatusService::get($s['id'], false) : StatusService::getMastodon($s['id'], false);
if (! $status || ! isset($status['account']) || ! isset($status['account']['id'])) {
return false;
}
} catch (\Exception $e) {
return false;
}
$status['account'] = $account;
if ($pid) {
$status['favourited'] = (bool) LikeService::liked($pid, $s['id']);
$status['reblogged'] = (bool) ReblogService::get($pid, $status['id']);
$status['bookmarked'] = (bool) BookmarkService::get($pid, $status['id']);
}
return $status;
})
->filter(function ($status) {
return $status && isset($status['account']);
})
->map(function ($status) use ($pid) {
if (! empty($status['reblog'])) {
$status['reblog']['favourited'] = (bool) LikeService::liked($pid, $status['reblog']['id']);
$status['reblog']['reblogged'] = (bool) ReblogService::get($pid, $status['reblog']['id']);
$status['bookmarked'] = (bool) BookmarkService::get($pid, $status['id']);
}
return $status;
})
->map(function ($status) use ($homeFilters) {
$filterResults = CustomFilter::applyCachedFilters($homeFilters, $status);
if (! empty($filterResults)) {
$status['filtered'] = $filterResults;
$shouldHide = collect($filterResults)->contains(function ($result) {
return $result['filter']['filter_action'] === 'hide';
});
if ($shouldHide) {
return null;
}
}
return $status;
})
->filter()
->take($limit)
->values();
} else {
$res = Status::select(
'id',
'profile_id',
'type',
'visibility',
'in_reply_to_id',
'reblog_of_id',
)
->whereNull($nullFields)
->whereIntegerInRaw('profile_id', $following)
->whereIn('type', $inTypes)
->whereIn('visibility', ['public', 'unlisted', 'private'])
->orderByDesc('id')
->take(($limit * 2))
->get()
->map(function ($s) use ($pid, $napi) {
try {
$account = $napi ? AccountService::get($s['profile_id'], true) : AccountService::getMastodon($s['profile_id'], true);
if (! $account) {
return false;
}
$status = $napi ? StatusService::get($s['id'], false) : StatusService::getMastodon($s['id'], false);
if (! $status || ! isset($status['account']) || ! isset($status['account']['id'])) {
return false;
}
} catch (\Exception $e) {
return false;
}
$status['account'] = $account;
$res = collect($res)
->map(function ($id) use ($napi) {
return $napi ? StatusService::get($id, false) : StatusService::getMastodon($id, false);
})
->filter(function ($res) {
return $res && isset($res['account']);
})
->filter(function ($s) use ($includeReblogs) {
return $includeReblogs ? true : $s['reblog'] == null;
})
->map(function ($status) use ($homeFilters) {
$filterResults = CustomFilter::applyCachedFilters($homeFilters, $status);
if ($pid) {
$status['favourited'] = (bool) LikeService::liked($pid, $s['id']);
$status['reblogged'] = (bool) ReblogService::get($pid, $status['id']);
$status['bookmarked'] = (bool) BookmarkService::get($pid, $status['id']);
}
if (! empty($filterResults)) {
$status['filtered'] = $filterResults;
$shouldHide = collect($filterResults)->contains(function ($result) {
return $result['filter']['filter_action'] === 'hide';
});
return $status;
})
->filter(function ($status) {
return $status && isset($status['account']);
})
->map(function ($status) use ($pid) {
if (! empty($status['reblog'])) {
$status['reblog']['favourited'] = (bool) LikeService::liked($pid, $status['reblog']['id']);
$status['reblog']['reblogged'] = (bool) ReblogService::get($pid, $status['reblog']['id']);
$status['bookmarked'] = (bool) BookmarkService::get($pid, $status['id']);
if ($shouldHide) {
return null;
}
}
return $status;
})
->map(function ($status) use ($homeFilters) {
$filterResults = CustomFilter::applyCachedFilters($homeFilters, $status);
if (! empty($filterResults)) {
$status['filtered'] = $filterResults;
$shouldHide = collect($filterResults)->contains(function ($result) {
return $result['filter']['filter_action'] === 'hide';
});
if ($shouldHide) {
return null;
}
}
return $status;
})
->filter()
->take($limit)
->map(function ($status) use ($pid) {
if ($pid) {
$status['favourited'] = (bool) LikeService::liked($pid, $status['id']);
$status['reblogged'] = (bool) ReblogService::get($pid, $status['id']);
$status['bookmarked'] = (bool) BookmarkService::get($pid, $status['id']);
}
return $status;
})
->filter()
->take($limit)
->values();
}
return $status;
})
->values();
$baseUrl = config('app.url').'/api/v1/timelines/home?limit='.$limit.'&';
$minId = $res->map(function ($s) {
@ -2909,7 +2720,7 @@ class ApiV1Controller extends Controller
$minId = null;
}
if ($maxId) {
if ($maxId && $res->count() >= $limit) {
$link = '<'.$baseUrl.'max_id='.$minId.'>; rel="next"';
}

145
app/Http/Controllers/PublicApiController.php
Unescape Escape View File

@ -232,127 +232,38 @@ class PublicApiController extends Controller
$user = $request->user();
$filtered = $user ? UserFilterService::filters($user->profile_id) : [];
$hideNsfw = config('instance.hide_nsfw_on_public_feeds');
if (config('exp.cached_public_timeline') == false) {
if ($min || $max) {
$dir = $min ? '>' : '<';
$id = $min ?? $max;
$timeline = Status::select(
'id',
'profile_id',
'type',
'scope',
'local'
)
->where('id', $dir, $id)
->whereNull(['in_reply_to_id', 'reblog_of_id'])
->whereIn('type', ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])
->whereLocal(true)
->when($hideNsfw, function ($q, $hideNsfw) {
return $q->where('is_nsfw', false);
})
->whereScope('public')
->orderBy('id', 'desc')
->limit($limit)
->get()
->map(function ($s) use ($user) {
$status = StatusService::getFull($s->id, $user->profile_id);
if (! $status) {
return false;
}
$status['favourited'] = (bool) LikeService::liked($user->profile_id, $s->id);
$status['bookmarked'] = (bool) BookmarkService::get($user->profile_id, $s->id);
$status['reblogged'] = (bool) ReblogService::get($user->profile_id, $s->id);
return $status;
})
->filter(function ($s) use ($filtered) {
return $s && isset($s['account']) && in_array($s['account']['id'], $filtered) == false;
})
->values();
$res = $timeline->toArray();
} else {
$timeline = Status::select(
'id',
'uri',
'caption',
'profile_id',
'type',
'in_reply_to_id',
'reblog_of_id',
'is_nsfw',
'scope',
'local',
'reply_count',
'comments_disabled',
'created_at',
'place_id',
'likes_count',
'reblogs_count',
'updated_at'
)
->whereNull(['in_reply_to_id', 'reblog_of_id'])
->whereIn('type', ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])
->whereLocal(true)
->when($hideNsfw, function ($q, $hideNsfw) {
return $q->where('is_nsfw', false);
})
->whereScope('public')
->orderBy('id', 'desc')
->limit($limit)
->get()
->map(function ($s) use ($user) {
$status = StatusService::getFull($s->id, $user->profile_id);
if (! $status) {
return false;
}
$status['favourited'] = (bool) LikeService::liked($user->profile_id, $s->id);
$status['bookmarked'] = (bool) BookmarkService::get($user->profile_id, $s->id);
$status['reblogged'] = (bool) ReblogService::get($user->profile_id, $s->id);
return $status;
})
->filter(function ($s) use ($filtered) {
return $s && isset($s['account']) && in_array($s['account']['id'], $filtered) == false;
})
->values();
$res = $timeline->toArray();
Cache::remember('api:v1:timelines:public:cache_check', 10368000, function () {
if (PublicTimelineService::count() == 0) {
PublicTimelineService::warmCache(true, 400);
}
} else {
Cache::remember('api:v1:timelines:public:cache_check', 10368000, function () {
if (PublicTimelineService::count() == 0) {
PublicTimelineService::warmCache(true, 400);
}
});
});
if ($max) {
$feed = PublicTimelineService::getRankedMaxId($max, $limit);
} elseif ($min) {
$feed = PublicTimelineService::getRankedMinId($min, $limit);
} else {
$feed = PublicTimelineService::get(0, $limit);
}
if ($max) {
$feed = PublicTimelineService::getRankedMaxId($max, $limit);
} elseif ($min) {
$feed = PublicTimelineService::getRankedMinId($min, $limit);
} else {
$feed = PublicTimelineService::get(0, $limit);
}
$res = collect($feed)
->take($limit)
->map(function ($k) use ($user) {
$status = StatusService::get($k);
if ($status && isset($status['account']) && $user) {
$status['favourited'] = (bool) LikeService::liked($user->profile_id, $k);
$status['bookmarked'] = (bool) BookmarkService::get($user->profile_id, $k);
$status['reblogged'] = (bool) ReblogService::get($user->profile_id, $k);
$status['relationship'] = RelationshipService::get($user->profile_id, $status['account']['id']);
}
$res = collect($feed)
->take($limit)
->map(function ($k) use ($user) {
$status = StatusService::get($k);
if ($status && isset($status['account']) && $user) {
$status['favourited'] = (bool) LikeService::liked($user->profile_id, $k);
$status['bookmarked'] = (bool) BookmarkService::get($user->profile_id, $k);
$status['reblogged'] = (bool) ReblogService::get($user->profile_id, $k);
$status['relationship'] = RelationshipService::get($user->profile_id, $status['account']['id']);
}
return $status;
})
->filter(function ($s) use ($filtered) {
return $s && isset($s['account']) && in_array($s['account']['id'], $filtered) == false;
})
->values()
->toArray();
}
return $status;
})
->filter(function ($s) use ($filtered) {
return $s && isset($s['account']) && in_array($s['account']['id'], $filtered) == false;
})
->values()
->toArray();
return response()->json($res);
}

4
app/Jobs/HomeFeedPipeline/FeedRemoveDomainPipeline.php
Unescape Escape View File

@ -69,10 +69,6 @@ class FeedRemoveDomainPipeline implements ShouldBeUniqueUntilProcessing, ShouldQ
*/
public function handle(): void
{
if (! config('exp.cached_home_timeline')) {
return;
}
if ($this->batch()->cancelled()) {
return;
}

10
app/Jobs/StatusPipeline/StatusEntityLexer.php
Unescape Escape View File

@ -178,12 +178,10 @@ class StatusEntityLexer implements ShouldQueue
$status = $this->status;
StatusService::refresh($status->id);
if (config('exp.cached_home_timeline')) {
if ($status->in_reply_to_id === null &&
in_array($status->scope, ['public', 'unlisted', 'private'])
) {
FeedInsertPipeline::dispatch($status->id, $status->profile_id)->onQueue('feed');
}
if ($status->in_reply_to_id === null &&
in_array($status->scope, ['public', 'unlisted', 'private'])
) {
FeedInsertPipeline::dispatch($status->id, $status->profile_id)->onQueue('feed');
}
$this->deliver();
}

10
app/Observers/StatusObserver.php
Unescape Escape View File

@ -71,12 +71,10 @@ class StatusObserver
ImportService::clearImportedFiles($status->profile_id);
}
if (config('exp.cached_home_timeline')) {
if ($status->uri) {
FeedRemoveRemotePipeline::dispatch($status->id, $status->profile_id)->onQueue('feed');
} else {
FeedRemovePipeline::dispatch($status->id, $status->profile_id)->onQueue('feed');
}
if ($status->uri) {
FeedRemoveRemotePipeline::dispatch($status->id, $status->profile_id)->onQueue('feed');
} else {
FeedRemovePipeline::dispatch($status->id, $status->profile_id)->onQueue('feed');
}
}

5
config/exp.php
Unescape Escape View File

@ -22,11 +22,6 @@ return [
// Poll statuses (alpha)
'polls' => env('EXP_POLLS', false),
// Cached public timeline for larger instances (beta)
'cached_public_timeline' => env('EXP_CPT', true),
'cached_home_timeline' => env('EXP_CHT', true),
// Groups (unreleased)
'gps' => env('EXP_GPS', false),

5
resources/views/admin/diagnostics/home.blade.php
Unescape Escape View File

@ -311,11 +311,6 @@
<td><strong>EXP_POLLS</strong></td>
<td><span>{{config_cache('exp.polls') ? '✅ true' : '❌ false' }}</span></td>
</tr>
<tr>
<td><span class="badge badge-primary">EXP</span></td>
<td><strong>EXP_CPT</strong></td>
<td><span>{{config_cache('exp.cached_public_timeline') ? '✅ true' : '❌ false' }}</span></td>
</tr>
<tr>
<td><span class="badge badge-primary">EXP</span></td>
<td><strong>EXP_GPS</strong></td>

Write Preview
Loading…
Cancel
Save