aiden
/
pixelfed
mirror of https://github.com/pixelfed/pixelfed
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add DB fallback, cache TTL, and configurable dropoffs for all timelines

Browse Source 
Improvements to all three timeline services (Home, Public, Network):

1. DB fallback on cache exhaustion:
   - Add fallbackMaxId() to each service that queries the DB when
     Redis returns empty during max_id pagination
   - Users can now scroll infinitely regardless of cache size

2. Cache TTL (24 hours):
   - All sorted sets now expire after 24h, triggering a fresh rebuild
   - Prevents stale data from accumulating
   - Configurable via PF_HOME_TIMELINE_CACHE_TTL,
     INSTANCE_PUBLIC_TIMELINE_CACHE_TTL, INSTANCE_NETWORK_TIMELINE_CACHE_TTL

3. Configurable dropoff limits:
   - Home: 400 -> 800 (per user, via PF_HOME_TIMELINE_CACHE_DROPOFF)
   - Public: 400 -> 10000 (via INSTANCE_PUBLIC_TIMELINE_CACHE_DROPOFF)
   - Network: already 10000

4. Increased get() cap from 100 to 500 in all services
pull/6586/head
Your Name 3 months ago
parent b08da33a3c
commit 8d156bda1a
6 changed files with 111 additions and 12 deletions
Show Stats Download Patch File Download Diff File

9
app/Http/Controllers/Api/ApiV1Controller.php
Unescape Escape View File

@ -2650,6 +2650,9 @@ class ApiV1Controller extends Controller
$res = HomeTimelineService::getRankedMinId($pid, $min ?? 0, $paddedLimit);
} else {
$res = HomeTimelineService::getRankedMaxId($pid, $max ?? 0, $paddedLimit);
if (empty($res) && $max) {
$res = HomeTimelineService::fallbackMaxId($pid, $max, $paddedLimit);
}
}
} else {
$res = HomeTimelineService::get($pid, 0, $paddedLimit);
@ -2824,6 +2827,9 @@ class ApiV1Controller extends Controller
if ($max) {
$feed = NetworkTimelineService::getRankedMaxId($max, $limit + 5);
if (empty($feed)) {
$feed = NetworkTimelineService::fallbackMaxId($max, $limit + 5);
}
} elseif ($min) {
$feed = NetworkTimelineService::getRankedMinId($min, $limit + 5);
} else {
@ -2838,6 +2844,9 @@ class ApiV1Controller extends Controller
if ($max) {
$feed = PublicTimelineService::getRankedMaxId($max, $limit + 5);
if (empty($feed)) {
$feed = PublicTimelineService::fallbackMaxId($max, $limit + 5);
}
} elseif ($min) {
$feed = PublicTimelineService::getRankedMinId($min, $limit + 5);
} else {

6
app/Http/Controllers/PublicApiController.php
Unescape Escape View File

@ -240,6 +240,9 @@ class PublicApiController extends Controller
if ($max) {
$feed = PublicTimelineService::getRankedMaxId($max, $limit);
if (empty($feed)) {
$feed = PublicTimelineService::fallbackMaxId($max, $limit);
}
} elseif ($min) {
$feed = PublicTimelineService::getRankedMinId($min, $limit);
} else {
@ -450,6 +453,9 @@ class PublicApiController extends Controller
if ($max) {
$feed = NetworkTimelineService::getRankedMaxId($max, $limit);
if (empty($feed)) {
$feed = NetworkTimelineService::fallbackMaxId($max, $limit);
}
} elseif ($min) {
$feed = NetworkTimelineService::getRankedMinId($min, $limit);
} else {

35
app/Services/HomeTimelineService.php
Unescape Escape View File

@ -16,8 +16,8 @@ class HomeTimelineService
public static function get($id, $start = 0, $stop = 10)
{
if ($stop > 100) {
$stop = 100;
if ($stop > 500) {
$stop = 500;
}
return Redis::zrevrange(self::CACHE_KEY.$id, $start, $stop);
@ -49,11 +49,18 @@ class HomeTimelineService
public static function add($id, $val)
{
if (self::count($id) >= 400) {
if (self::count($id) >= config('instance.timeline.home.cache_dropoff', 800)) {
Redis::zpopmin(self::CACHE_KEY.$id);
}
return Redis::zadd(self::CACHE_KEY.$id, $val, $val);
$result = Redis::zadd(self::CACHE_KEY.$id, $val, $val);
$ttl = Redis::ttl(self::CACHE_KEY.$id);
if ($ttl < 0) {
Redis::expire(self::CACHE_KEY.$id, config('instance.timeline.home.cache_ttl', 86400));
}
return $result;
}
public static function rem($id, $val)
@ -66,6 +73,26 @@ class HomeTimelineService
return Redis::zcard(self::CACHE_KEY.$id);
}
public static function fallbackMaxId($id, $max, $limit = 10)
{
$following = Cache::remember('profile:following:'.$id, 1209600, function () use ($id) {
$following = Follower::whereProfileId($id)->pluck('following_id');
return $following->push($id)->toArray();
});
return Status::where('id', '<', $max)
->whereIntegerInRaw('profile_id', $following)
->whereNull(['in_reply_to_id', 'reblog_of_id'])
->whereIn('type', ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])
->whereIn('visibility', ['public', 'unlisted', 'private'])
->orderByDesc('id')
->limit($limit)
->pluck('id')
->values()
->toArray();
}
public static function warmCache($id, $force = false, $limit = 100, $returnIds = false)
{
if (self::count($id) == 0 || $force == true) {

34
app/Services/NetworkTimelineService.php
Unescape Escape View File

@ -11,8 +11,8 @@ class NetworkTimelineService
public static function get($start = 0, $stop = 10)
{
if ($stop > 100) {
$stop = 100;
if ($stop > 500) {
$stop = 500;
}
return Redis::zrevrange(self::CACHE_KEY, $start, $stop);
@ -48,7 +48,14 @@ class NetworkTimelineService
Redis::zpopmin(self::CACHE_KEY);
}
return Redis::zadd(self::CACHE_KEY, $val, $val);
$result = Redis::zadd(self::CACHE_KEY, $val, $val);
$ttl = Redis::ttl(self::CACHE_KEY);
if ($ttl < 0) {
Redis::expire(self::CACHE_KEY, config('instance.timeline.network.cache_ttl', 86400));
}
return $result;
}
public static function rem($val)
@ -66,6 +73,27 @@ class NetworkTimelineService
return Redis::zcard(self::CACHE_KEY);
}
public static function fallbackMaxId($max, $limit = 10)
{
$hideNsfw = config('instance.hide_nsfw_on_public_feeds');
$maxHoursOld = config('instance.timeline.network.max_hours_old', 2160);
return Status::where('id', '<', $max)
->whereNotNull('uri')
->whereNull(['in_reply_to_id', 'reblog_of_id'])
->whereIn('type', ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])
->whereScope('public')
->where('created_at', '>', now()->subHours($maxHoursOld))
->when($hideNsfw, function ($q) {
return $q->where('is_nsfw', false);
})
->orderByDesc('id')
->limit($limit)
->pluck('id')
->values()
->toArray();
}
public static function deleteByProfileId($profileId)
{
$res = Redis::zrange(self::CACHE_KEY, 0, '-1');

33
app/Services/PublicTimelineService.php
Unescape Escape View File

@ -11,8 +11,8 @@ class PublicTimelineService
public static function get($start = 0, $stop = 10)
{
if ($stop > 100) {
$stop = 100;
if ($stop > 500) {
$stop = 500;
}
return Redis::zrevrange(self::CACHE_KEY, $start, $stop);
@ -44,11 +44,18 @@ class PublicTimelineService
public static function add($val)
{
if (self::count() > 400) {
if (self::count() > config('instance.timeline.local.cache_dropoff', 10000)) {
Redis::zpopmin(self::CACHE_KEY);
}
return Redis::zadd(self::CACHE_KEY, $val, $val);
$result = Redis::zadd(self::CACHE_KEY, $val, $val);
$ttl = Redis::ttl(self::CACHE_KEY);
if ($ttl < 0) {
Redis::expire(self::CACHE_KEY, config('instance.timeline.local.cache_ttl', 86400));
}
return $result;
}
public static function rem($val)
@ -66,6 +73,24 @@ class PublicTimelineService
return Redis::zcard(self::CACHE_KEY);
}
public static function fallbackMaxId($max, $limit = 10)
{
$hideNsfw = config('instance.hide_nsfw_on_public_feeds');
return Status::where('id', '<', $max)
->whereNull(['uri', 'in_reply_to_id', 'reblog_of_id'])
->whereIn('type', ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])
->whereScope('public')
->when($hideNsfw, function ($q) {
return $q->where('is_nsfw', false);
})
->orderByDesc('id')
->limit($limit)
->pluck('id')
->values()
->toArray();
}
public static function deleteByProfileId($profileId)
{
$res = Redis::zrange(self::CACHE_KEY, 0, '-1');

6
config/instance.php
Unescape Escape View File

@ -26,15 +26,19 @@ return [
'timeline' => [
'home' => [
'cached' => env('PF_HOME_TIMELINE_CACHE', false),
'cache_ttl' => env('PF_HOME_TIMELINE_CACHE_TTL', 900),
'cache_ttl' => env('PF_HOME_TIMELINE_CACHE_TTL', 86400),
'cache_dropoff' => env('PF_HOME_TIMELINE_CACHE_DROPOFF', 800),
],
'local' => [
'is_public' => env('INSTANCE_PUBLIC_LOCAL_TIMELINE', false),
'cache_dropoff' => env('INSTANCE_PUBLIC_TIMELINE_CACHE_DROPOFF', 10000),
'cache_ttl' => env('INSTANCE_PUBLIC_TIMELINE_CACHE_TTL', 86400),
],
'network' => [
'cache_dropoff' => env('INSTANCE_NETWORK_TIMELINE_CACHE_DROPOFF', 10000),
'cache_ttl' => env('INSTANCE_NETWORK_TIMELINE_CACHE_TTL', 86400),
'max_hours_old' => env('INSTANCE_NETWORK_TIMELINE_CACHE_MAX_HOUR_INGEST', 2160),
],
],

Write Preview
Loading…
Cancel
Save