You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
pixelfed/app/Jobs/StoryPipeline/StoryFetch.php

789 lines
23 KiB
PHP

<?php
namespace App\Jobs\StoryPipeline;
use App\Services\MediaPathService;
use App\Services\StoryIndexService;
use App\Services\StoryService;
use App\Story;
use App\Util\ActivityPub\Helpers;
use App\Util\ActivityPub\Validator\StoryValidator;
use App\Util\Lexer\Bearcap;
use Cache;
use Exception;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Http\Client\ConnectionException;
use Illuminate\Http\Client\RequestException;
use Illuminate\Http\File;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Http;
use Illuminate\Support\Facades\Storage;
use Illuminate\Support\Facades\Validator;
use Illuminate\Support\Str;
use Log;
class StoryFetch implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected $activity;
private const MAX_DURATION = 300;
private const REQUEST_TIMEOUT = 30;
private const MAX_REDIRECTS = 3;
// Rate limiting
public $tries = 3;
public $maxExceptions = 2;
public $backoff = [30, 60, 120];
/**
* Create a new job instance.
*/
public function __construct($activity)
{
$this->activity = $activity;
}
/**
* Execute the job.
*/
public function handle()
{
if (config('app.dev_log')) {
Log::info('StoryFetch job started', ['activity_id' => $this->activity['id'] ?? 'unknown']);
}
try {
$this->processStoryFetch();
} catch (Exception $e) {
if (config('app.dev_log')) {
Log::error('StoryFetch job failed', [
'activity_id' => $this->activity['id'] ?? 'unknown',
'error' => $e->getMessage(),
'trace' => $e->getTraceAsString(),
]);
}
throw $e;
}
}
/**
* Main processing logic
*/
private function processStoryFetch()
{
if (! $this->validateActivityStructure()) {
if (config('app.dev_log')) {
Log::warning('Invalid activity structure', ['activity' => $this->activity]);
}
return;
}
$activity = $this->activity;
$activityId = $activity['id'];
$activityActor = $activity['actor'];
if (! $this->validateDomainConsistency($activityId, $activityActor)) {
if (config('app.dev_log')) {
Log::warning('Domain mismatch detected', [
'activity_id' => $activityId,
'actor' => $activityActor,
]);
}
return;
}
// Rate limiting check
if ($this->isRateLimited($activityActor)) {
if (config('app.dev_log')) {
Log::info('Rate limited', ['actor' => $activityActor]);
}
$this->release(3600); // Retry in 1 hour
return;
}
// Decode and validate bearcap token
$bearcap = $this->validateBearcap($activity['object']['object'] ?? null);
if (! $bearcap) {
return;
}
$url = $bearcap['url'];
$token = $bearcap['token'];
// Additional domain validation for bearcap URL
if (! $this->validateDomainConsistency($activityId, $url)) {
if (config('app.dev_log')) {
Log::warning('Bearcap URL domain mismatch', [
'activity_id' => $activityId,
'bearcap_url' => $url,
]);
}
return;
}
// Fetch and validate story data
$payload = $this->fetchStoryPayload($url, $token);
if (! $payload) {
return;
}
// Validate payload structure and security
if (! $this->validatePayload($payload)) {
return;
}
// Fetch and validate profile
$profile = $this->fetchAndValidateProfile($payload['attributedTo']);
if (! $profile) {
return;
}
// Download and process media with security checks
$mediaResult = $this->downloadAndValidateMedia($payload, $profile);
if (! $mediaResult) {
return;
}
// Create story record with transaction
$this->createStoryRecord($payload, $profile, $mediaResult);
}
/**
* Validate basic activity structure
*/
private function validateActivityStructure(): bool
{
$validator = Validator::make($this->activity, [
'id' => 'required|url|max:2000',
'actor' => 'required|url|max:2000',
'object.object' => 'required|string|max:1000',
]);
return ! $validator->fails();
}
/**
* Enhanced domain consistency validation
*/
private function validateDomainConsistency(string $url1, string $url2): bool
{
$host1 = parse_url($url1, PHP_URL_HOST);
$host2 = parse_url($url2, PHP_URL_HOST);
if (! $host1 || ! $host2) {
return false;
}
// Normalize hosts (remove www prefix if present)
$host1 = ltrim(strtolower($host1), 'www.');
$host2 = ltrim(strtolower($host2), 'www.');
return $host1 === $host2;
}
/**
* Rate limiting check
*/
private function isRateLimited(string $actor): bool
{
$domain = parse_url($actor, PHP_URL_HOST);
$cacheKey = "story_fetch_rate_limit:{$domain}";
$currentCount = Cache::get($cacheKey, 0);
// Allow 5000 story fetches per hour per domain
if ($currentCount >= 5000) {
return true;
}
Cache::put($cacheKey, $currentCount + 1, 3600);
return false;
}
/**
* Enhanced bearcap validation
*/
private function validateBearcap(?string $bearcapString): ?array
{
if (! $bearcapString) {
if (config('app.dev_log')) {
Log::warning('Empty bearcap string');
}
return null;
}
try {
$bearcap = Bearcap::decode($bearcapString);
if (! $bearcap || ! isset($bearcap['url'], $bearcap['token'])) {
if (config('app.dev_log')) {
Log::warning('Invalid bearcap structure');
}
return null;
}
// Validate URL format
if (! filter_var($bearcap['url'], FILTER_VALIDATE_URL)) {
if (config('app.dev_log')) {
Log::warning('Invalid bearcap URL', ['url' => $bearcap['url']]);
}
return null;
}
// Validate token format (should be non-empty)
if (empty($bearcap['token']) || strlen($bearcap['token']) < 10) {
if (config('app.dev_log')) {
Log::warning('Invalid bearcap token');
}
return null;
}
return $bearcap;
} catch (Exception $e) {
if (config('app.dev_log')) {
Log::warning('Bearcap decode failed', ['error' => $e->getMessage()]);
}
return null;
}
}
/**
* Enhanced story payload fetching with security
*/
private function fetchStoryPayload(string $url, string $token): ?array
{
$version = config('pixelfed.version');
$appUrl = config('app.url');
$headers = [
'Accept' => 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"',
'Authorization' => 'Bearer '.$token,
'User-Agent' => "(Pixelfed/{$version}; +{$appUrl})",
];
try {
$response = Http::withHeaders($headers)
->timeout(self::REQUEST_TIMEOUT)
->connectTimeout(10)
->retry(2, 1000)
->withOptions([
'verify' => true,
'max_redirects' => self::MAX_REDIRECTS,
])
->get($url);
if (! $response->successful()) {
if (config('app.dev_log')) {
Log::warning('Story fetch failed', [
'url' => $url,
'status' => $response->status(),
]);
}
return null;
}
$payload = $response->json();
if (! is_array($payload)) {
if (config('app.dev_log')) {
Log::warning('Invalid JSON payload received');
}
return null;
}
return $payload;
} catch (RequestException|ConnectionException $e) {
if (config('app.dev_log')) {
Log::warning('HTTP request failed', [
'url' => $url,
'error' => $e->getMessage(),
]);
}
return null;
} catch (Exception $e) {
if (config('app.dev_log')) {
Log::error('Unexpected error in story fetch', [
'url' => $url,
'error' => $e->getMessage(),
]);
}
return null;
}
}
/**
* Enhanced payload validation
*/
private function validatePayload(array $payload): bool
{
if (! StoryValidator::validate($payload)) {
if (config('app.dev_log')) {
Log::warning('Story validator failed');
}
return false;
}
// Payload security validations
$validator = Validator::make($payload, [
'id' => 'required|url|max:2000',
'attributedTo' => 'required|url|max:2000',
'attachment.url' => 'required|url|max:2000',
'attachment.type' => 'required|in:Image,Video',
'attachment.mediaType' => 'required|string|max:100',
'duration' => 'nullable|integer|min:0|max:'.self::MAX_DURATION,
'published' => 'required|date',
'expiresAt' => 'required|date|after:published',
'can_reply' => 'boolean',
'can_react' => 'boolean',
]);
if ($validator->fails()) {
if (config('app.dev_log')) {
Log::warning('Payload validation failed', ['errors' => $validator->errors()]);
}
return false;
}
// Validate media URL
if (! Helpers::validateUrl($payload['attachment']['url'])) {
if (config('app.dev_log')) {
Log::warning('Invalid attachment URL');
}
return false;
}
// Validate MIME type
$mimeType = $payload['attachment']['mediaType'];
$allowedMimeTypes = $this->getAllowedMimeTypes();
if (! in_array($mimeType, $allowedMimeTypes)) {
if (config('app.dev_log')) {
Log::warning('Invalid MIME type', [
'mime' => $mimeType,
'type' => $payload['attachment']['type'],
'allowed' => $allowedMimeTypes,
]);
}
return false;
}
return true;
}
/**
* Enhanced profile fetching with validation
*/
private function fetchAndValidateProfile(string $attributedTo)
{
try {
$profile = Helpers::profileFetch($attributedTo);
if (! $profile || ! $profile->id) {
if (config('app.dev_log')) {
Log::warning('Profile fetch failed', ['attributed_to' => $attributedTo]);
}
return null;
}
// Check if profile is blocked or suspended
if ($profile->status !== null && in_array($profile->status, ['suspended', 'deleted'])) {
if (config('app.dev_log')) {
Log::info('Profile is suspended/deleted', ['profile_id' => $profile->id]);
}
return null;
}
return $profile;
} catch (Exception $e) {
if (config('app.dev_log')) {
Log::error('Profile fetch error', [
'attributed_to' => $attributedTo,
'error' => $e->getMessage(),
]);
}
return null;
}
}
/**
* Enhanced media download with comprehensive security
*/
private function downloadAndValidateMedia(array $payload, $profile): ?array
{
$mediaUrl = $payload['attachment']['url'];
$ext = strtolower(pathinfo(parse_url($mediaUrl, PHP_URL_PATH), PATHINFO_EXTENSION));
$allowedExtensions = $this->getAllowedExtensions();
if (! in_array($ext, $allowedExtensions)) {
if (config('app.dev_log')) {
Log::warning('Invalid file extension', ['extension' => $ext, 'allowed' => $allowedExtensions]);
}
return null;
}
$fileName = $this->generateSecureFileName($ext);
$storagePath = MediaPathService::story($profile);
$tmpBase = storage_path('app/remcache/');
$tmpPath = $profile->id.'-'.$fileName;
$tmpName = $tmpBase.$tmpPath;
if (! is_dir($tmpBase)) {
mkdir($tmpBase, 0755, true);
}
try {
$contextOptions = [
'ssl' => [
'verify_peer' => true,
'verify_peername' => true,
'allow_self_signed' => false,
'SNI_enabled' => true,
],
'http' => [
'timeout' => self::REQUEST_TIMEOUT,
'max_redirects' => self::MAX_REDIRECTS,
'user_agent' => 'Pixelfed/'.config('pixelfed.version'),
],
];
$ctx = stream_context_create($contextOptions);
$data = $this->downloadWithSizeLimit($mediaUrl, $ctx);
if (! $data) {
return null;
}
if (file_put_contents($tmpName, $data) === false) {
if (config('app.dev_log')) {
Log::error('Failed to write temp file', ['temp_name' => $tmpName]);
}
return null;
}
if (! $this->validateDownloadedFile($tmpName, $payload['attachment']['mediaType'])) {
unlink($tmpName);
return null;
}
$disk = Storage::disk(config('filesystems.default'));
$path = $disk->putFileAs($storagePath, new File($tmpName), $fileName, 'public');
$size = filesize($tmpName);
unlink($tmpName);
if (! $path) {
if (config('app.dev_log')) {
Log::error('Failed to store file permanently');
}
return null;
}
return [
'path' => $path,
'size' => $size,
'filename' => $fileName,
];
} catch (Exception $e) {
if (file_exists($tmpName)) {
unlink($tmpName);
}
if (config('app.dev_log')) {
Log::error('Media download failed', [
'url' => $mediaUrl,
'error' => $e->getMessage(),
]);
}
return null;
}
}
/**
* Download with size limit enforcement
*/
private function downloadWithSizeLimit(string $url, $context): ?string
{
$maxFileSizeBytes = $this->getMaxFileSizeBytes();
$handle = fopen($url, 'r', false, $context);
if (! $handle) {
if (config('app.dev_log')) {
Log::warning('Failed to open URL stream', ['url' => $url]);
}
return null;
}
$data = '';
$size = 0;
while (! feof($handle) && $size < $maxFileSizeBytes) {
$chunk = fread($handle, 8192);
if ($chunk === false) {
break;
}
$data .= $chunk;
$size += strlen($chunk);
}
fclose($handle);
if ($size >= $maxFileSizeBytes) {
if (config('app.dev_log')) {
Log::warning('File too large', ['size' => $size, 'limit' => $maxFileSizeBytes]);
}
return null;
}
return $data;
}
/**
* Validate downloaded file
*/
private function validateDownloadedFile(string $filePath, string $expectedMimeType): bool
{
// Check file exists and is readable
if (! is_readable($filePath)) {
if (config('app.dev_log')) {
Log::warning('Downloaded file not readable', ['path' => $filePath]);
}
return false;
}
// Get actual MIME type
$finfo = new \finfo(FILEINFO_MIME_TYPE);
$actualMimeType = $finfo->file($filePath);
if ($actualMimeType !== $expectedMimeType) {
if (config('app.dev_log')) {
Log::warning('MIME type mismatch', [
'expected' => $expectedMimeType,
'actual' => $actualMimeType,
]);
}
return false;
}
// Additional file type specific validations
if (str_starts_with($actualMimeType, 'image/')) {
return $this->validateImageFile($filePath);
} elseif (str_starts_with($actualMimeType, 'video/')) {
return $this->validateVideoFile($filePath);
}
return true;
}
/**
* Validate image file
*/
private function validateImageFile(string $filePath): bool
{
$imageInfo = getimagesize($filePath);
if (! $imageInfo) {
if (config('app.dev_log')) {
Log::warning('Invalid image file', ['path' => $filePath]);
}
return false;
}
// Check reasonable dimensions (not too large, not too small)
[$width, $height] = $imageInfo;
if ($width < 1 || $height < 1 || $width != 1080 || $height != 1920) {
if (config('app.dev_log')) {
Log::warning('Image dimensions out of range', [
'width' => $width,
'height' => $height,
]);
}
return false;
}
return true;
}
/**
* Basic video file validation
*/
private function validateVideoFile(string $filePath): bool
{
// Todo: improved video file header checks
$size = filesize($filePath);
$maxSize = $this->getMaxFileSizeBytes();
return $size > 0 && $size <= $maxSize;
}
/**
* Get allowed MIME types from config
*/
private function getAllowedMimeTypes(): array
{
$mediaTypes = config_cache('pixelfed.media_types', 'image/jpeg,image/png');
return array_map('trim', explode(',', $mediaTypes));
}
/**
* Get allowed file extensions based on MIME types from config
*/
private function getAllowedExtensions(): array
{
$mimeTypes = $this->getAllowedMimeTypes();
$extensions = [];
$mimeToExtension = [
'image/jpeg' => ['jpg', 'jpeg'],
'image/png' => ['png'],
'image/gif' => ['gif'],
'image/webp' => ['webp'],
'image/heic' => ['heic', 'heif'],
'image/avif' => ['avif'],
'video/mp4' => ['mp4'],
'video/webm' => ['webm'],
'video/mov' => ['mov'],
'video/quicktime' => ['mov', 'qt'],
];
foreach ($mimeTypes as $mimeType) {
if (isset($mimeToExtension[$mimeType])) {
$extensions = array_merge($extensions, $mimeToExtension[$mimeType]);
}
}
return array_unique($extensions);
}
/**
* Get max file size in bytes from config (config is in KB)
*/
private function getMaxFileSizeBytes(): int
{
$maxSizeKb = config('pixelfed.max_photo_size', 15000);
return $maxSizeKb * 1024;
}
/**
* Generate cryptographically secure filename
*/
private function generateSecureFileName(string $extension): string
{
$random1 = Str::random(random_int(2, 12));
$random2 = Str::random(random_int(32, 35));
$random3 = Str::random(random_int(1, 14));
return $random1.'_'.$random2.'_'.$random3.'.'.$extension;
}
/**
* Create story record with transaction safety
*/
private function createStoryRecord(array $payload, $profile, array $mediaResult): void
{
DB::transaction(function () use ($payload, $profile, $mediaResult) {
// Check for duplicate by object_id
if (Story::where('object_id', $payload['id'])->exists()) {
if (config('app.dev_log')) {
Log::info('Story already exists', ['object_id' => $payload['id']]);
}
return;
}
$type = $payload['attachment']['type'] === 'Image' ? 'photo' : 'video';
$story = new Story;
$story->profile_id = $profile->id;
$story->object_id = $payload['id'];
$story->size = $mediaResult['size'];
$story->mime = data_get($payload, 'attachment.mediaType');
$story->duration = $payload['duration'] ?? null;
$story->media_url = data_get($payload, 'attachment.url');
$story->type = $type;
$story->public = false;
$story->local = false;
$story->active = true;
$story->path = $mediaResult['path'];
$story->view_count = 0;
$story->can_reply = $payload['can_reply'] ?? false;
$story->can_react = $payload['can_react'] ?? false;
$story->created_at = now()->parse($payload['published']);
$story->expires_at = now()->parse($payload['expiresAt']);
$story->save();
// Index the story
$index = app(StoryIndexService::class);
$index->indexStory($story);
// Clear cache
StoryService::delLatest($story->profile_id);
if (config('app.dev_log')) {
Log::info('Story created successfully', [
'story_id' => $story->id,
'profile_id' => $profile->id,
]);
}
});
}
/**
* Handle job failure
*/
public function failed(Exception $exception)
{
if (config('app.dev_log')) {
Log::error('StoryFetch job failed permanently', [
'activity_id' => $this->activity['id'] ?? 'unknown',
'error' => $exception->getMessage(),
'attempts' => $this->attempts(),
]);
}
}
}