From 375858f09de8047d15cd859b6261eb2f3d58b901 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Mon, 31 Mar 2025 23:43:13 -0600 Subject: [PATCH] Update AccountImport, improve webp support --- app/Http/Controllers/ImportPostController.php | 197 +++++++++++++----- resources/assets/components/AccountImport.vue | 38 +++- 2 files changed, 168 insertions(+), 67 deletions(-) diff --git a/app/Http/Controllers/ImportPostController.php b/app/Http/Controllers/ImportPostController.php index 47456b2b3..e491019f8 100644 --- a/app/Http/Controllers/ImportPostController.php +++ b/app/Http/Controllers/ImportPostController.php @@ -103,67 +103,95 @@ class ImportPostController extends Controller $uid = $request->user()->id; $pid = $request->user()->profile_id; + $successCount = 0; + $errors = []; + foreach($request->input('files') as $file) { - $media = $file['media']; - $c = collect($media); - $postHash = hash('sha256', $c->toJson()); - $exts = $c->map(function($m) { - $fn = last(explode('/', $m['uri'])); - return last(explode('.', $fn)); - }); - $postType = 'photo'; - - if($exts->count() > 1) { - if($exts->contains('mp4')) { - if($exts->contains('jpg', 'png')) { - $postType = 'photo:video:album'; - } else { - $postType = 'video:album'; - } - } else { - $postType = 'photo:album'; + try { + $media = $file['media']; + $c = collect($media); + + $firstUri = isset($media[0]['uri']) ? $media[0]['uri'] : ''; + $postHash = hash('sha256', $c->toJson() . $firstUri); + + $exists = ImportPost::where('user_id', $uid) + ->where('post_hash', $postHash) + ->exists(); + + if ($exists) { + $errors[] = "Duplicate post detected. Skipping..."; + continue; } - } else { - if(in_array($exts[0], ['jpg', 'png'])) { - $postType = 'photo'; - } else if(in_array($exts[0], ['mp4'])) { - $postType = 'video'; + + $exts = $c->map(function($m) { + $fn = last(explode('/', $m['uri'])); + return last(explode('.', $fn)); + }); + + $postType = $this->determinePostType($exts); + + $ip = new ImportPost; + $ip->user_id = $uid; + $ip->profile_id = $pid; + $ip->post_hash = $postHash; + $ip->service = 'instagram'; + $ip->post_type = $postType; + $ip->media_count = $c->count(); + + $ip->media = $c->map(function($m) { + return [ + 'uri' => $m['uri'], + 'title' => $this->formatHashtags($m['title'] ?? ''), + 'creation_timestamp' => $m['creation_timestamp'] ?? null + ]; + })->toArray(); + + $ip->caption = $c->count() > 1 ? + $this->formatHashtags($file['title'] ?? '') : + $this->formatHashtags($ip->media[0]['title'] ?? ''); + + $originalFilename = last(explode('/', $ip->media[0]['uri'] ?? '')); + $ip->filename = $this->sanitizeFilename($originalFilename); + + $ip->metadata = $c->map(function($m) { + return [ + 'uri' => $m['uri'], + 'media_metadata' => isset($m['media_metadata']) ? $m['media_metadata'] : null + ]; + })->toArray(); + + $creationTimestamp = $c->count() > 1 ? + ($file['creation_timestamp'] ?? null) : + ($media[0]['creation_timestamp'] ?? null); + + if ($creationTimestamp) { + $ip->creation_date = now()->parse($creationTimestamp); + $ip->creation_year = $ip->creation_date->format('y'); + $ip->creation_month = $ip->creation_date->format('m'); + $ip->creation_day = $ip->creation_date->format('d'); + } else { + $ip->creation_date = now(); + $ip->creation_year = now()->format('y'); + $ip->creation_month = now()->format('m'); + $ip->creation_day = now()->format('d'); } - } - $ip = new ImportPost; - $ip->user_id = $uid; - $ip->profile_id = $pid; - $ip->post_hash = $postHash; - $ip->service = 'instagram'; - $ip->post_type = $postType; - $ip->media_count = $c->count(); - $ip->media = $c->map(function($m) { - return [ - 'uri' => $m['uri'], - 'title' => $this->formatHashtags($m['title']), - 'creation_timestamp' => $m['creation_timestamp'] - ]; - })->toArray(); - $ip->caption = $c->count() > 1 ? $this->formatHashtags($file['title']) : $this->formatHashtags($ip->media[0]['title']); - $ip->filename = last(explode('/', $ip->media[0]['uri'])); - $ip->metadata = $c->map(function($m) { - return [ - 'uri' => $m['uri'], - 'media_metadata' => isset($m['media_metadata']) ? $m['media_metadata'] : null - ]; - })->toArray(); - $ip->creation_date = $c->count() > 1 ? now()->parse($file['creation_timestamp']) : now()->parse($media[0]['creation_timestamp']); - $ip->creation_year = now()->parse($ip->creation_date)->format('y'); - $ip->creation_month = now()->parse($ip->creation_date)->format('m'); - $ip->creation_day = now()->parse($ip->creation_date)->format('d'); - $ip->save(); - - ImportService::getImportedFiles($pid, true); - ImportService::getPostCount($pid, true); + $ip->save(); + $successCount++; + + ImportService::getImportedFiles($pid, true); + ImportService::getPostCount($pid, true); + } catch (\Exception $e) { + $errors[] = $e->getMessage(); + \Log::error('Import error: ' . $e->getMessage()); + continue; + } } + return [ - 'msg' => 'Success' + 'success' => true, + 'msg' => 'Successfully imported ' . $successCount . ' posts', + 'errors' => $errors ]; } @@ -173,7 +201,17 @@ class ImportPostController extends Controller $this->checkPermissions($request); - $mimes = config('import.instagram.allow_video_posts') ? 'mimetypes:image/png,image/jpeg,video/mp4' : 'mimetypes:image/png,image/jpeg'; + $allowedMimeTypes = ['image/png', 'image/jpeg']; + + if (config('import.instagram.allow_image_webp') && str_contains(config_cache('pixelfed.media_types'), 'image/webp')) { + $allowedMimeTypes[] = 'image/webp'; + } + + if (config('import.instagram.allow_video_posts')) { + $allowedMimeTypes[] = 'video/mp4'; + } + + $mimes = 'mimetypes:' . implode(',', $allowedMimeTypes); $this->validate($request, [ 'file' => 'required|array|max:10', @@ -186,7 +224,12 @@ class ImportPostController extends Controller ]); foreach($request->file('file') as $file) { - $fileName = $file->getClientOriginalName(); + $extension = $file->getClientOriginalExtension(); + + $originalName = pathinfo($file->getClientOriginalName(), PATHINFO_FILENAME); + $safeFilename = preg_replace('/[^a-zA-Z0-9_.-]/', '_', $originalName); + $fileName = $safeFilename . '.' . $extension; + $file->storeAs('imports/' . $request->user()->id . '/', $fileName); } @@ -197,6 +240,46 @@ class ImportPostController extends Controller ]; } + + private function determinePostType($exts) + { + if ($exts->count() > 1) { + if ($exts->contains('mp4')) { + if ($exts->contains('jpg', 'png', 'webp')) { + return 'photo:video:album'; + } else { + return 'video:album'; + } + } else { + return 'photo:album'; + } + } else { + if ($exts->isEmpty()) { + return 'photo'; + } + + $ext = $exts[0]; + + if (in_array($ext, ['jpg', 'jpeg', 'png', 'webp'])) { + return 'photo'; + } else if (in_array($ext, ['mp4'])) { + return 'video'; + } else { + return 'photo'; + } + } + } + + private function sanitizeFilename($filename) + { + $parts = explode('.', $filename); + $extension = array_pop($parts); + $originalName = implode('.', $parts); + + $safeFilename = preg_replace('/[^a-zA-Z0-9_.-]/', '_', $originalName); + return $safeFilename . '.' . $extension; + } + protected function checkPermissions($request, $abortOnFail = true) { $user = $request->user(); diff --git a/resources/assets/components/AccountImport.vue b/resources/assets/components/AccountImport.vue index f2b1ed193..94b96cbb7 100644 --- a/resources/assets/components/AccountImport.vue +++ b/resources/assets/components/AccountImport.vue @@ -367,7 +367,7 @@ }, async filterPostMeta(media) { - let fbfix = await this.fixFacebookEncoding(media); + let fbfix = await this.fixFacebookEncoding(media); let json = JSON.parse(fbfix); /* Sometimes the JSON isn't an array, when there's only one post */ if (!Array.isArray(json)) { @@ -422,24 +422,32 @@ this.filterPostMeta(media); let imgs = await Promise.all(entries.filter(entry => { - return (entry.filename.startsWith('media/posts/') || entry.filename.startsWith('media/other/')) && (entry.filename.endsWith('.png') || entry.filename.endsWith('.jpg') || entry.filename.endsWith('.mp4')); + const supportedFormats = ['.png', '.jpg', '.jpeg', '.mp4']; + if (this.config.allow_image_webp) { + supportedFormats.push('.webp'); + } + return (entry.filename.startsWith('media/posts/') || entry.filename.startsWith('media/other/')) && + supportedFormats.some(format => entry.filename.endsWith(format)); }) .map(async entry => { + const supportedFormats = ['.png', '.jpg', '.jpeg', '.mp4']; + if (this.config.allow_image_webp) { + supportedFormats.push('.webp'); + } + if( ( entry.filename.startsWith('media/posts/') || entry.filename.startsWith('media/other/') - ) && ( - entry.filename.endsWith('.png') || - entry.filename.endsWith('.jpg') || - entry.filename.endsWith('.mp4') - ) + ) && + supportedFormats.some(format => entry.filename.endsWith(format)) ) { let types = { 'png': 'image/png', 'jpg': 'image/jpeg', 'jpeg': 'image/jpeg', - 'mp4': 'video/mp4' + 'mp4': 'video/mp4', + 'webp': 'image/webp' } let type = types[entry.filename.split('/').pop().split('.').pop()]; let blob = await entry.getData(new zip.BlobWriter(type)); @@ -517,6 +525,15 @@ return res; }, + getFilename(filename) { + const baseName = filename.split('/').pop(); + + const extension = baseName.split('.').pop(); + const originalName = baseName.substring(0, baseName.lastIndexOf('.')); + const updatedFilename = originalName.replace(/[^a-zA-Z0-9_.-]/g, '_'); + return updatedFilename + '.' + extension; + }, + handleImport() { swal('Importing...', "Please wait while we upload your imported posts.\n Keep this page open and do not navigate away.", 'success'); this.importButtonLoading = true; @@ -527,8 +544,9 @@ chunks.forEach(c => { let formData = new FormData(); c.map((e, idx) => { - let file = new File([e.file], e.filename); - formData.append('file['+ idx +']', file, e.filename.split('/').pop()); + let chunkedFilename = this.getFilename(e.filename); + let file = new File([e.file], chunkedFilename); + formData.append('file['+ idx +']', file, chunkedFilename); }) axios.post( '/api/local/import/ig/media',