mirror of https://github.com/usememos/memos
feat(attachments): add Live Photo and Motion Photo support (#5810)
boojack
2 months ago
committed by
GitHub
parent
894b3eb045
commit
4b4e719470
@ -0,0 +1,110 @@
|
||||
package motionphoto
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"regexp"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
type Detection struct {
|
||||
VideoStart int
|
||||
PresentationTimestampUs int64
|
||||
}
|
||||
|
||||
var (
|
||||
motionPhotoMarkerRegex = regexp.MustCompile(`(?i)(?:Camera:MotionPhoto|GCamera:MotionPhoto|MicroVideo)["'=:\s>]+1`)
|
||||
presentationRegex = regexp.MustCompile(`(?i)(?:Camera:MotionPhotoPresentationTimestampUs|GCamera:MotionPhotoPresentationTimestampUs)["'=:\s>]+(-?\d+)`)
|
||||
microVideoOffsetRegex = regexp.MustCompile(`(?i)(?:Camera:MicroVideoOffset|GCamera:MicroVideoOffset)["'=:\s>]+(\d+)`)
|
||||
)
|
||||
|
||||
const maxMetadataScanBytes = 256 * 1024
|
||||
|
||||
func DetectJPEG(blob []byte) *Detection {
|
||||
if len(blob) < 16 || !bytes.HasPrefix(blob, []byte{0xFF, 0xD8}) {
|
||||
return nil
|
||||
}
|
||||
|
||||
text := string(blob[:min(len(blob), maxMetadataScanBytes)])
|
||||
if !motionPhotoMarkerRegex.MatchString(text) {
|
||||
return nil
|
||||
}
|
||||
|
||||
videoStart := detectVideoStart(blob, text)
|
||||
if videoStart < 0 || videoStart >= len(blob) {
|
||||
return nil
|
||||
}
|
||||
|
||||
return &Detection{
|
||||
VideoStart: videoStart,
|
||||
PresentationTimestampUs: parsePresentationTimestampUs(text),
|
||||
}
|
||||
}
|
||||
|
||||
func ExtractVideo(blob []byte) ([]byte, *Detection) {
|
||||
detection := DetectJPEG(blob)
|
||||
if detection == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
videoBlob := blob[detection.VideoStart:]
|
||||
if !looksLikeMP4(videoBlob) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
return videoBlob, detection
|
||||
}
|
||||
|
||||
func detectVideoStart(blob []byte, text string) int {
|
||||
if matches := microVideoOffsetRegex.FindStringSubmatch(text); len(matches) == 2 {
|
||||
if offset, err := strconv.Atoi(matches[1]); err == nil && offset > 0 && offset < len(blob) {
|
||||
start := len(blob) - offset
|
||||
if looksLikeMP4(blob[start:]) {
|
||||
return start
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return findEmbeddedMP4Start(blob)
|
||||
}
|
||||
|
||||
func parsePresentationTimestampUs(text string) int64 {
|
||||
matches := presentationRegex.FindStringSubmatch(text)
|
||||
if len(matches) != 2 {
|
||||
return 0
|
||||
}
|
||||
|
||||
value, err := strconv.ParseInt(matches[1], 10, 64)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func findEmbeddedMP4Start(blob []byte) int {
|
||||
searchFrom := len(blob)
|
||||
for searchFrom > 8 {
|
||||
index := bytes.LastIndex(blob[:searchFrom], []byte("ftyp"))
|
||||
if index < 4 {
|
||||
return -1
|
||||
}
|
||||
|
||||
start := index - 4
|
||||
if looksLikeMP4(blob[start:]) {
|
||||
return start
|
||||
}
|
||||
|
||||
searchFrom = index - 1
|
||||
}
|
||||
|
||||
return -1
|
||||
}
|
||||
|
||||
func looksLikeMP4(blob []byte) bool {
|
||||
if len(blob) < 12 || !bytes.Equal(blob[4:8], []byte("ftyp")) {
|
||||
return false
|
||||
}
|
||||
|
||||
size := binary.BigEndian.Uint32(blob[:4])
|
||||
return size == 1 || size >= 8
|
||||
}
|
||||
@ -0,0 +1,25 @@
|
||||
package motionphoto
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/usememos/memos/internal/testutil"
|
||||
)
|
||||
|
||||
func TestDetectJPEG(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
blob := testutil.BuildMotionPhotoJPEG()
|
||||
detection := DetectJPEG(blob)
|
||||
require.NotNil(t, detection)
|
||||
require.Positive(t, detection.VideoStart)
|
||||
require.EqualValues(t, 123456, detection.PresentationTimestampUs)
|
||||
|
||||
videoBlob, extracted := ExtractVideo(blob)
|
||||
require.NotNil(t, extracted)
|
||||
require.True(t, bytes.Equal(videoBlob[:4], []byte{0x00, 0x00, 0x00, 0x10}))
|
||||
require.Equal(t, []byte("ftyp"), videoBlob[4:8])
|
||||
}
|
||||
@ -0,0 +1,18 @@
|
||||
package testutil
|
||||
|
||||
// BuildMotionPhotoJPEG returns a minimal JPEG blob with Motion Photo metadata
|
||||
// and an embedded MP4 header for tests.
|
||||
func BuildMotionPhotoJPEG() []byte {
|
||||
return append(
|
||||
[]byte{
|
||||
0xFF, 0xD8, 0xFF, 0xE1,
|
||||
},
|
||||
append(
|
||||
[]byte(`<?xpacket begin=""?><rdf:Description GCamera:MotionPhoto="1" GCamera:MotionPhotoPresentationTimestampUs="123456"></rdf:Description>`),
|
||||
[]byte{
|
||||
0xFF, 0xD9,
|
||||
0x00, 0x00, 0x00, 0x10, 'f', 't', 'y', 'p', 'i', 's', 'o', 'm', 0x00, 0x00, 0x00, 0x00,
|
||||
}...,
|
||||
)...,
|
||||
)
|
||||
}
|
||||
@ -0,0 +1,69 @@
|
||||
package v1
|
||||
|
||||
import (
|
||||
v1pb "github.com/usememos/memos/proto/gen/api/v1"
|
||||
storepb "github.com/usememos/memos/proto/gen/store"
|
||||
"github.com/usememos/memos/store"
|
||||
)
|
||||
|
||||
func convertMotionMediaFromStore(motion *storepb.MotionMedia) *v1pb.MotionMedia {
|
||||
if motion == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
return &v1pb.MotionMedia{
|
||||
Family: v1pb.MotionMediaFamily(motion.Family),
|
||||
Role: v1pb.MotionMediaRole(motion.Role),
|
||||
GroupId: motion.GroupId,
|
||||
PresentationTimestampUs: motion.PresentationTimestampUs,
|
||||
HasEmbeddedVideo: motion.HasEmbeddedVideo,
|
||||
}
|
||||
}
|
||||
|
||||
func convertMotionMediaToStore(motion *v1pb.MotionMedia) *storepb.MotionMedia {
|
||||
if motion == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
return &storepb.MotionMedia{
|
||||
Family: storepb.MotionMediaFamily(motion.Family),
|
||||
Role: storepb.MotionMediaRole(motion.Role),
|
||||
GroupId: motion.GroupId,
|
||||
PresentationTimestampUs: motion.PresentationTimestampUs,
|
||||
HasEmbeddedVideo: motion.HasEmbeddedVideo,
|
||||
}
|
||||
}
|
||||
|
||||
func getAttachmentMotionMedia(attachment *store.Attachment) *storepb.MotionMedia {
|
||||
if attachment == nil || attachment.Payload == nil {
|
||||
return nil
|
||||
}
|
||||
return attachment.Payload.MotionMedia
|
||||
}
|
||||
|
||||
func isAndroidMotionContainer(motion *storepb.MotionMedia) bool {
|
||||
return motion != nil &&
|
||||
motion.Family == storepb.MotionMediaFamily_ANDROID_MOTION_PHOTO &&
|
||||
motion.Role == storepb.MotionMediaRole_CONTAINER &&
|
||||
motion.HasEmbeddedVideo
|
||||
}
|
||||
|
||||
func ensureAttachmentPayload(payload *storepb.AttachmentPayload) *storepb.AttachmentPayload {
|
||||
if payload != nil {
|
||||
return payload
|
||||
}
|
||||
return &storepb.AttachmentPayload{}
|
||||
}
|
||||
|
||||
func isMultiMemberMotionGroup(attachments []*store.Attachment) bool {
|
||||
if len(attachments) < 2 {
|
||||
return false
|
||||
}
|
||||
for _, attachment := range attachments {
|
||||
motion := getAttachmentMotionMedia(attachment)
|
||||
if motion == nil || motion.GroupId == "" {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
@ -0,0 +1,179 @@
|
||||
import type { Attachment } from "@/types/proto/api/v1/attachment_service_pb";
|
||||
import {
|
||||
getAttachmentMotionClipUrl,
|
||||
getAttachmentMotionGroupId,
|
||||
getAttachmentThumbnailUrl,
|
||||
getAttachmentType,
|
||||
getAttachmentUrl,
|
||||
isAndroidMotionContainer,
|
||||
isAppleLivePhotoStill,
|
||||
isAppleLivePhotoVideo,
|
||||
isMotionAttachment,
|
||||
} from "./attachment";
|
||||
|
||||
export interface PreviewMediaItem {
|
||||
id: string;
|
||||
kind: "image" | "video";
|
||||
sourceUrl: string;
|
||||
posterUrl?: string;
|
||||
filename: string;
|
||||
isMotion: boolean;
|
||||
presentationTimestampUs?: bigint;
|
||||
}
|
||||
|
||||
export interface AttachmentVisualItem {
|
||||
id: string;
|
||||
kind: "image" | "video" | "motion";
|
||||
filename: string;
|
||||
posterUrl: string;
|
||||
sourceUrl: string;
|
||||
attachmentNames: string[];
|
||||
attachments: Attachment[];
|
||||
previewItem: PreviewMediaItem;
|
||||
mimeType: string;
|
||||
}
|
||||
|
||||
export function buildAttachmentVisualItems(attachments: Attachment[]): AttachmentVisualItem[] {
|
||||
const attachmentsByGroup = new Map<string, Attachment[]>();
|
||||
for (const attachment of attachments) {
|
||||
const groupId = getAttachmentMotionGroupId(attachment);
|
||||
if (!groupId) {
|
||||
continue;
|
||||
}
|
||||
const group = attachmentsByGroup.get(groupId) ?? [];
|
||||
group.push(attachment);
|
||||
attachmentsByGroup.set(groupId, group);
|
||||
}
|
||||
|
||||
const consumedGroups = new Set<string>();
|
||||
const items: AttachmentVisualItem[] = [];
|
||||
|
||||
for (const attachment of attachments) {
|
||||
if (isAndroidMotionContainer(attachment)) {
|
||||
items.push(buildAndroidMotionItem(attachment));
|
||||
continue;
|
||||
}
|
||||
|
||||
const groupId = getAttachmentMotionGroupId(attachment);
|
||||
if (!groupId || consumedGroups.has(groupId)) {
|
||||
if (!groupId) {
|
||||
items.push(buildSingleAttachmentItem(attachment));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
const group = attachmentsByGroup.get(groupId) ?? [];
|
||||
const still = group.find(isAppleLivePhotoStill);
|
||||
const video = group.find(isAppleLivePhotoVideo);
|
||||
if (still && video && group.length === 2) {
|
||||
items.push(buildAppleMotionItem(still, video));
|
||||
consumedGroups.add(groupId);
|
||||
continue;
|
||||
}
|
||||
|
||||
items.push(buildSingleAttachmentItem(attachment));
|
||||
consumedGroups.add(groupId);
|
||||
for (const member of group) {
|
||||
if (member.name === attachment.name) {
|
||||
continue;
|
||||
}
|
||||
items.push(buildSingleAttachmentItem(member));
|
||||
}
|
||||
}
|
||||
|
||||
return dedupeVisualItems(items);
|
||||
}
|
||||
|
||||
export function countLogicalAttachmentItems(attachments: Attachment[]): number {
|
||||
const visualAttachments = attachments.filter(
|
||||
(attachment) =>
|
||||
getAttachmentType(attachment) === "image/*" || getAttachmentType(attachment) === "video/*" || isMotionAttachment(attachment),
|
||||
);
|
||||
const visualNames = new Set(visualAttachments.map((attachment) => attachment.name));
|
||||
const visualCount = buildAttachmentVisualItems(visualAttachments).length;
|
||||
const nonVisualCount = attachments.filter((attachment) => !visualNames.has(attachment.name)).length;
|
||||
return visualCount + nonVisualCount;
|
||||
}
|
||||
|
||||
function buildSingleAttachmentItem(attachment: Attachment): AttachmentVisualItem {
|
||||
const attachmentType = getAttachmentType(attachment);
|
||||
const sourceUrl = getAttachmentUrl(attachment);
|
||||
const posterUrl = attachmentType === "image/*" ? getAttachmentThumbnailUrl(attachment) : sourceUrl;
|
||||
const previewKind = attachmentType === "video/*" ? "video" : "image";
|
||||
|
||||
return {
|
||||
id: attachment.name,
|
||||
kind: attachmentType === "video/*" ? "video" : "image",
|
||||
filename: attachment.filename,
|
||||
posterUrl,
|
||||
sourceUrl,
|
||||
attachmentNames: [attachment.name],
|
||||
attachments: [attachment],
|
||||
previewItem: {
|
||||
id: attachment.name,
|
||||
kind: previewKind,
|
||||
sourceUrl,
|
||||
posterUrl,
|
||||
filename: attachment.filename,
|
||||
isMotion: false,
|
||||
},
|
||||
mimeType: attachment.type,
|
||||
};
|
||||
}
|
||||
|
||||
function buildAppleMotionItem(still: Attachment, video: Attachment): AttachmentVisualItem {
|
||||
const sourceUrl = getAttachmentUrl(video);
|
||||
const posterUrl = getAttachmentThumbnailUrl(still);
|
||||
|
||||
return {
|
||||
id: getAttachmentMotionGroupId(still) ?? still.name,
|
||||
kind: "motion",
|
||||
filename: still.filename,
|
||||
posterUrl,
|
||||
sourceUrl,
|
||||
attachmentNames: [still.name, video.name],
|
||||
attachments: [still, video],
|
||||
previewItem: {
|
||||
id: getAttachmentMotionGroupId(still) ?? still.name,
|
||||
kind: "video",
|
||||
sourceUrl,
|
||||
posterUrl,
|
||||
filename: still.filename,
|
||||
isMotion: true,
|
||||
},
|
||||
mimeType: still.type,
|
||||
};
|
||||
}
|
||||
|
||||
function buildAndroidMotionItem(attachment: Attachment): AttachmentVisualItem {
|
||||
return {
|
||||
id: attachment.name,
|
||||
kind: "motion",
|
||||
filename: attachment.filename,
|
||||
posterUrl: getAttachmentThumbnailUrl(attachment),
|
||||
sourceUrl: getAttachmentMotionClipUrl(attachment),
|
||||
attachmentNames: [attachment.name],
|
||||
attachments: [attachment],
|
||||
previewItem: {
|
||||
id: attachment.name,
|
||||
kind: "video",
|
||||
sourceUrl: getAttachmentMotionClipUrl(attachment),
|
||||
posterUrl: getAttachmentThumbnailUrl(attachment),
|
||||
filename: attachment.filename,
|
||||
isMotion: true,
|
||||
presentationTimestampUs: attachment.motionMedia?.presentationTimestampUs,
|
||||
},
|
||||
mimeType: attachment.type,
|
||||
};
|
||||
}
|
||||
|
||||
function dedupeVisualItems(items: AttachmentVisualItem[]): AttachmentVisualItem[] {
|
||||
const seen = new Set<string>();
|
||||
return items.filter((item) => {
|
||||
if (seen.has(item.id)) {
|
||||
return false;
|
||||
}
|
||||
seen.add(item.id);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
Loading…
Reference in New Issue