MediaPipe, decoded.
Every channel, every model, live.

const face = await FaceLandmarker.createFromOptions(vision, {
  baseOptions: {
    modelAssetPath: 'https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/latest/face_landmarker.task',
    delegate: 'GPU'
  },
  runningMode: 'VIDEO',
  numFaces: 1,
  outputFaceBlendshapes: true,           // 52 ARKit weights — drives MetaHuman / Live Link Face
  outputFacialTransformationMatrixes: true // 4×4 head pose
});

const r = face.detectForVideo(videoEl, performance.now());
// r.faceLandmarks[0]                    → 478 mesh points (x, y, z)
// r.faceBlendshapes[0].categories       → 52 weighted shapes [{ categoryName, score }]
// r.facialTransformationMatrixes[0].data → Float32Array(16) — column-major 4×4

const hands = await HandLandmarker.createFromOptions(vision, {
  baseOptions: {
    modelAssetPath: 'https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/latest/hand_landmarker.task',
    delegate: 'GPU'
  },
  runningMode: 'VIDEO',
  numHands: 2
});

const r = hands.detectForVideo(videoEl, performance.now());
// r.landmarks      → [hand1Landmarks, hand2Landmarks] each: 21 × {x,y,z}
// r.worldLandmarks → metric coords, wrist-relative
// r.handedness     → [[{ categoryName: 'Left'|'Right', score }], ...]

// Mirror-aware handedness:
function trueHand(label, mirrored) {
  if (!mirrored) return label;
  return label === 'Left' ? 'Right' : 'Left';
}

const gestures = await GestureRecognizer.createFromOptions(vision, {
  baseOptions: {
    modelAssetPath: 'https://storage.googleapis.com/mediapipe-models/gesture_recognizer/gesture_recognizer/float16/latest/gesture_recognizer.task',
    delegate: 'GPU'
  },
  runningMode: 'VIDEO',
  numHands: 2
});

const r = gestures.recognizeForVideo(videoEl, performance.now());
// r.gestures     → [[{ categoryName: 'Thumb_Up', score: 0.93 }], ...]  (top-1 per hand)
// r.handedness   → [[{ categoryName: 'Left'|'Right', score }], ...]
// r.landmarks    → 21 image-space landmarks per hand
// r.worldLandmarks → metric coords per hand

// Init three landmarkers once
const [pose, face, hands] = await Promise.all([
  PoseLandmarker.createFromOptions(vision, { runningMode: 'VIDEO', numPoses: 1, baseOptions: poseModel }),
  FaceLandmarker.createFromOptions(vision, { runningMode: 'VIDEO', numFaces: 1,
    outputFaceBlendshapes: true, outputFacialTransformationMatrixes: true, baseOptions: faceModel }),
  HandLandmarker.createFromOptions(vision, { runningMode: 'VIDEO', numHands: 2, baseOptions: handModel })
]);

// per RAF tick, run all three with the same timestamp
const t = performance.now();
const [poseR, faceR, handsR] = [
  pose.detectForVideo(videoEl, t),
  face.detectForVideo(videoEl, t),
  hands.detectForVideo(videoEl, t)
];

// Merge into a single holistic payload
const holistic = {
  timestamp: t,
  pose:        poseR.landmarks?.[0]      ?? null,
  poseWorld:   poseR.worldLandmarks?.[0] ?? null,
  face:        faceR.faceLandmarks?.[0]  ?? null,
  blendshapes: faceR.faceBlendshapes?.[0]?.categories ?? null,
  headMatrix:  faceR.facialTransformationMatrixes?.[0]?.data ?? null,
  hands:       handsR.landmarks?.map((lm, i) => ({
    label: handsR.handedness[i][0].categoryName,
    score: handsR.handedness[i][0].score,
    landmarks: lm,
    world: handsR.worldLandmarks[i]
  })) ?? []
};

import { HolisticLandmarker, FilesetResolver }
  from '@mediapipe/tasks-vision';

const holistic = await HolisticLandmarker.createFromOptions(vision, {
  baseOptions: { modelAssetPath: '.../holistic_landmarker.task', delegate: 'GPU' },
  runningMode: 'VIDEO',
  outputFaceBlendshapes: true,
  minFaceDetectionConfidence: 0.5,
  minHandLandmarksConfidence: 0.5,
  minPoseDetectionConfidence: 0.5
});

const r = holistic.detectForVideo(videoEl, performance.now());
// r.poseLandmarks, r.poseWorldLandmarks
// r.faceLandmarks, r.faceBlendshapes
// r.leftHandLandmarks, r.rightHandLandmarks
// r.leftHandWorldLandmarks, r.rightHandWorldLandmarks

const det = await ObjectDetector.createFromOptions(vision, {
  baseOptions: {
    modelAssetPath: 'https://storage.googleapis.com/mediapipe-models/object_detector/efficientdet_lite0/float16/latest/efficientdet_lite0.task',
    delegate: 'GPU'
  },
  runningMode: 'VIDEO',
  scoreThreshold: 0.5,
  categoryAllowlist: ['person', 'cup', 'laptop']
});

const r = det.detectForVideo(videoEl, performance.now());
// r.detections → [{ boundingBox: {originX, originY, width, height},
//                  categories: [{ categoryName, score, index }] }, ...]

const seg = await ImageSegmenter.createFromOptions(vision, {
  baseOptions: {
    modelAssetPath: 'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_multiclass_256x256/float32/latest/selfie_multiclass_256x256.tflite',
    delegate: 'GPU'
  },
  runningMode: 'VIDEO',
  outputCategoryMask: true,
  outputConfidenceMasks: false
});

const r = seg.segmentForVideo(videoEl, performance.now());
// r.categoryMask → MPMask  (.getAsUint8Array() → H*W bytes, each a class idx 0–5)
// IMPORTANT: call r.close() when you're done with the masks to free GPU memory
r.close();