From 70c6cf8e7efde9bdbce013a493b577170f9b3d75 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sat, 8 Aug 2026 04:21:50 -0700 Subject: [PATCH] feat: add new FAL video families and image models Video (plugins/video_gen/fal): Seedance 2.5, MiniMax H3, Seedance 2.0 Mini, FLUX 3, Grok Imagine 1.5, Gemini Omni Flash (i2v-only). New family capability flags: - duration_int: endpoints that take duration as a JSON integer - resolution_aliases: maps 720p/1080p-style values onto non-standard enums (H3's 768P/2K/4K) - image_drop_keys: strips keys the family's i2v endpoint rejects (aspect_ratio on Seedance 2.5 / H3 / Grok 1.5) Image (tools/image_generation_tool): Seedream 5.0 Pro (+edit) and Lite, Ideogram V4 instant + fast, Qwen Image 3 (+edit), MAI Image 2.5 Pro, Nano Banana 2 Lite (+edit), Recraft V4.1. Every new endpoint live-tested against fal.run through the real payload builders + submit path: 18/18 pass (t2v, i2v, t2i, and edit probes). Note: several new endpoints return HTTP 409 from the Nous Portal FAL proxy allowlist until it is updated portal-side; BYOK FAL_KEY works today and the existing 4xx guidance message covers it. --- plugins/video_gen/fal/__init__.py | 151 +++++++++++-- tests/plugins/video_gen/test_fal_plugin.py | 87 ++++++++ ...st_video_generation_tool_surface_matrix.py | 8 + tools/image_generation_tool.py | 200 ++++++++++++++++++ 4 files changed, 434 insertions(+), 12 deletions(-) diff --git a/plugins/video_gen/fal/__init__.py b/plugins/video_gen/fal/__init__.py index 1a67a5260f..f2a76dca1e 100644 --- a/plugins/video_gen/fal/__init__.py +++ b/plugins/video_gen/fal/__init__.py @@ -14,10 +14,15 @@ Model families (each with t2v + i2v endpoints): pixverse-v6 fal-ai/pixverse/v6/text-to-video / fal-ai/pixverse/v6/image-to-video Premium tier: - veo3.1 fal-ai/veo3.1 / fal-ai/veo3.1/image-to-video - seedance-2.0 bytedance/seedance-2.0/text-to-video / bytedance/seedance-2.0/image-to-video - kling-v3-4k fal-ai/kling-video/v3/4k/text-to-video / fal-ai/kling-video/v3/4k/image-to-video - happy-horse alibaba/happy-horse/text-to-video / alibaba/happy-horse/image-to-video + veo3.1 fal-ai/veo3.1 / fal-ai/veo3.1/image-to-video + seedance-2.0 bytedance/seedance-2.0/text-to-video / bytedance/seedance-2.0/image-to-video + seedance-2.5 bytedance/seedance-2.5/text-to-video / bytedance/seedance-2.5/image-to-video + minimax-h3 minimax/h3/text-to-video / minimax/h3/image-to-video + kling-v3-4k fal-ai/kling-video/v3/4k/text-to-video / fal-ai/kling-video/v3/4k/image-to-video + happy-horse alibaba/happy-horse/text-to-video / alibaba/happy-horse/image-to-video + + Cheap tier (continued): + seedance-2.0-mini bytedance/seedance-2.0/mini/text-to-video / bytedance/seedance-2.0/mini/image-to-video Selection precedence for the active family: 1. ``model=`` arg from the tool call @@ -96,6 +101,20 @@ FAL_FAMILIES: Dict[str, Dict[str, Any]] = { "audio": True, "negative": True, }, + "seedance-2.0-mini": { + "display": "Seedance 2.0 Mini", + "speed": "~30-90s", + "price": "cheap", + "strengths": "ByteDance. Faster/cheaper Seedance tier, audio + lip-sync, 4-15s.", + "tier": "cheap", + "text_endpoint": "bytedance/seedance-2.0/mini/text-to-video", + "image_endpoint": "bytedance/seedance-2.0/mini/image-to-video", + "aspect_ratios": ("21:9", "16:9", "4:3", "1:1", "3:4", "9:16"), + "resolutions": ("480p", "720p"), + "durations": (4, 15), + "audio": True, + "negative": False, + }, # ─── Expensive / premium tier ────────────────────────────────────── "veo3.1": { "display": "Veo 3.1", @@ -128,6 +147,99 @@ FAL_FAMILIES: Dict[str, Dict[str, Any]] = { "audio": True, "negative": False, }, + "seedance-2.5": { + "display": "Seedance 2.5", + "speed": "~60-180s", + "price": "premium", + "strengths": "ByteDance flagship. Native 30s single-pass, audio in the same latent space, lip-sync.", + "tier": "premium", + "text_endpoint": "bytedance/seedance-2.5/text-to-video", + "image_endpoint": "bytedance/seedance-2.5/image-to-video", + # i2v accepts only "auto" for aspect_ratio (it follows the input + # image), so aspect_ratio is dropped for image jobs via + # image_drop_keys. + "image_drop_keys": ("aspect_ratio",), + "aspect_ratios": ("21:9", "16:9", "4:3", "1:1", "3:4", "9:16"), + "resolutions": ("480p", "720p"), + "durations": (4, 30), + "audio": True, + "negative": False, + }, + "minimax-h3": { + "display": "MiniMax H3", + "speed": "~60-180s", + "price": "premium", + "strengths": "MiniMax frontier. Native 2K (up to 4K), 5-15s, seven aspect ratios.", + "tier": "premium", + "text_endpoint": "minimax/h3/text-to-video", + "image_endpoint": "minimax/h3/image-to-video", + # H3 takes duration as a JSON integer, not the stringified form + # most FAL endpoints use. + "duration_int": True, + # i2v derives the aspect ratio from the input image and rejects + # the key entirely. + "image_drop_keys": ("aspect_ratio",), + "aspect_ratios": ("21:9", "16:9", "4:3", "1:1", "3:4", "9:16"), + # H3 uses capitalized/2K-style resolution enums — mapped from the + # tool's usual 720p/1080p-style values via resolution_aliases. + "resolutions": ("768P", "2K", "4K"), + "resolution_aliases": { + "480p": "768P", "540p": "768P", "720p": "768P", "768p": "768P", + "1080p": "2K", "2k": "2K", "4k": "4K", "2160p": "4K", + }, + "durations": (5, 15), + "audio": False, # audio is native/always-on; no generate_audio key + "negative": False, + }, + "flux-3": { + "display": "FLUX 3 (via FAL)", + "speed": "~60-120s", + "price": "premium", + "strengths": "Black Forest Labs frontier video. Native audio, 5-20s, 8 aspect ratios.", + "tier": "premium", + "text_endpoint": "blackforestlabs/flux-3/text-to-video", + "image_endpoint": "blackforestlabs/flux-3/image-to-video", + # FLUX 3 duration enum is "auto" | 5..20 as JSON integers. + "duration_int": True, + "aspect_ratios": ("21:9", "2:1", "16:9", "4:3", "1:1", "3:4", "9:16"), + "resolutions": ("720p", "1080p"), + "durations": (5, 20), + "audio": True, + "negative": False, + }, + "grok-imagine-1.5": { + "display": "Grok Imagine 1.5 (via FAL)", + "speed": "~30-90s", + "price": "premium", + "strengths": "xAI. Fast stylized video with audio, 1-15s, cheap per second.", + "tier": "premium", + "text_endpoint": "xai/grok-imagine-video/v1.5/text-to-video", + "image_endpoint": "xai/grok-imagine-video/v1.5/image-to-video", + "duration_int": True, + # i2v derives aspect from the input image; the key is t2v-only. + "image_drop_keys": ("aspect_ratio",), + "aspect_ratios": ("16:9", "4:3", "3:2", "1:1", "2:3", "3:4", "9:16"), + "resolutions": ("480p", "720p", "1080p"), + "durations": (1, 15), + "audio": False, # audio is native; no generate_audio key + "negative": False, + }, + "gemini-omni-flash": { + "display": "Gemini Omni Flash (via FAL)", + "speed": "~60-120s", + "price": "premium", + "strengths": "Google. Image-to-video with audio, physics-grounded motion, 3-10s.", + "tier": "premium", + # No text-to-video endpoint on FAL — image/reference only. + "text_endpoint": None, + "image_endpoint": "google/gemini-omni-flash/image-to-video", + "duration_int": True, + "aspect_ratios": ("16:9", "9:16"), + "resolutions": None, + "durations": (3, 10), + "audio": False, # audio is native; no generate_audio key + "negative": False, + }, "kling-v3-4k": { "display": "Kling v3 4K", "speed": "~120-300s", @@ -270,16 +382,25 @@ def _build_payload( # otherwise let the endpoint auto-crop / use its default if family.get("resolutions"): - if resolution in family["resolutions"]: - payload["resolution"] = resolution + # Some families use non-standard resolution enums (e.g. MiniMax H3's + # "768P"/"2K"/"4K"); resolution_aliases maps the tool's usual + # 720p/1080p-style values onto them. + aliases = family.get("resolution_aliases") or {} + resolved = aliases.get((resolution or "").lower(), resolution) + if resolved in family["resolutions"]: + payload["resolution"] = resolved # else: let the endpoint default clamped = _clamp_duration(family, duration) if clamped is not None and family.get("durations"): - # FAL exposes duration as a string in the queue API ("8" not 8). - # Some families (e.g. veo3.1) require a unit suffix ("4s" not "4"). - suffix = family.get("duration_suffix", "") - payload["duration"] = f"{clamped}{suffix}" + if family.get("duration_int"): + # A few endpoints (MiniMax H3) require duration as a JSON integer. + payload["duration"] = clamped + else: + # FAL exposes duration as a string in the queue API ("8" not 8). + # Some families (e.g. veo3.1) require a unit suffix ("4s" not "4"). + suffix = family.get("duration_suffix", "") + payload["duration"] = f"{clamped}{suffix}" if family.get("audio") and audio is not None: payload["generate_audio"] = bool(audio) @@ -287,6 +408,12 @@ def _build_payload( if family.get("negative") and negative_prompt: payload["negative_prompt"] = negative_prompt + # Keys the family's image-to-video endpoint rejects outright (e.g. + # Seedance 2.5 / MiniMax H3 derive aspect_ratio from the input image). + if image_url: + for key in family.get("image_drop_keys", ()): # type: ignore[assignment] + payload.pop(key, None) + return payload @@ -456,7 +583,7 @@ class FALVideoGenProvider(VideoGenProvider): return { "name": "FAL", "badge": "paid", - "tag": "LTX, Pixverse, Veo 3.1, Seedance 2.0, Kling 4K, Happy Horse — text-to-video & image-to-video", + "tag": "LTX, Pixverse, Seedance 2.0/2.5 + Mini, MiniMax H3, Veo 3.1, Kling 4K, Happy Horse — text-to-video & image-to-video", "env_vars": [ { "key": "FAL_KEY", @@ -608,7 +735,7 @@ class FALVideoGenProvider(VideoGenProvider): prompt=prompt, modality=modality_used, aspect_ratio=aspect_ratio if "aspect_ratio" in payload else "", - duration=int("".join(c for c in payload["duration"] if c.isdigit()) or "0") if "duration" in payload else 0, + duration=int("".join(c for c in str(payload["duration"]) if c.isdigit()) or "0") if "duration" in payload else 0, provider="fal", extra=extra, ) diff --git a/tests/plugins/video_gen/test_fal_plugin.py b/tests/plugins/video_gen/test_fal_plugin.py index e17494943e..f26364a923 100644 --- a/tests/plugins/video_gen/test_fal_plugin.py +++ b/tests/plugins/video_gen/test_fal_plugin.py @@ -49,6 +49,93 @@ def test_kling_4k_uses_start_image_url(): assert "image_url" not in payload +def test_minimax_h3_int_duration_and_resolution_alias(): + """MiniMax H3 requires duration as a JSON integer and uses the + 768P/2K/4K resolution enum — the tool's 720p/1080p values must map.""" + from plugins.video_gen.fal import FAL_FAMILIES, _build_payload + + meta = FAL_FAMILIES["minimax-h3"] + payload = _build_payload( + meta, + prompt="x", + image_url=None, + duration=7, + aspect_ratio="16:9", + resolution="720p", + negative_prompt=None, + audio=True, + seed=None, + ) + assert payload["duration"] == 7 and isinstance(payload["duration"], int) + assert payload["resolution"] == "768P" + assert payload["aspect_ratio"] == "16:9" + # H3 has no generate_audio key (audio is native/always-on) + assert "generate_audio" not in payload + + hi = _build_payload( + meta, prompt="x", image_url=None, duration=5, aspect_ratio="16:9", + resolution="1080p", negative_prompt=None, audio=None, seed=None, + ) + assert hi["resolution"] == "2K" + + +def test_image_drop_keys_strips_aspect_ratio_on_i2v(): + """Seedance 2.5 / MiniMax H3 / Grok 1.5 i2v endpoints derive the + aspect ratio from the input image; sending the key is rejected.""" + from plugins.video_gen.fal import FAL_FAMILIES, _build_payload + + for fid in ("seedance-2.5", "minimax-h3", "grok-imagine-1.5"): + meta = FAL_FAMILIES[fid] + i2v = _build_payload( + meta, prompt="x", image_url="https://example.com/i.png", + duration=5, aspect_ratio="16:9", resolution="480p", + negative_prompt=None, audio=None, seed=None, + ) + assert "aspect_ratio" not in i2v, fid + # ...but text-to-video keeps it + t2v = _build_payload( + meta, prompt="x", image_url=None, duration=5, + aspect_ratio="16:9", resolution="480p", + negative_prompt=None, audio=None, seed=None, + ) + assert t2v.get("aspect_ratio") == "16:9", fid + + +def test_seedance_25_string_duration_up_to_30(): + """Seedance 2.5 keeps the stringified duration convention and supports + the full 4-30s range.""" + from plugins.video_gen.fal import FAL_FAMILIES, _build_payload + + meta = FAL_FAMILIES["seedance-2.5"] + payload = _build_payload( + meta, prompt="x", image_url=None, duration=30, aspect_ratio="1:1", + resolution="480p", negative_prompt=None, audio=True, seed=None, + ) + assert payload["duration"] == "30" + assert payload["generate_audio"] is True + + +def test_gemini_omni_flash_is_image_only(): + """Gemini Omni Flash has no t2v endpoint on FAL — text jobs must + error cleanly instead of submitting to a None endpoint.""" + from plugins.video_gen.fal import FAL_FAMILIES + + meta = FAL_FAMILIES["gemini-omni-flash"] + assert meta.get("text_endpoint") is None + assert meta.get("image_endpoint") + + +def test_every_family_has_required_metadata(): + """Invariant: every family entry carries the picker-facing metadata and + at least one endpoint.""" + from plugins.video_gen.fal import FAL_FAMILIES + + for fid, meta in FAL_FAMILIES.items(): + assert meta.get("display"), fid + assert meta.get("tier") in {"cheap", "premium"}, fid + assert meta.get("text_endpoint") or meta.get("image_endpoint"), fid + + class TestFamilyRouting: """The headline behavior: image_url presence picks the endpoint.""" diff --git a/tests/tools/test_video_generation_tool_surface_matrix.py b/tests/tools/test_video_generation_tool_surface_matrix.py index d67be18ae9..88d2e5e781 100644 --- a/tests/tools/test_video_generation_tool_surface_matrix.py +++ b/tests/tools/test_video_generation_tool_surface_matrix.py @@ -148,6 +148,14 @@ def test_fal_text_only_routes_to_text_endpoint(matrix_env, family_id): {"prompt": "a dog running"}, ) + # Image-only families (e.g. gemini-omni-flash) must reject text-only + # jobs with a clean modality error instead of submitting anywhere. + if not FAL_FAMILIES[family_id].get("text_endpoint"): + assert result["success"] is False, family_id + assert result.get("error_type") == "modality_unsupported", result + assert not fal_calls, f"{family_id} submitted despite no text endpoint" + return + assert result["success"] is True, f"{family_id}: {result.get('error')}" assert result["modality"] == "text" assert result["provider"] == "fal" diff --git a/tools/image_generation_tool.py b/tools/image_generation_tool.py index fcb4e3c8fb..05a6e966ca 100644 --- a/tools/image_generation_tool.py +++ b/tools/image_generation_tool.py @@ -417,6 +417,206 @@ FAL_MODELS: Dict[str, Dict[str, Any]] = { }, "upscale": False, }, + "bytedance/seedream/v5/pro/text-to-image": { + "display": "Seedream 5.0 Pro", + "speed": "~10s", + "strengths": "ByteDance flagship, dense layouts, native text in 14 languages", + "price": "$0.0675/image (≤1536²)", + "size_style": "image_size_preset", + # Pro requires total pixels between 1024x1024 and 2048x2048 — + # explicit ImageSize dicts keep every aspect inside that window. + "sizes": { + "landscape": {"width": 2048, "height": 1152}, + "square": {"width": 1536, "height": 1536}, + "portrait": {"width": 1152, "height": 2048}, + }, + "defaults": { + "num_images": 1, + "output_format": "png", + "enable_safety_checker": False, + }, + "supports": { + "prompt", "image_size", "num_images", "output_format", + "sync_mode", "enable_safety_checker", + }, + "upscale": False, + # Region-precise editing with up to 10 reference images. + "edit_endpoint": "bytedance/seedream/v5/pro/edit", + "edit_supports": { + "prompt", "image_urls", "image_size", "num_images", + "output_format", "sync_mode", "enable_safety_checker", + }, + "max_reference_images": 10, + }, + "bytedance/seedream/v5/lite/text-to-image": { + "display": "Seedream 5.0 Lite", + "speed": "~5s", + "strengths": "Fast/cheap Seedream tier, high-res output", + "price": "$0.035/image", + "size_style": "image_size_preset", + # Lite wants total pixels between 2560x1440 and 4096x4096. + "sizes": { + "landscape": {"width": 3840, "height": 2160}, + "square": {"width": 2048, "height": 2048}, + "portrait": {"width": 2160, "height": 3840}, + }, + "defaults": { + "num_images": 1, + "enable_safety_checker": False, + }, + "supports": { + "prompt", "image_size", "num_images", "max_images", + "sync_mode", "enable_safety_checker", + }, + "upscale": False, + }, + "ideogram/v4/instant": { + "display": "Ideogram V4 (Instant)", + "speed": "<1s", + "strengths": "Latest Ideogram typography, posters/logos, instant", + "price": "$0.0075/MP", + "size_style": "image_size_preset", + "sizes": { + "landscape": "landscape_16_9", + "square": "square_hd", + "portrait": "portrait_16_9", + }, + "defaults": { + "expansion_model": "Medium", + "output_format": "png", + "enable_safety_checker": False, + }, + "supports": { + "prompt", "image_size", "expansion_model", "num_images", + "seed", "sync_mode", "enable_safety_checker", "output_format", + }, + "upscale": False, + }, + "ideogram/v4/fast": { + "display": "Ideogram V4 (Fast)", + "speed": "~1s", + "strengths": "Ideogram V4 quality tiers via rendering_speed", + "price": "$0.005-0.018/MP", + "size_style": "image_size_preset", + "sizes": { + "landscape": "landscape_16_9", + "square": "square_hd", + "portrait": "portrait_16_9", + }, + "defaults": { + "expansion_model": "Medium", + "rendering_speed": "BALANCED", + }, + "supports": { + "prompt", "image_size", "expansion_model", "rendering_speed", + "num_images", "seed", "sync_mode", + }, + "upscale": False, + }, + "alibaba/qwen-image-3/text-to-image": { + "display": "Qwen Image 3", + "speed": "~8s", + "strengths": "Complex CN/EN text rendering, prompt-guided resolution", + "price": "$0.04 (1K) / $0.075 (2K) per image", + "size_style": "image_size_preset", + "sizes": { + "landscape": "landscape_16_9", + "square": "square_hd", + "portrait": "portrait_16_9", + }, + "defaults": { + "num_images": 1, + "output_format": "png", + "enable_prompt_expansion": False, # avoid the LLM rewrite surprise + "enable_safety_checker": False, + }, + "supports": { + "prompt", "negative_prompt", "image_size", "num_images", + "seed", "sync_mode", "output_format", + "enable_prompt_expansion", "enable_safety_checker", + }, + "upscale": False, + # Qwen Image 3 edit: 1-3 reference images, identity-preserving edits. + "edit_endpoint": "alibaba/qwen-image-3/edit", + "edit_supports": { + "prompt", "image_urls", "negative_prompt", "num_images", + "seed", "sync_mode", "output_format", + "enable_prompt_expansion", "enable_safety_checker", + }, + "max_reference_images": 3, + }, + "microsoft/mai-image-2.5-pro": { + "display": "MAI Image 2.5 Pro", + "speed": "~10s", + "strengths": "Microsoft flagship, hero imagery, precise typography", + "price": "~$0.17/image", + "size_style": "aspect_ratio", + "sizes": { + "landscape": "16:9", + "square": "1:1", + "portrait": "9:16", + }, + "defaults": { + "num_images": 1, + "output_format": "png", + }, + "supports": { + "prompt", "aspect_ratio", "num_images", "output_format", + "sync_mode", + }, + "upscale": False, + }, + "google/nano-banana-2-lite": { + "display": "Nano Banana 2 Lite", + "speed": "<2s", + "strengths": "Gemini image family, sub-2s, 14 aspect ratios incl. extreme", + "price": "~$0.04/image (1K fixed)", + "size_style": "aspect_ratio", + "sizes": { + "landscape": "16:9", + "square": "1:1", + "portrait": "9:16", + }, + "defaults": { + "num_images": 1, + "output_format": "png", + "safety_tolerance": "5", + }, + "supports": { + "prompt", "aspect_ratio", "num_images", "seed", + "output_format", "safety_tolerance", "sync_mode", + "system_prompt", "limit_generations", "thinking_level", + }, + "upscale": False, + # Fast multi-turn local edits with reference images via `image_urls`. + "edit_endpoint": "google/nano-banana-2-lite/edit", + "edit_supports": { + "prompt", "image_urls", "aspect_ratio", "num_images", + "seed", "output_format", "safety_tolerance", "sync_mode", + "system_prompt", + }, + "max_reference_images": 4, + }, + "fal-ai/recraft/v4.1/text-to-image": { + "display": "Recraft V4.1", + "speed": "~8s", + "strengths": "Design-first raster, brand systems, editorial", + "price": "$0.035/image", + "size_style": "image_size_preset", + "sizes": { + "landscape": "landscape_16_9", + "square": "square_hd", + "portrait": "portrait_16_9", + }, + "defaults": { + "enable_safety_checker": False, + }, + "supports": { + "prompt", "image_size", "enable_safety_checker", + "colors", "background_color", + }, + "upscale": False, + }, } # Default model is the fastest reasonable option. Kept cheap and sub-1s.