视频理解
要了解视频生成,请参阅 Gemini Omni Flash 指导。
Gemini 模型可以处理视频,从而支持许多以前需要特定领域模型的前沿开发人员用例。 Gemini 的一些视觉功能包括:描述、分段和从视频中提取信息,回答有关视频内容的问题,以及引用视频中的特定时间戳。
您可以通过以下方式提供视频作为 Gemini 的输入:
| 输入法 | 最大尺寸 | 推荐用例 |
|---|---|---|
| 文件 API | 20GB(付费)/2GB(免费) | 大文件(100MB+)、长视频(10分钟以上)、可重复使用的文件。 |
| 云存储注册 | 2GB(每个文件,无存储限制) | 大文件 (100MB+)、长视频 (10 分钟以上)、持久、可重复使用的文件。 |
| 内联数据 | < 100MB | 文件小(<100MB)、持续时间短(<1分钟)、一次性输入。 |
| YouTube 网址 | N/A | 公开 YouTube 视频。 |
要了解其他文件输入方法,例如使用外部 URL 或 Google Cloud 中存储的文件,请参阅 文件输入方式 指导。
上传视频文件 #
以下代码下载示例视频,然后使用 Files API,等待处理完毕,然后使用上传的文件参考来总结视频。
from google import genai
import time
client = genai.Client()
myfile = client.files.upload(file="path/to/sample.mp4")
while not myfile.state or myfile.state.name != "ACTIVE":
print("Processing video...")
time.sleep(5)
myfile = client.files.get(name=myfile.name)
interaction = client.interactions.create(
model="gemini-4-hydrogen",
input=[
{"type": "video", "uri": myfile.uri, "mime_type": myfile.mime_type},
{"type": "text", "text": "Summarize this video. Then create a quiz with an answer key based on the information in this video."}
]
)
print(interaction.output_text)
import { GoogleGenAI } from "@google/genai";
const ai = new GoogleGenAI({});
async function main() {
const myfile = await ai.files.upload({
file: "path/to/sample.mp4",
config: { mimeType: "video/mp4" },
});
let getFile = await ai.files.get({ name: myfile.name || '' });
while (getFile.state === 'PROCESSING') {
getFile = await ai.files.get({ name: myfile.name || '' });
console.log(`current file status: ${getFile.state}`);
console.log('File is still processing, retrying in 5 seconds');
await new Promise((resolve) => {
setTimeout(resolve, 5000);
});
}
if (getFile.state === 'FAILED') {
throw new Error('File processing failed.');
}
const interaction = await ai.interactions.create({
model: "gemini-4-hydrogen",
input: [
{ type: "video", uri: myfile.uri, mime_type: myfile.mimeType },
{ type: "text", text: "Summarize this video. Then create a quiz with an answer key based on the information in this video." }
],
});
console.log(interaction.output_text);
}
await main();
import com.google.genai.Client;
import com.google.genai.gaos.models.interactions.Content;
import com.google.genai.gaos.models.interactions.CreateModelInteraction;
import com.google.genai.gaos.models.interactions.Interaction;
import com.google.genai.gaos.models.interactions.InteractionsInput;
import com.google.genai.gaos.models.interactions.Model;
import com.google.genai.gaos.models.interactions.TextContent;
import com.google.genai.gaos.models.interactions.VideoContent;
import com.google.genai.gaos.models.interactions.VideoContentMimeType;
import com.google.genai.gaos.models.operations.CreateInteractionRequestBody;
import com.google.genai.types.File;
import com.google.genai.types.FileState;
import com.google.genai.types.UploadFileConfig;
import java.util.Arrays;
import java.util.List;
Client client = new Client();
File myfile =
client.files.upload(
"path/to/sample.mp4", UploadFileConfig.builder().mimeType("video/mp4").build());
while (!myfile.state().isPresent()
|| myfile.state().get().knownEnum() != FileState.Known.ACTIVE) {
System.out.println("Processing video...");
Thread.sleep(5000);
myfile = client.files.get(myfile.name().get(), null);
}
Content videoContent =
VideoContent.builder()
.uri(myfile.uri().get())
.mimeType(VideoContentMimeType.of(myfile.mimeType().get()))
.build();
Content textContent =
TextContent.builder()
.text(
"Summarize this video. Then create a quiz with an answer key based on the information in this video.")
.build();
List<Content> contents = Arrays.asList(videoContent, textContent);
CreateModelInteraction params =
CreateModelInteraction.builder()
.model(Model.of("gemini-4-hydrogen"))
.input(InteractionsInput.ofContent(contents))
.build();
Interaction interaction =
client.interactions.create(CreateInteractionRequestBody.of(params)).interaction().get();
System.out.println(interaction.outputText().orElse(""));
package main
import (
"context"
"fmt"
"log"
"time"
"google.golang.org/genai"
"google.golang.org/genai/interactions/models/interactions"
"google.golang.org/genai/interactions/models/operations"
)
func main() {
ctx := context.Background()
client, err := genai.NewClient(ctx, nil)
if err != nil {
log.Fatal(err)
}
myfile, err := client.Files.UploadFromPath(ctx, "path/to/sample.mp4", &genai.UploadFileConfig{
MIMEType: "video/mp4",
})
if err != nil {
log.Fatal(err)
}
for myfile.State != genai.FileStateActive {
fmt.Println("Processing video...")
time.Sleep(5 * time.Second)
myfile, err = client.Files.Get(ctx, myfile.Name, nil)
if err != nil {
log.Fatal(err)
}
}
contents := []interactions.Content{
interactions.NewContent(interactions.VideoContent{
URI: genai.Ptr(myfile.URI),
MimeType: interactions.VideoContentMimeType(myfile.MIMEType).ToPointer(),
}),
interactions.NewContent(interactions.TextContent{
Text: "Summarize this video. Then create a quiz with an answer key based on the information in this video.",
}),
}
res, err := client.Interactions.Create(ctx, operations.CreateInteractionRequest{
Body: operations.NewCreateInteractionRequestBody(interactions.CreateModelInteraction{
Model: interactions.Model("gemini-4-hydrogen"),
Input: interactions.NewInteractionsInput(contents),
}),
})
if err != nil {
log.Fatal(err)
}
if res.Interaction.OutputText != nil {
fmt.Println(*res.Interaction.OutputText)
}
}
VIDEO_PATH="path/to/sample.mp4"
MIME_TYPE=$(file -b --mime-type "${VIDEO_PATH}")
NUM_BYTES=$(wc -c < "${VIDEO_PATH}")
DISPLAY_NAME=VIDEO
tmp_header_file=upload-header.tmp
echo "Starting file upload..."
curl "https://generativelanguage.googleapis.com/upload/v1beta/files" \
-H "x-goog-api-key: $GEMINI_API_KEY" \
-D ${tmp_header_file} \
-H "X-Goog-Upload-Protocol: resumable" \
-H "X-Goog-Upload-Command: start" \
-H "X-Goog-Upload-Header-Content-Length: ${NUM_BYTES}" \
-H "X-Goog-Upload-Header-Content-Type: ${MIME_TYPE}" \
-H "Content-Type: application/json" \
-d "{'file': {'display_name': '${DISPLAY_NAME}'}}" 2> /dev/null
upload_url=$(grep -i "x-goog-upload-url: " "${tmp_header_file}" | cut -d" " -f2 | tr -d "\r")
rm "${tmp_header_file}"
echo "Uploading video data..."
curl "${upload_url}" \
-H "Content-Length: ${NUM_BYTES}" \
-H "X-Goog-Upload-Offset: 0" \
-H "X-Goog-Upload-Command: upload, finalize" \
--data-binary "@${VIDEO_PATH}" 2> /dev/null > file_info.json
file_uri=$(jq -r ".file.uri" file_info.json)
file_name=$(jq -r ".file.name" file_info.json)
echo file_uri=$file_uri
echo "File uploaded successfully. File URI: ${file_uri}"
echo "Waiting for file to be processed..."
while true; do
curl -s "https://generativelanguage.googleapis.com/v1beta/${file_name}" \
-H "x-goog-api-key: $GEMINI_API_KEY" > file_status.json
state=$(jq -r ".state" file_status.json)
echo "Current state: $state"
if [ "$state" == "ACTIVE" ]; then
break
elif [ "$state" == "FAILED" ]; then
echo "File processing failed."
exit 1
fi
sleep 5
done
echo "Generating content from video..."
curl -X POST "https://generativelanguage.googleapis.com/v1beta/interactions" \
-H "x-goog-api-key: $GEMINI_API_KEY" \
-H 'Content-Type: application/json' \
-d '{
"model": "gemini-4-hydrogen",
"input": [
{"type": "video", "uri": "'${file_uri}'", "mime_type": "'${MIME_TYPE}'"},
{"type": "text", "text": "Summarize this video. Then create a quiz with an answer key based on the information in this video."}
]
}' 2> /dev/null > response.json
jq ".steps[].content[0].text" response.json
FILE=$(gemini-api files upload path/to/sample.mp4 --wait)
gemini-api analyze "Summarize this video. Then create a quiz with an answer key based on the information in this video." \
-i "$FILE"
要优化 token 效率和性能,请考虑使用 智能体视频处理.
当总请求大小(包括文件、文本提示、系统指令等)大于 20 MB、视频持续时间较长或打算在多个提示中使用相同视频时,请始终使用 Files API。文件 API 直接接受视频文件格式。
要了解有关使用媒体文件的更多信息,请参阅 Files API.
内嵌传递视频数据 #
您可以直接在请求中传递较小的视频,而不是使用文件 API 上传视频文件。这适用于总请求大小低于 20MB 的较短视频。
以下是提供内嵌视频数据的示例:
from google import genai
import base64
video_file_name = "/path/to/your/video.mp4"
video_bytes = open(video_file_name, 'rb').read()
client = genai.Client()
interaction = client.interactions.create(
model='gemini-4-hydrogen',
input=[
{"type": "text", "text": "Please summarize the video in 3 sentences."},
{
"type": "video",
"data": base64.b64encode(video_bytes).decode('utf-8'),
"mime_type": "video/mp4"
}
]
)
print(interaction.output_text)
import { GoogleGenAI } from "@google/genai";
import * as fs from "node:fs";
const ai = new GoogleGenAI({});
const base64VideoFile = fs.readFileSync("path/to/small-sample.mp4", {
encoding: "base64",
});
const interaction = await ai.interactions.create({
model: "gemini-4-hydrogen",
input: [
{ type: "text", text: "Please summarize the video in 3 sentences." },
{
type: "video",
data: base64VideoFile,
mime_type: "video/mp4",
}
],
});
console.log(interaction.output_text);
import com.google.genai.Client;
import com.google.genai.gaos.models.interactions.Content;
import com.google.genai.gaos.models.interactions.CreateModelInteraction;
import com.google.genai.gaos.models.interactions.Interaction;
import com.google.genai.gaos.models.interactions.InteractionsInput;
import com.google.genai.gaos.models.interactions.Model;
import com.google.genai.gaos.models.interactions.TextContent;
import com.google.genai.gaos.models.interactions.VideoContent;
import com.google.genai.gaos.models.interactions.VideoContentMimeType;
import com.google.genai.gaos.models.operations.CreateInteractionRequestBody;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.Base64;
import java.util.List;
String videoFileName = "/path/to/your/video.mp4";
byte[] videoBytes = Files.readAllBytes(Paths.get(videoFileName));
String base64Video = Base64.getEncoder().encodeToString(videoBytes);
Client client = new Client();
Content textContent =
TextContent.builder().text("Please summarize the video in 3 sentences.").build();
Content videoContent =
VideoContent.builder()
.data(base64Video)
.mimeType(VideoContentMimeType.VIDEO_MP4)
.build();
List<Content> contents = Arrays.asList(textContent, videoContent);
CreateModelInteraction params =
CreateModelInteraction.builder()
.model(Model.of("gemini-4-hydrogen"))
.input(InteractionsInput.ofContent(contents))
.build();
Interaction interaction =
client.interactions.create(CreateInteractionRequestBody.of(params)).interaction().get();
System.out.println(interaction.outputText().orElse(""));
package main
import (
"context"
"encoding/base64"
"fmt"
"log"
"os"
"google.golang.org/genai"
"google.golang.org/genai/interactions/models/interactions"
"google.golang.org/genai/interactions/models/operations"
)
func main() {
ctx := context.Background()
client, err := genai.NewClient(ctx, nil)
if err != nil {
log.Fatal(err)
}
videoFileName := "/path/to/your/video.mp4"
videoBytes, err := os.ReadFile(videoFileName)
if err != nil {
log.Fatal(err)
}
base64Video := base64.StdEncoding.EncodeToString(videoBytes)
contents := []interactions.Content{
interactions.NewContent(interactions.TextContent{
Text: "Please summarize the video in 3 sentences.",
}),
interactions.NewContent(interactions.VideoContent{
Data: genai.Ptr(base64Video),
MimeType: interactions.VideoContentMimeTypeVideoMp4.ToPointer(),
}),
}
res, err := client.Interactions.Create(ctx, operations.CreateInteractionRequest{
Body: operations.NewCreateInteractionRequestBody(interactions.CreateModelInteraction{
Model: interactions.Model("gemini-4-hydrogen"),
Input: interactions.NewInteractionsInput(contents),
}),
})
if err != nil {
log.Fatal(err)
}
if res.Interaction.OutputText != nil {
fmt.Println(*res.Interaction.OutputText)
}
}
VIDEO_PATH=/path/to/your/video.mp4
if [[ "$(base64 --version 2>&1)" = *"FreeBSD"* ]]; then
B64FLAGS="--input"
else
B64FLAGS="-w0"
fi
curl -X POST "https://generativelanguage.googleapis.com/v1beta/interactions" \
-H "x-goog-api-key: $GEMINI_API_KEY" \
-H 'Content-Type: application/json' \
-d '{
"model": "gemini-4-hydrogen",
"input": [
{"type": "text", "text": "Please summarize the video in 3 sentences."},
{
"type": "video",
"data": "'$(base64 $B64FLAGS $VIDEO_PATH)'",
"mime_type": "video/mp4"
}
]
}' 2> /dev/null
gemini-api analyze "Please summarize the video in 3 sentences." -i /path/to/your/video.mp4
传递 YouTube URL #
您可以将 YouTube URL 作为请求的一部分直接传递给 Gemini API,如下所示:
from google import genai
client = genai.Client()
interaction = client.interactions.create(
model='gemini-4-hydrogen',
input=[
{"type": "text", "text": "Please summarize the video in 3 sentences."},
{
"type": "video",
"uri": "https://www.youtube.com/watch?v=9hE5-98ZeCg"
}
]
)
print(interaction.output_text)
import { GoogleGenAI } from "@google/genai";
const ai = new GoogleGenAI({});
const interaction = await ai.interactions.create({
model: "gemini-4-hydrogen",
input: [
{ type: "text", text: "Please summarize the video in 3 sentences." },
{
type: "video",
uri: "https://www.youtube.com/watch?v=9hE5-98ZeCg",
}
],
});
console.log(interaction.output_text);
import com.google.genai.Client;
import com.google.genai.gaos.models.interactions.Content;
import com.google.genai.gaos.models.interactions.CreateModelInteraction;
import com.google.genai.gaos.models.interactions.Interaction;
import com.google.genai.gaos.models.interactions.InteractionsInput;
import com.google.genai.gaos.models.interactions.Model;
import com.google.genai.gaos.models.interactions.TextContent;
import com.google.genai.gaos.models.interactions.VideoContent;
import com.google.genai.gaos.models.operations.CreateInteractionRequestBody;
import java.util.Arrays;
import java.util.List;
Client client = new Client();
Content textContent =
TextContent.builder().text("Please summarize the video in 3 sentences.").build();
Content videoContent =
VideoContent.builder()
.uri("https://www.youtube.com/watch?v=9hE5-98ZeCg")
.build();
List<Content> contents = Arrays.asList(textContent, videoContent);
CreateModelInteraction params =
CreateModelInteraction.builder()
.model(Model.of("gemini-4-hydrogen"))
.input(InteractionsInput.ofContent(contents))
.build();
Interaction interaction =
client.interactions.create(CreateInteractionRequestBody.of(params)).interaction().get();
System.out.println(interaction.outputText().orElse(""));
package main
import (
"context"
"fmt"
"log"
"google.golang.org/genai"
"google.golang.org/genai/interactions/models/interactions"
"google.golang.org/genai/interactions/models/operations"
)
func main() {
ctx := context.Background()
client, err := genai.NewClient(ctx, nil)
if err != nil {
log.Fatal(err)
}
contents := []interactions.Content{
interactions.NewContent(interactions.TextContent{
Text: "Please summarize the video in 3 sentences.",
}),
interactions.NewContent(interactions.VideoContent{
URI: genai.Ptr("https://www.youtube.com/watch?v=9hE5-98ZeCg"),
}),
}
res, err := client.Interactions.Create(ctx, operations.CreateInteractionRequest{
Body: operations.NewCreateInteractionRequestBody(interactions.CreateModelInteraction{
Model: interactions.Model("gemini-4-hydrogen"),
Input: interactions.NewInteractionsInput(contents),
}),
})
if err != nil {
log.Fatal(err)
}
if res.Interaction.OutputText != nil {
fmt.Println(*res.Interaction.OutputText)
}
}
curl -X POST "https://generativelanguage.googleapis.com/v1beta/interactions" \
-H "x-goog-api-key: $GEMINI_API_KEY" \
-H 'Content-Type: application/json' \
-d '{
"model": "gemini-4-hydrogen",
"input": [
{"type": "text", "text": "Please summarize the video in 3 sentences."},
{
"type": "video",
"uri": "https://www.youtube.com/watch?v=9hE5-98ZeCg"
}
]
}' 2> /dev/null
gemini-api analyze "Please summarize the video in 3 sentences." \
-i "https://www.youtube.com/watch?v=9hE5-98ZeCg"
限制:
- 对于免费套餐,您每天不能上传超过 8 小时的 YouTube 视频。
- 对于付费套餐,视频长度没有限制。
- 对于 Gemini 2.5, 之前的型号,您只能根据请求上传 1 视频。对于 Gemini 2.5 及更高版本,每个请求最多可以上传 10 视频。
- 您只能上传公共视频(不能上传私有或不公开的视频)。
智能体视频理解 #
默认情况下,所有型号(包括 Gemini 4)的视频输入均使用静态处理(以 1 FPS 提取帧)。 Gemini 4 Carbon、Gemini 4 Argon、Gemini 4 Hydrogen、3.8 Flash、3.6 Flash 和 3.5 Flash Lite 型号还支持 智能体视频理解,一种选择加入模式,模型动态探索视频时间线,有选择地检查记录并根据提示动态自适应调整帧速率和分辨率。
| 模式 | 说明 | 支持的模型 |
|---|---|---|
| 静止的 (默认) | 以固定速率 (1 FPS) 提取帧并将其一次性放入上下文中。所有 Gemini 型号的默认设置,包括 Gemini 4。 | 所有 Gemini 型号 |
| 智能体 | 该模型动态导航视频时间线,根据提示仅加载所需的内容。最多88% 更多的token-高效和〜7% 长篇内容的质量更高。 | Gemini 4 Carbon、Gemini 4 Argon、Gemini 4 Hydrogen、3.8 Flash、3.6 Flash、3.5 Flash精简版 |
选择处理模式 #
选择与您的视频长度和查询类型匹配的处理模式:
- 静态(默认): 最适合短剪辑(5 分钟以下)、整个录制相关的广泛的完整视频摘要、延迟敏感的查询或整个剪辑的帧级精度。在短视频或需要检查大部分视频的宽泛提示上,静态处理避免了额外的思考 token (
total_thought_tokens)和工具调用往返(total_tool_use_tokens)由时间线导航引起。 - 智能体: 最适合长视频(超过 5–10 分钟)、大海捞针检索或针对特定时刻或主题的查询。该模型首先检查文字记录,并根据需要仅加载相关的视觉片段,从而将视频输入 token 的使用量减少高达 88%,而无需填充上下文窗口。
设置处理模式 #
要使用智能体视频理解,请设置 "processing": "agentic" 关于视频部分 input 数组。
智能体视频处理在长视频上可能需要更长的时间,因为模型会执行多次检查来导航时间线。为了避免在网络连接断开时丢失进度,以下示例使用 background 参数以在服务器上异步运行任务。您还可以通过省略来同步运行请求而无需后台执行 background 参数,适用于较短的视频。
有关轮询、检索结果和管理后台任务的详细信息,请参阅 后台执行 指导。
import time
from google import genai
client = genai.Client()
# Upload a long video
video_file = client.files.upload(file="path/to/lecture.mp4")
while video_file.state.name == "PROCESSING":
time.sleep(2)
video_file = client.files.get(name=video_file.name)
# Use agentic processing with background execution
interaction = client.interactions.create(
model="gemini-4-hydrogen",
input=[
{
"type": "video",
"uri": video_file.uri,
"mime_type": video_file.mime_type,
"processing": "agentic"
},
{"type": "text", "text": "What are the three main arguments presented?"}
],
background=True,
)
while interaction.status == "in_progress":
time.sleep(5)
interaction = client.interactions.get(interaction.id)
print(interaction.output_text)
import { GoogleGenAI } from "@google/genai";
const ai = new GoogleGenAI({});
// Upload a long video
let videoFile = await ai.files.upload({
file: "path/to/lecture.mp4",
config: { mimeType: "video/mp4" }
});
while (videoFile.state === "PROCESSING") {
await new Promise((resolve) => setTimeout(resolve, 2000));
videoFile = await ai.files.get({ name: videoFile.name || '' });
}
// Use agentic processing with background execution
let interaction = await ai.interactions.create({
model: "gemini-4-hydrogen",
input: [
{
type: "video",
uri: videoFile.uri,
mime_type: videoFile.mimeType,
processing: "agentic"
},
{ type: "text", text: "What are the three main arguments presented?" }
],
background: true,
});
while (interaction.status === "in_progress") {
await new Promise((resolve) => setTimeout(resolve, 5000));
interaction = await ai.interactions.get(interaction.id);
}
console.log(interaction.output_text);
# 1. Start the interaction in the background
interaction=$(curl -s -X POST "https://generativelanguage.googleapis.com/v1beta/interactions" \
-H "x-goog-api-key: $GEMINI_API_KEY" \
-H 'Content-Type: application/json' \
-d '{
"model": "gemini-4-hydrogen",
"input": [
{
"type": "video",
"uri": "'${file_uri}'",
"mime_type": "video/mp4",
"processing": "agentic"
},
{"type": "text", "text": "What are the three main arguments presented?"}
],
"background": true
}')
id=$(echo "$interaction" | jq -r '.id')
# 2. Poll until the interaction finishes
while [ "$(echo "$interaction" | jq -r '.status')" = "in_progress" ]; do
sleep 5
interaction=$(curl -s "https://generativelanguage.googleapis.com/v1beta/interactions/$id" \
-H "x-goog-api-key: $GEMINI_API_KEY")
done
echo "$interaction" | jq '.'
FILE=$(gemini-api files upload path/to/lecture.mp4 --wait)
ID=$(gemini-api agent run -m gemini-4-hydrogen --background --stream=false --jq '.id' --body '{"input": [
{"type": "video", "uri": "'"$FILE"'", "processing": "agentic"},
{"type": "text", "text": "What are the three main arguments presented?"}
]}')
gemini-api agent status "$ID" --stream
应对步骤 #
智能体处理添加了两种新的步骤类型 steps 数组:
processing_call:模型请求视频片段或音频脚本,由id.processing_result:该负载的结果,链接为call_id.
这些似乎与 thought 步骤(启用摘要时)并在最终步骤之前 model_output 步骤。它们可用于在 UI 中显示进度跟踪,但不需要响应。
以下示例显示了具有交错处理步骤的响应负载:
{
"steps": [
{
"type": "thought",
"signature": "sig_thought_1",
"summary": [
{
"type": "text",
"text": "Inspecting transcript for key discussion topics..."
}
]
},
{
"type": "processing_call",
"id": "call_01",
"signature": "sig_call_01"
},
{
"type": "processing_result",
"call_id": "call_01",
"signature": "sig_result_01"
},
{
"type": "thought",
"signature": "sig_thought_2",
"summary": [
{
"type": "text",
"text": "Loading visual frames to verify slide content..."
}
]
},
{
"type": "processing_call",
"id": "call_02",
"signature": "sig_call_02"
},
{
"type": "processing_result",
"call_id": "call_02",
"signature": "sig_result_02"
},
{
"type": "thought",
"signature": "sig_thought_3",
"summary": [
{
"type": "text",
"text": "Synthesizing answer from gathered evidence..."
}
]
},
{
"type": "model_output",
"content": [
{
"type": "text",
"text": "The three main arguments presented in the lecture are..."
}
]
}
]
}
跨视频混合处理模式 #
为同一个请求中的每个视频设置不同的处理模式:
import time
from google import genai
client = genai.Client()
lecture = client.files.upload(file="path/to/long-lecture.mp4")
experiment = client.files.upload(file="path/to/short-experiment.mp4")
# Use agentic processing with background execution
interaction = client.interactions.create(
model="gemini-4-hydrogen",
input=[
{
"type": "video",
"uri": lecture.uri,
"mime_type": lecture.mime_type,
"processing": "agentic" # Use agentic video understanding
},
{
"type": "video",
"uri": experiment.uri,
"mime_type": experiment.mime_type,
"processing": "static" # Use static processing
},
{"type": "text", "text": "Compare the lecture content with the experiment results."}
],
background=True,
)
while interaction.status == "in_progress":
time.sleep(5)
interaction = client.interactions.get(interaction.id)
print(interaction.output_text)
import { GoogleGenAI } from "@google/genai";
const ai = new GoogleGenAI({});
const lecture = await ai.files.upload({
file: "path/to/long-lecture.mp4",
config: { mimeType: "video/mp4" }
});
const experiment = await ai.files.upload({
file: "path/to/short-experiment.mp4",
config: { mimeType: "video/mp4" }
});
// Use agentic processing with background execution
let interaction = await ai.interactions.create({
model: "gemini-4-hydrogen",
input: [
{
type: "video",
uri: lecture.uri,
mime_type: lecture.mimeType,
processing: "agentic" // Use agentic video understanding
},
{
type: "video",
uri: experiment.uri,
mime_type: experiment.mimeType,
processing: "static" // Use static processing
},
{ type: "text", text: "Compare the lecture content with the experiment results." }
],
background: true,
});
while (interaction.status === "in_progress") {
await new Promise((resolve) => setTimeout(resolve, 5000));
interaction = await ai.interactions.get(interaction.id);
}
console.log(interaction.output_text);
# 1. Start the interaction in the background
interaction=$(curl -s -X POST "https://generativelanguage.googleapis.com/v1beta/interactions" \
-H "x-goog-api-key: $GEMINI_API_KEY" \
-H 'Content-Type: application/json' \
-d '{
"model": "gemini-4-hydrogen",
"input": [
{
"type": "video",
"uri": "'${lecture_uri}'",
"mime_type": "video/mp4",
"processing": "agentic"
},
{
"type": "video",
"uri": "'${experiment_uri}'",
"mime_type": "video/mp4",
"processing": "static"
},
{"type": "text", "text": "Compare the lecture content with the experiment results."}
],
"background": true
}')
id=$(echo "$interaction" | jq -r '.id')
# 2. Poll until the interaction finishes
while [ "$(echo "$interaction" | jq -r '.status')" = "in_progress" ]; do
sleep 5
interaction=$(curl -s "https://generativelanguage.googleapis.com/v1beta/interactions/$id" \
-H "x-goog-api-key: $GEMINI_API_KEY")
done
echo "$interaction" | jq '.'
多轮视频对话 #
对话中的视频上下文得以保留。使用智能体处理时:
- 有状态模式 (使用
previous_interaction_id):服务器保留视频上下文。不需要额外的处理。 - 无状态模式 (使用
step_list):无状态模式下,响应包括processing_call和processing_result对视频上下文进行编码的步骤。您必须在下一个请求中包含响应中的所有步骤step_list以保留视频上下文。虽然忽略它们目前不会返回 API 错误,但视频上下文会丢失,从而显着降低后续问题的响应质量。请注意,后续请求中发送的返回步骤会影响输入 token 计数。
参考内容中的时间戳 #
您可以使用以下形式的时间戳询问有关视频中特定时间点的问题 MM:SS.
prompt = "What are the examples given at 00:05 and 00:10 supposed to show us?"
const prompt = "What are the examples given at 00:05 and 00:10 supposed to show us?";
String prompt = "What are the examples given at 00:05 and 00:10 supposed to show us?";
prompt := "What are the examples given at 00:05 and 00:10 supposed to show us?"
PROMPT="What are the examples given at 00:05 and 00:10 supposed to show us?"
从视频中提取详细见解 #
Gemini 型号通过处理来自以下两个方面的信息,提供了理解视频内容的强大功能: 音频和视频 溪流。这使您可以提取丰富的详细信息,包括生成视频中发生的情况的描述并回答有关其内容的问题。
对于视觉描述,模型以以下速率对视频进行采样 1 每秒帧数 (FPS)。此默认采样率适用于大多数内容,但请注意,它可能会错过快速运动或快速场景变化的视频中的细节。
prompt = "Describe the key events in this video, providing both audio and visual details. Include timestamps for salient moments."
const prompt = "Describe the key events in this video, providing both audio and visual details. Include timestamps for salient moments.";
String prompt =
"Describe the key events in this video, providing both audio and visual details. Include timestamps for salient moments.";
prompt := "Describe the key events in this video, providing both audio and visual details. Include timestamps for salient moments."
PROMPT="Describe the key events in this video, providing both audio and visual details. Include timestamps for salient moments."
自定义视频处理 #
您可以通过设置剪辑间隔或提供自定义帧速率采样来自定义 Gemini API 中的视频处理。仅在处理视频时才支持这些自定义选项 "static" 模式。
设置剪辑间隔 #
您可以通过设置剪辑视频 start_offset 和 end_offset 在 processing 配置对象。将每个偏移量指定为以秒结尾的字符串 "s",例如 "1200s" 或 "10.5s".
interaction = client.interactions.create(
model="gemini-4-hydrogen",
input=[
{
"type": "video",
"uri": video_file.uri,
"mime_type": video_file.mime_type,
"processing": {
"type": "static",
"start_offset": "1200s",
"end_offset": "1500s",
},
},
{"type": "text", "text": "Summarize this section of the video."},
],
)
print(interaction.output_text)
const interaction = await ai.interactions.create({
model: "gemini-4-hydrogen",
input: [
{
type: "video",
uri: videoFile.uri,
mime_type: videoFile.mimeType,
processing: {
type: "static",
start_offset: "1200s",
end_offset: "1500s",
},
},
{ type: "text", text: "Summarize this section of the video." },
],
});
console.log(interaction.output_text);
curl -X POST "https://generativelanguage.googleapis.com/v1beta/interactions" \
-H "x-goog-api-key: $GEMINI_API_KEY" \
-H 'Content-Type: application/json' \
-d '{
"model": "gemini-4-hydrogen",
"input": [
{
"type": "video",
"uri": "'${file_uri}'",
"mime_type": "video/mp4",
"processing": {
"type": "static",
"start_offset": "1200s",
"end_offset": "1500s"
}
},
{"type": "text", "text": "Summarize this section of the video."}
]
}' 2> /dev/null
FILE=$(gemini-api files upload path/to/sample.mp4 --wait)
gemini-api generate -m gemini-4-hydrogen --body '{"input": [
{"type": "video", "uri": "'"$FILE"'",
"processing": {"type": "static", "start_offset": "1200s", "end_offset": "1500s"}},
{"type": "text", "text": "Summarize this section of the video."}
]}'
设置自定义帧速率 #
您可以通过传递一个来设置自定义帧速率采样 fps 论据中的 processing 配置对象。
interaction = client.interactions.create(
model="gemini-4-hydrogen",
input=[
{
"type": "video",
"uri": video_file.uri,
"mime_type": video_file.mime_type,
"processing": {
"type": "static",
"fps": 0.5, # Sample 1 frame every 2 seconds
},
},
{"type": "text", "text": "Describe the scene changes in this video."},
],
)
print(interaction.output_text)
const interaction = await ai.interactions.create({
model: "gemini-4-hydrogen",
input: [
{
type: "video",
uri: videoFile.uri,
mime_type: videoFile.mimeType,
processing: {
type: "static",
fps: 0.5, // Sample 1 frame every 2 seconds
},
},
{ type: "text", text: "Describe the scene changes in this video." },
],
});
console.log(interaction.output_text);
curl -X POST "https://generativelanguage.googleapis.com/v1beta/interactions" \
-H "x-goog-api-key: $GEMINI_API_KEY" \
-H 'Content-Type: application/json' \
-d '{
"model": "gemini-4-hydrogen",
"input": [
{
"type": "video",
"uri": "'${file_uri}'",
"mime_type": "video/mp4",
"processing": {
"type": "static",
"fps": 0.5
}
},
{"type": "text", "text": "Describe the scene changes in this video."}
]
}' 2> /dev/null
支持的视频格式 #
Gemini 支持以下视频格式 MIME 类型:
video/mp4video/mpegvideo/movvideo/avivideo/x-flvvideo/mpgvideo/webmvideo/wmvvideo/3gpp
有关视频的技术细节 #
支持的模型和上下文:所有Gemini型号都可以处理视频数据。 - 配备 1M 上下文窗口的型号默认可处理长达 3 小时的视频(在低媒体分辨率下),或在高媒体分辨率下处理长达 1 小时的视频。
加工方式:Gemini 4 Carbon、Gemini 4 Argon、3.8 Flash、3.6 Flash、3.5 Flash Lite 及更高版本支持两种视频处理模式:- 静止的:在 1 FPS 中提取帧并将其放入上下文中(所有型号均默认)。音频以 1Kbps(单通道)处理。时间戳每秒添加一次。最适合短剪辑或每帧都很重要的情况(例如逐帧检查)。请注意,由于 1 FPS 采样率,快速动作序列可能会丢失细节。
- 智能体:模型会动态浏览视频,按需加载转写文本、视频帧和音频。对于长视频内容,token 用量最多可减少 88%;但对于短片(不足 5 分钟)或宽泛查询,内部推理及工具往返可能增加首个 token 延迟(TTFT)和 token 总用量。最适合长视频和有针对性的查询,以优化 token 成本与回答质量。支持 Gemini 4 Carbon、Gemini 4 Argon、3.8 Flash、3.6 Flash 和 3.5 Flash Lite。参见 智能体视频理解 了解详情。
代币计算(静态模式):视频的每一秒都按如下方式标记: - 各个帧(以 1 FPS 采样): - 如果
media_resolution设置为低时,帧以每帧 66 token 标记。否则,帧将在每帧 258 token 处进行标记。
音频:每秒 32 token。
元数据也包括在内。
总计:默认(低)媒体分辨率下每秒大约 100 token 视频,或高媒体分辨率下每秒大约 300 token 视频。
代币计算(智能体模式):token的使用根据内容复杂性和模型的导航策略而变化。视频探索过程中生成的导航推理 token 计为 认为token (
total_thought_tokens),而按需加载的帧、音频和文字记录则被视为工具使用 token(total_tool_use_tokens)。对于长格式内容,智能体处理通常比静态处理少使用最多 88% 的 token,因为模型仅加载回答提示所需的文字记录和/或帧和/或音频(请参阅 token指南).媒体分辨率:Gemini 3 引入了对多模态视觉处理的精细控制
media_resolution参数。的media_resolution参数决定了 每个输入图像或视频帧分配的 token 的最大数量。 更高的分辨率可以提高模型读取精细文本或识别小细节的能力,但会增加 token 的使用率和延迟。的media_resolution和processing参数是独立的:您可以在同一视频输入上设置这两个参数。
有关 token 计算的更多详细信息,请参阅 token 指导。
- 时间戳格式:在提示中提及视频中的特定时刻时,请使用
MM:SS格式(例如,01:15为 1 分钟和 15 秒)。 - 及时安置:如果将文本和单个视频组合在一起,请放置文本提示 之后 视频部分在
input数组。 - 长请求超时:对于需要延长处理时间或复杂的多步骤推理的视频,请使用流式传输或 后台执行。在高需求下经历后端重试的同步、非流式请求可能会超出连接或身份验证 token 有效性窗口,这可能会表现为意外
401 Unauthorized或超时错误。流式传输使连接保持活动状态并显示中间推理和工具调用进度。