ControlNet Video

Generate video with ControlNet

curl --request POST \
  --url https://modelslab.com/api/v6/video/controlnet \
  --header 'Content-Type: application/json' \
  --data '
{
  "key": "<string>",
  "model_id": "h3-minimax-controlnet",
  "prompt": "<string>",
  "controlnet_type": "<string>",
  "init_video": "<string>",
  "convert_to_controlnet_input": false,
  "duration": 5,
  "width": 1280,
  "height": 704,
  "num_inference_steps": 40,
  "seed": 123,
  "instant_response": true,
  "temp": false,
  "webhook": "<string>",
  "track_id": "<string>"
}
'

import requests

url = "https://modelslab.com/api/v6/video/controlnet"

payload = {
    "key": "<string>",
    "model_id": "h3-minimax-controlnet",
    "prompt": "<string>",
    "controlnet_type": "<string>",
    "init_video": "<string>",
    "convert_to_controlnet_input": False,
    "duration": 5,
    "width": 1280,
    "height": 704,
    "num_inference_steps": 40,
    "seed": 123,
    "instant_response": True,
    "temp": False,
    "webhook": "<string>",
    "track_id": "<string>"
}
headers = {"Content-Type": "application/json"}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {'Content-Type': 'application/json'},
  body: JSON.stringify({
    key: '<string>',
    model_id: 'h3-minimax-controlnet',
    prompt: '<string>',
    controlnet_type: '<string>',
    init_video: '<string>',
    convert_to_controlnet_input: false,
    duration: 5,
    width: 1280,
    height: 704,
    num_inference_steps: 40,
    seed: 123,
    instant_response: true,
    temp: false,
    webhook: '<string>',
    track_id: '<string>'
  })
};

fetch('https://modelslab.com/api/v6/video/controlnet', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://modelslab.com/api/v6/video/controlnet",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'key' => '<string>',
    'model_id' => 'h3-minimax-controlnet',
    'prompt' => '<string>',
    'controlnet_type' => '<string>',
    'init_video' => '<string>',
    'convert_to_controlnet_input' => false,
    'duration' => 5,
    'width' => 1280,
    'height' => 704,
    'num_inference_steps' => 40,
    'seed' => 123,
    'instant_response' => true,
    'temp' => false,
    'webhook' => '<string>',
    'track_id' => '<string>'
  ]),
  CURLOPT_HTTPHEADER => [
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://modelslab.com/api/v6/video/controlnet"

	payload := strings.NewReader("{\n  \"key\": \"<string>\",\n  \"model_id\": \"h3-minimax-controlnet\",\n  \"prompt\": \"<string>\",\n  \"controlnet_type\": \"<string>\",\n  \"init_video\": \"<string>\",\n  \"convert_to_controlnet_input\": false,\n  \"duration\": 5,\n  \"width\": 1280,\n  \"height\": 704,\n  \"num_inference_steps\": 40,\n  \"seed\": 123,\n  \"instant_response\": true,\n  \"temp\": false,\n  \"webhook\": \"<string>\",\n  \"track_id\": \"<string>\"\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://modelslab.com/api/v6/video/controlnet")
  .header("Content-Type", "application/json")
  .body("{\n  \"key\": \"<string>\",\n  \"model_id\": \"h3-minimax-controlnet\",\n  \"prompt\": \"<string>\",\n  \"controlnet_type\": \"<string>\",\n  \"init_video\": \"<string>\",\n  \"convert_to_controlnet_input\": false,\n  \"duration\": 5,\n  \"width\": 1280,\n  \"height\": 704,\n  \"num_inference_steps\": 40,\n  \"seed\": 123,\n  \"instant_response\": true,\n  \"temp\": false,\n  \"webhook\": \"<string>\",\n  \"track_id\": \"<string>\"\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://modelslab.com/api/v6/video/controlnet")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Content-Type"] = 'application/json'
request.body = "{\n  \"key\": \"<string>\",\n  \"model_id\": \"h3-minimax-controlnet\",\n  \"prompt\": \"<string>\",\n  \"controlnet_type\": \"<string>\",\n  \"init_video\": \"<string>\",\n  \"convert_to_controlnet_input\": false,\n  \"duration\": 5,\n  \"width\": 1280,\n  \"height\": 704,\n  \"num_inference_steps\": 40,\n  \"seed\": 123,\n  \"instant_response\": true,\n  \"temp\": false,\n  \"webhook\": \"<string>\",\n  \"track_id\": \"<string>\"\n}"

response = http.request(request)
puts response.read_body

{
  "status": "success",
  "generationTime": 123,
  "id": 123,
  "output": [
    "<string>"
  ],
  "proxy_links": [
    "<string>"
  ],
  "future_links": [
    "<string>"
  ],
  "meta": {},
  "eta": 123,
  "message": "<string>",
  "tip": "<string>",
  "fetch_result": "<string>"
}

{
  "status": "error",
  "message": "<string>"
}

{
  "status": "error",
  "message": "<string>"
}

{
  "status": "error",
  "message": "<string>"
}

POST

video

controlnet

Generate video with ControlNet

curl --request POST \
  --url https://modelslab.com/api/v6/video/controlnet \
  --header 'Content-Type: application/json' \
  --data '
{
  "key": "<string>",
  "model_id": "h3-minimax-controlnet",
  "prompt": "<string>",
  "controlnet_type": "<string>",
  "init_video": "<string>",
  "convert_to_controlnet_input": false,
  "duration": 5,
  "width": 1280,
  "height": 704,
  "num_inference_steps": 40,
  "seed": 123,
  "instant_response": true,
  "temp": false,
  "webhook": "<string>",
  "track_id": "<string>"
}
'

import requests

url = "https://modelslab.com/api/v6/video/controlnet"

payload = {
    "key": "<string>",
    "model_id": "h3-minimax-controlnet",
    "prompt": "<string>",
    "controlnet_type": "<string>",
    "init_video": "<string>",
    "convert_to_controlnet_input": False,
    "duration": 5,
    "width": 1280,
    "height": 704,
    "num_inference_steps": 40,
    "seed": 123,
    "instant_response": True,
    "temp": False,
    "webhook": "<string>",
    "track_id": "<string>"
}
headers = {"Content-Type": "application/json"}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {'Content-Type': 'application/json'},
  body: JSON.stringify({
    key: '<string>',
    model_id: 'h3-minimax-controlnet',
    prompt: '<string>',
    controlnet_type: '<string>',
    init_video: '<string>',
    convert_to_controlnet_input: false,
    duration: 5,
    width: 1280,
    height: 704,
    num_inference_steps: 40,
    seed: 123,
    instant_response: true,
    temp: false,
    webhook: '<string>',
    track_id: '<string>'
  })
};

fetch('https://modelslab.com/api/v6/video/controlnet', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://modelslab.com/api/v6/video/controlnet",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'key' => '<string>',
    'model_id' => 'h3-minimax-controlnet',
    'prompt' => '<string>',
    'controlnet_type' => '<string>',
    'init_video' => '<string>',
    'convert_to_controlnet_input' => false,
    'duration' => 5,
    'width' => 1280,
    'height' => 704,
    'num_inference_steps' => 40,
    'seed' => 123,
    'instant_response' => true,
    'temp' => false,
    'webhook' => '<string>',
    'track_id' => '<string>'
  ]),
  CURLOPT_HTTPHEADER => [
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://modelslab.com/api/v6/video/controlnet"

	payload := strings.NewReader("{\n  \"key\": \"<string>\",\n  \"model_id\": \"h3-minimax-controlnet\",\n  \"prompt\": \"<string>\",\n  \"controlnet_type\": \"<string>\",\n  \"init_video\": \"<string>\",\n  \"convert_to_controlnet_input\": false,\n  \"duration\": 5,\n  \"width\": 1280,\n  \"height\": 704,\n  \"num_inference_steps\": 40,\n  \"seed\": 123,\n  \"instant_response\": true,\n  \"temp\": false,\n  \"webhook\": \"<string>\",\n  \"track_id\": \"<string>\"\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://modelslab.com/api/v6/video/controlnet")
  .header("Content-Type", "application/json")
  .body("{\n  \"key\": \"<string>\",\n  \"model_id\": \"h3-minimax-controlnet\",\n  \"prompt\": \"<string>\",\n  \"controlnet_type\": \"<string>\",\n  \"init_video\": \"<string>\",\n  \"convert_to_controlnet_input\": false,\n  \"duration\": 5,\n  \"width\": 1280,\n  \"height\": 704,\n  \"num_inference_steps\": 40,\n  \"seed\": 123,\n  \"instant_response\": true,\n  \"temp\": false,\n  \"webhook\": \"<string>\",\n  \"track_id\": \"<string>\"\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://modelslab.com/api/v6/video/controlnet")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Content-Type"] = 'application/json'
request.body = "{\n  \"key\": \"<string>\",\n  \"model_id\": \"h3-minimax-controlnet\",\n  \"prompt\": \"<string>\",\n  \"controlnet_type\": \"<string>\",\n  \"init_video\": \"<string>\",\n  \"convert_to_controlnet_input\": false,\n  \"duration\": 5,\n  \"width\": 1280,\n  \"height\": 704,\n  \"num_inference_steps\": 40,\n  \"seed\": 123,\n  \"instant_response\": true,\n  \"temp\": false,\n  \"webhook\": \"<string>\",\n  \"track_id\": \"<string>\"\n}"

response = http.request(request)
puts response.read_body

{
  "status": "success",
  "generationTime": 123,
  "id": 123,
  "output": [
    "<string>"
  ],
  "proxy_links": [
    "<string>"
  ],
  "future_links": [
    "<string>"
  ],
  "meta": {},
  "eta": 123,
  "message": "<string>",
  "tip": "<string>",
  "fetch_result": "<string>"
}

{
  "status": "error",
  "message": "<string>"
}

{
  "status": "error",
  "message": "<string>"
}

{
  "status": "error",
  "message": "<string>"
}

Generate a video from a prompt while keeping the structure of a control video: edges, depth, pose and similar signals. Send the ControlNet video as init_video, and set controlnet_type to the kind of signal it carries.

model_id is required and must be h3-minimax-controlnet. controlnet_type must be one of the ControlNet models listed on ModelsLab, such as canny, depth, openpose, lineart or scribble.

init_video should be a ControlNet video, meaning a video that already shows the control signal, such as a canny edge map, a depth map or an OpenPose skeleton. It is not the source footage. Its signal must match controlnet_type; for example, send a canny edge video with controlnet_type: canny.

The maximum resolution is 1440 pixels: width and height must each be between 320 and 1440.

Request

Make a POST request to below endpoint and pass the required parameters in the request body.

curl

--request POST 'https://modelslab.com/api/v6/video/controlnet' \

Body

json

{
    "key": "your_api_key",
    "model_id": "h3-minimax-controlnet",
    "prompt": "A cinematic astronaut walking gently on the moon, preserve the helmet silhouette and arm positions from the control video",
    "controlnet_type": "canny",
    "init_video": "https://huggingface.co/alibaba-pai/MiniMax-H3-Fun-Controlnet-Union-2.0/resolve/7d2c95de2e351ed6a0b360af45f62daa04c26f88/asset/canny.mp4",
    "convert_to_controlnet_input": false,
    "duration": 5,
    "width": 1280,
    "height": 704,
    "num_inference_steps": 40,
    "seed": 43,
    "webhook": null,
    "track_id": null
}

Keep convert_to_controlnet_input set to false when init_video is a ControlNet video, which is the expected input. Setting it to true makes the model extract the controlnet_type signal from an ordinary video first.

Response

The request is queued and answers right away with processing. The future_links field holds the URLs where the video will appear. Poll Fetch Video with the returned id, or pass a webhook to be notified when it’s ready.

json

{
    "status": "processing",
    "tip": "Get 20x faster image generation using enterprise plan. Click here : https://modelslab.com/enterprise",
    "eta": 80,
    "message": "Try to fetch request after seconds estimated",
    "fetch_result": "https://modelslab.com/api/v6/video/fetch/123456789",
    "id": 123456789,
    "output": [],
    "future_links": [
        "https://pub-3626123a908346a7a8be8d9295f44e26.r2.dev/generations/7f3c2a1e-1b2c-4d5e-8f90-1234567890ab-0.mp4"
    ],
    "meta": {
        "prompt": "A cinematic astronaut walking gently on the moon, preserve the helmet silhouette and arm positions from the control video",
        "controlnet_type": "canny",
        "control_video": "https://huggingface.co/alibaba-pai/MiniMax-H3-Fun-Controlnet-Union-2.0/resolve/7d2c95de2e351ed6a0b360af45f62daa04c26f88/asset/canny.mp4",
        "convert_to_controlnet_input": false,
        "duration": 5,
        "width": 1280,
        "height": 704,
        "num_inference_steps": 40,
        "num_samples": 1,
        "seed": 43
    }
}

Body

application/json

key

string

required

Your API Key used for request authorization

model_id

enum<string>

required

The model to use. This endpoint only serves h3-minimax-controlnet.

Available options:

h3-minimax-controlnet

prompt

string

required

Text prompt describing the video to generate.

controlnet_type

string

required

The ControlNet model the control video represents, e.g. canny, depth, openpose, lineart, scribble. Must be one of the ControlNet models listed on ModelsLab.

init_video

string<uri>

required

URL of the ControlNet video: a video that already shows the control signal (e.g. a canny edge map, depth map or OpenPose skeleton) matching controlnet_type, not the source footage. The output follows its structure.

convert_to_controlnet_input

boolean

default:false

Set to true to send an ordinary video and let the model extract the controlnet_type signal from it. Keep false when init_video is a ControlNet video, which is the expected input.

duration

integer

default:5

Output length in seconds. Values outside 5–15 are clamped.

Required range: 5 <= x <= 15

width

integer

default:1280

Output width in pixels, from 320 to 1440.

Required range: 320 <= x <= 1440

height

integer

default:704

Output height in pixels, from 320 to 1440.

Required range: 320 <= x <= 1440

num_inference_steps

integer

default:40

Number of denoising steps. Higher values may improve quality but increase processing time.

seed

integer | null

Seed for reproducible results. Random if omitted.

instant_response

boolean

default:true

Return future links immediately instead of waiting for the result.

temp

boolean

default:false

Store the output in temporary storage.

webhook

string<uri>

URL to receive a POST call once generation completes.

track_id

string

ID returned in the webhook payload to identify this request.

Response

ControlNet video generation response

status

enum<string>

Status of the video generation

Available options:

success,

processing,

error

generationTime

number

Time taken to generate the video in seconds

integer

Unique identifier for the video generation

output

string<uri>[]

Array of generated video URLs

proxy_links

string<uri>[]

Array of proxy video URLs

future_links

string<uri>[]

Array of future video URLs for queued requests

Using the APIs

Our AI APIs

Request

Body

Response

Body

Response

​Request

​Body

​Response

Body

Response

Request

Body

Response