javascriptnext.jsvercelgoogle-geminigoogle-generativeai

Passing URLs as video paths in @google/generative AI and Next.js

I'm looking for a solution to upload videos from a URL when using @google/generative-ai in Next.js. I have figured out how to use videos that are stored on my machine, but not on external sources.

This is my current function to upload videos to @google/generative-ai:

"use server"

const { GoogleGenerativeAI } = require("@google/generative-ai");
import { GoogleAIFileManager, FileState } from "@google/generative-ai/server";
import { redirect } from "next/navigation";
import fetchVideoById from "./fetchVideoById";

// Initialize GoogleAIFileManager with your API_KEY.
const fileManager = new GoogleAIFileManager(process.env.API_KEY);


// Access your API key as an environment variable (see "Set up your API key" above)
const genAI = new GoogleGenerativeAI(process.env.API_KEY);

// Choose a Gemini model.
const model = genAI.getGenerativeModel({
  model: "gemini-1.5-pro",
});

export async function generateSummary(formData) {
  const rows = await fetchVideoById(formData.get("id"))
  const url = rows["url"]
  // by the way, this returns the URL for the video that is stored in my database, so assume that      the URL is valid.

  console.log("Uploading file...")
    const fileManager = new GoogleAIFileManager(process.env.API_KEY);

    // Upload the file and specify a display name.
    const uploadResponse = await fileManager.uploadFile(url, {
      mimeType: "video/mp4",
      displayName: rows["title"],
    });

    // View the response.
    console.log(`Uploaded file ${uploadResponse.file.displayName} as: ${uploadResponse.file.uri}`);
    const name = uploadResponse.file.name;

    // Poll getFile() on a set interval (10 seconds here) to check file state.
    let file = await fileManager.getFile(name);
    while (file.state === FileState.PROCESSING) {
      process.stdout.write(".")
      // Fetch the file from the API again
      file = await fileManager.getFile(name)
    }

    if (file.state === FileState.FAILED) {
      throw new Error("Video processing failed.");
    }

    // When file.state is ACTIVE, the file is ready to be used for inference.
    console.log(`File ${file.displayName} is ready for inference as ${file.uri}`);

    const result = await model.generateContent([
      {
        fileData: {
          mimeType: uploadResponse.file.mimeType,
          fileUri: uploadResponse.file.uri
        }
      },
      { text: "Summarize this video." },
    ]);

  // Handle the response of generated text
  console.log(result.response.text())

  return result.response.text()

  console.log("Deleting file...")

  await fileManager.deleteFile(file.name);

  console.log("Deleted file.")
  
}

This is the error that I get:

Error: ENOENT: no such file or directory, open 'C:\Users\n_mac\Desktop\Coding\summa\front-end\https:\m9x5emw6q3oaze3r.public.blob.vercel-storage.com\monkeyman64\6999DBC5-2D93-4220-BC43-3C16C9A5D9C6-IZzFC1THZXPSgeAK1NPo3uCVxA091l.mp4'

As you can see, it's searching on my machine for the file, when the files are stored in Vercel Blob.

Any help is appreciated.

Solution

  • When I saw the script of uploadFile, it seemed that the file was uploaded as multipart. Ref And, it seems that the filePath is required to be the file path of the local PC. Unfortunately, it seems that in the current stage, the URL cannot be directly used.

    So, in this case, how about downloading the data from the URL and uploading it to Gemini without creating a file? When this is reflected in the script, how about the following modification?

    From:

    const fileManager = new GoogleAIFileManager(process.env.API_KEY);

// Upload the file and specify a display name.
const uploadResponse = await fileManager.uploadFile(url, {
  mimeType: "video/mp4",
  displayName: rows["title"],
});

// View the response.
console.log(`Uploaded file ${uploadResponse.file.displayName} as: ${uploadResponse.file.uri}`);
const name = uploadResponse.file.name;

    To:

    const fileManager = new GoogleAIFileManager(process.env.API_KEY);


// Download data.
const res = await fetch(url);
const buffer = await res.arrayBuffer();

// Upload the downloaded data.
const formData = new FormData();
const metadata = { file: { mimeType: "video/mp4", displayName: rows["title"] } };
formData.append("metadata", new Blob([JSON.stringify(metadata)], { contentType: "application/json" }));
formData.append("file", new Blob([buffer], { type: "video/mp4" }));
const res2 = await fetch(
  `https://generativelanguage.googleapis.com/upload/v1beta/files?uploadType=multipart&key=${fileManager.apiKey}`,
  { method: "post", body: formData }
);
const uploadResponse = await res2.json();


// View the response.
console.log(`Uploaded file ${uploadResponse.file.displayName} as: ${uploadResponse.file.uri}`);
const name = uploadResponse.file.name;

    Note:

    Reference: