up

.env

@@ -1,13 +1,15 @@
 # ------------- IMAGE API CONFIG --------------
 # Supported values:
 # - VIDEOCHAIN
-RENDERING_ENGINE="VIDEOCHAIN"
+# - REPLICATE
+RENDERING_ENGINE="REPLICATE"
 
 VIDEOCHAIN_API_URL="http://localhost:7860"
 VIDEOCHAIN_API_TOKEN=
 
-# Not supported yet
-REPLICATE_TOKEN=
+REPLICATE_API_TOKEN=
+REPLICATE_API_MODEL="stabilityai/sdxl"
+REPLICATE_API_MODEL_VERSION="da77bc59ee60423279fd632efb4795ab731d9e3ca9705ef3341091fb989b7eaf"
 
 # ------------- LLM API CONFIG ----------------
 # Supported values:

README.md

@@ -17,17 +17,19 @@ First, I would like to highlight that everything is open-source (see [here](http
 However the project isn't a monolithic Space that can be duplicated and ran immediately:
 it requires various components to run for the frontend, backend, LLM, SDXL etc.
 
-If you try to duplicate the project, you will see it requires some variables:
+If you try to duplicate the project and open the `.env` you will see it requires some variables:
 
 - `LLM_ENGINE`: can be either "INFERENCE_API" or "INFERENCE_ENDPOINT"
 - `HF_API_TOKEN`: necessary if you decide to use an inference api model or a custom inference endpoint
 - `HF_INFERENCE_ENDPOINT_URL`: necessary if you decide to use a custom inference endpoint
-- `RENDERING_ENGINE`: can only be "VIDEOCHAIN" for now, unless you code your custom solution
+- `RENDERING_ENGINE`: can only be "VIDEOCHAIN" or "REPLICATE" for now, unless you code your custom solution
 - `VIDEOCHAIN_API_URL`: url to the VideoChain API server
 - `VIDEOCHAIN_API_TOKEN`: secret token to access the VideoChain API server
+- `REPLICATE_API_TOKEN`: in case you want to use Replicate.com
+- `REPLICATE_API_MODEL`: optional, defaults to "stabilityai/sdxl"
+- `REPLICATE_API_MODEL_VERSION`: optional, in case you want to change the version
 
 In addition, there are some community sharing variables that you can just ignore.
-
 Those variables are not required to run the AI Comic Factory on your own website or computer
 (they are meant to create a connection with the Hugging Face community,
 and thus only make sense for official Hugging Face apps):
@@ -104,10 +106,23 @@ You will have to [clone](https://huggingface.co/spaces/jbilcke-hf/VideoChain-API
 Unfortunately, I haven't had the time to write the documentation for VideoChain yet.
 (When I do I will update this document to point to the VideoChain's README)
 
-### Option 2: Use another SDXL API
 
-If you fork the project you will be able to modify the code to use the Stable Diffusion technology of your choice (local, open-source, your custom HF Space etc)
+### Option 2: Use Replicate
 
-### Notes
+To use Replicate, create a `.env.local` configuration file:
+
+```bash
+RENDERING_ENGINE="REPLICATE"
+
+REPLICATE_API_TOKEN="Your Replicate token"
+
+REPLICATE_API_MODEL="stabilityai/sdxl"
+
+REPLICATE_API_MODEL_VERSION="da77bc59ee60423279fd632efb4795ab731d9e3ca9705ef3341091fb989b7eaf"
+```
+
+### Option 3: Use another SDXL API
+
+If you fork the project you will be able to modify the code to use the Stable Diffusion technology of your choice (local, open-source, proprietary, your custom HF Space etc).
 
-It is possible that I modify the AI Comic Factory to make it easier in the future (eg. add support for Replicate)
+It would even be something else, such as Dall-E.

package-lock.json

@@ -48,6 +48,7 @@
         "react-circular-progressbar": "^2.1.0",
         "react-dom": "18.2.0",
         "react-virtualized-auto-sizer": "^1.0.20",
+        "replicate": "^0.17.0",
         "sbd": "^1.0.19",
         "sharp": "^0.32.5",
         "styled-components": "^6.0.7",
@@ -7900,6 +7901,17 @@
         "jsesc": "bin/jsesc"
       }
     },
+    "node_modules/replicate": {
+      "version": "0.17.0",
+      "resolved": "https://registry.npmjs.org/replicate/-/replicate-0.17.0.tgz",
+      "integrity": "sha512-m/ZpGpHpYeUeExvgOm5a+dG/48q5ard2C7xJiOES1TDn6+0IlDOmTp75m/dPQU8xjd3g164ZwMMzJjqAZmNaRw==",
+      "engines": {
+        "git": ">=2.11.0",
+        "node": ">=18.0.0",
+        "npm": ">=7.19.0",
+        "yarn": ">=1.7.0"
+      }
+    },
     "node_modules/resolve": {
       "version": "1.22.4",
       "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.4.tgz",

package.json

@@ -49,6 +49,7 @@
     "react-circular-progressbar": "^2.1.0",
     "react-dom": "18.2.0",
     "react-virtualized-auto-sizer": "^1.0.20",
+    "replicate": "^0.17.0",
     "sbd": "^1.0.19",
     "sharp": "^0.32.5",
     "styled-components": "^6.0.7",

src/app/engine/render.ts

@@ -1,9 +1,17 @@
 "use server"
 
+import Replicate, { Prediction } from "replicate"
+
 import { RenderRequest, RenderedScene, RenderingEngine } from "@/types"
+import { generateSeed } from "@/lib/generateSeed"
+import { sleep } from "@/lib/sleep"
 
 const renderingEngine = `${process.env.RENDERING_ENGINE || ""}` as RenderingEngine
 
+const replicateToken = `${process.env.REPLICATE_API_TOKEN || ""}`
+const replicateModel = `${process.env.REPLICATE_API_MODEL || ""}`
+const replicateModelVersion = `${process.env.REPLICATE_API_MODEL_VERSION || ""}`
+
 // note: there is no / at the end in the variable
 // so we have to add it ourselves if needed
 const apiUrl = process.env.VIDEOCHAIN_API_URL
@@ -37,54 +45,89 @@ export async function newRender({
 
 
   try {
-    // console.log(`calling POST ${apiUrl}/render with prompt: ${prompt}`)
+    if (renderingEngine === "REPLICATE") {
+      if (!replicateToken) {
+        throw new Error(`you need to configure your REPLICATE_API_TOKEN in order to use the REPLICATE rendering engine`)
+      }
+      if (!replicateModel) {
+        throw new Error(`you need to configure your REPLICATE_API_MODEL in order to use the REPLICATE rendering engine`)
+      }
+      if (!replicateModelVersion) {
+        throw new Error(`you need to configure your REPLICATE_API_MODEL_VERSION in order to use the REPLICATE rendering engine`)
+      }
+      const replicate = new Replicate({ auth: replicateToken })
 
-    const res = await fetch(`${apiUrl}/render`, {
-      method: "POST",
-      headers: {
-        Accept: "application/json",
-        "Content-Type": "application/json",
-        Authorization: `Bearer ${process.env.VIDEOCHAIN_API_TOKEN}`,
-      },
-      body: JSON.stringify({
-        prompt,
-        // negativePrompt, unused for now
-        nbFrames: 1,
-        nbSteps: 25, // 20 = fast, 30 = better, 50 = best
-        actionnables: [], // ["text block"],
-        segmentation: "disabled", // "firstframe", // one day we will remove this param, to make it automatic
-        width,
-        height,
-
-        // no need to upscale right now as we generate tiny panels
-        // maybe later we can provide an "export" button to PDF
-        // unfortunately there are too many requests for upscaling,
-        // the server is always down
-        upscalingFactor: 1, // 2,
-
-        // analyzing doesn't work yet, it seems..
-        analyze: false, // analyze: true,
-
-        cache: "ignore"
-      } as Partial<RenderRequest>),
-      cache: 'no-store',
-    // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache)
-    // next: { revalidate: 1 }
-    })
+      // console.log("Calling replicate..")
+      const seed = generateSeed()
+      const prediction = await replicate.predictions.create({
+        version: replicateModelVersion,
+        input: { prompt, seed }
+      })
+      
+      // console.log("prediction:", prediction)
 
-    // console.log("res:", res)
-    // The return value is *not* serialized
-    // You can return Date, Map, Set, etc.
-    
-    // Recommendation: handle errors
-    if (res.status !== 200) {
-      // This will activate the closest `error.js` Error Boundary
-      throw new Error('Failed to fetch data')
-    }
-    
-    const response = (await res.json()) as RenderedScene
+      // no need to reply straight away: good things take time
+      // also our friends at Replicate won't like it if we spam them with requests
+      await sleep(4000)
 
-    return response
+      return {
+        renderId: prediction.id,
+        status: "pending",
+        assetUrl: "", 
+        alt: prompt,
+        error: prediction.error,
+        maskUrl: "",
+        segments: []
+      } as RenderedScene
+    } else {
+      // console.log(`calling POST ${apiUrl}/render with prompt: ${prompt}`)
+      const res = await fetch(`${apiUrl}/render`, {
+        method: "POST",
+        headers: {
+          Accept: "application/json",
+          "Content-Type": "application/json",
+          Authorization: `Bearer ${process.env.VIDEOCHAIN_API_TOKEN}`,
+        },
+        body: JSON.stringify({
+          prompt,
+          // negativePrompt, unused for now
+          nbFrames: 1,
+          nbSteps: 25, // 20 = fast, 30 = better, 50 = best
+          actionnables: [], // ["text block"],
+          segmentation: "disabled", // "firstframe", // one day we will remove this param, to make it automatic
+          width,
+          height,
+
+          // no need to upscale right now as we generate tiny panels
+          // maybe later we can provide an "export" button to PDF
+          // unfortunately there are too many requests for upscaling,
+          // the server is always down
+          upscalingFactor: 1, // 2,
+
+          // analyzing doesn't work yet, it seems..
+          analyze: false, // analyze: true,
+
+          cache: "ignore"
+        } as Partial<RenderRequest>),
+        cache: 'no-store',
+      // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache)
+      // next: { revalidate: 1 }
+      })
+
+
+      // console.log("res:", res)
+      // The return value is *not* serialized
+      // You can return Date, Map, Set, etc.
+      
+      // Recommendation: handle errors
+      if (res.status !== 200) {
+        // This will activate the closest `error.js` Error Boundary
+        throw new Error('Failed to fetch data')
+      }
+      
+      const response = (await res.json()) as RenderedScene
+      return response
+    }
   } catch (err) {
     console.error(err)
     return defaulResult
@@ -108,32 +151,83 @@ export async function getRender(renderId: string) {
   }
 
   try {
-    // console.log(`calling GET ${apiUrl}/render with renderId: ${renderId}`)
-    const res = await fetch(`${apiUrl}/render/${renderId}`, {
-      method: "GET",
-      headers: {
-        Accept: "application/json",
-        "Content-Type": "application/json",
-        Authorization: `Bearer ${process.env.VIDEOCHAIN_API_TOKEN}`,
-      },
-      cache: 'no-store',
-    // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache)
-    // next: { revalidate: 1 }
-    })
+    if (renderingEngine === "REPLICATE") {
+      if (!replicateToken) {
+        throw new Error(`you need to configure your REPLICATE_API_TOKEN in order to use the REPLICATE rendering engine`)
+      }
+      if (!replicateModel) {
+        throw new Error(`you need to configure your REPLICATE_API_MODEL in order to use the REPLICATE rendering engine`)
+      }
 
-    // console.log("res:", res)
-    // The return value is *not* serialized
-    // You can return Date, Map, Set, etc.
+      // const replicate = new Replicate({ auth: replicateToken })
+
+      // console.log("Calling replicate..")
+      // const prediction = await replicate.predictions.get(renderId)
+      // console.log("Prediction:", prediction)
+
+       // console.log(`calling GET https://api.replicate.com/v1/predictions/${renderId}`)
+       const res = await fetch(`https://api.replicate.com/v1/predictions/${renderId}`, {
+        method: "GET",
+        headers: {
+          // Accept: "application/json",
+          // "Content-Type": "application/json",
+          Authorization: `Token ${replicateToken}`,
+        },
+        cache: 'no-store',
+      // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache)
+      // next: { revalidate: 1 }
+      })
     
-    // Recommendation: handle errors
-    if (res.status !== 200) {
-      // This will activate the closest `error.js` Error Boundary
-      throw new Error('Failed to fetch data')
+      // console.log("res:", res)
+      // The return value is *not* serialized
+      // You can return Date, Map, Set, etc.
+      
+      // Recommendation: handle errors
+      if (res.status !== 200) {
+        // This will activate the closest `error.js` Error Boundary
+        throw new Error('Failed to fetch data')
+      }
+      
+      const response = (await res.json()) as any
+      // console.log("response:", response)
+
+      return  {
+        renderId,
+        status: response?.error ? "error" : response?.status === "succeeded" ?  "completed" : "pending",
+        assetUrl: `${response?.output || ""}`,
+        alt: `${response?.input?.prompt || ""}`,
+        error: `${response?.error || ""}`,
+        maskUrl: "",
+        segments: []
+      } as RenderedScene
+    } else {
+      // console.log(`calling GET ${apiUrl}/render with renderId: ${renderId}`)
+      const res = await fetch(`${apiUrl}/render/${renderId}`, {
+        method: "GET",
+        headers: {
+          Accept: "application/json",
+          "Content-Type": "application/json",
+          Authorization: `Bearer ${process.env.VIDEOCHAIN_API_TOKEN}`,
+        },
+        cache: 'no-store',
+      // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache)
+      // next: { revalidate: 1 }
+      })
+
+      // console.log("res:", res)
+      // The return value is *not* serialized
+      // You can return Date, Map, Set, etc.
+      
+      // Recommendation: handle errors
+      if (res.status !== 200) {
+        // This will activate the closest `error.js` Error Boundary
+        throw new Error('Failed to fetch data')
+      }
+      
+      const response = (await res.json()) as RenderedScene
+      // console.log("response:", response)
+      return response
     }
-    
-    const response = (await res.json()) as RenderedScene
-    // console.log("response:", response)
-    return response
   } catch (err) {
     console.error(err)
     defaulResult.status = "error"