I am willing to make a bounding box editor implementation in gradio by creating html and js by myself (where user can move bboxes, scale and draw new ones). But i can't find a way to load an image to canvas. I would appreciate any kind of help
The best thing i've accomplished so far is loading an image to an image html element:
import gradio as gr
import numpy as np
import base64
from PIL import Image
from io import BytesIO
def image_to_base64(img):
buffered = BytesIO()
img.save(buffered, format="PNG")
img_str = base64.b64encode(buffered.getvalue()).decode("utf-8")
return img_str
def image_uploaded(img):
img_pil = Image.fromarray(img)
img_str = image_to_base64(img_pil)
# HTML content with a canvas element
html_content = f"""
<html lang="en">
<body>
<img src="data:image/png;base64,{img_str}" alt="">
<canvas id="canvas"></canvas>
</body>
</html>
"""
return img, gr.HTML(html_content)
with gr.Blocks(js=None) as demo:
uploaded_image = gr.Image(type="numpy", show_label=False, interactive=True, show_download_button=False)
original_image = gr.Image(type="numpy", interactive=False, show_label=False, show_download_button=False)
bounding_box_editor = gr.HTML()
uploaded_image.upload(image_uploaded, inputs=uploaded_image, outputs=[original_image, bounding_box_editor])
demo.launch(share=False)
P.S. If there are other implementations to that - i would be happy to see them (where user can move bboxes, scale and draw new ones)
What I need help with is the JavaScript part to actually draw the uploaded image onto the canvas. Any guidance or examples would be greatly appreciated.
You can see it in many questions or tutorials
You have to get canvas, next get it context .getContext('2d') and next put image with .drawImage(img, x, y, width, height)
Something like this
var image = document.getElementById("image");
var canvas = document.getElementById("canvas");
var ctx = canvas.getContext('2d');
ctx.drawImage(image, 0, 0);
But I found out that gr.HTML() can't run <script> and this makes problem how to run above code.
But you may add script to gr.Block(js=script) with function which later you can use in onclick=".." assigned to button or in onload="..." assigned to image.
Full code:
import gradio as gr
import numpy as np
import base64
from PIL import Image
from io import BytesIO
def image_to_base64(img):
buffered = BytesIO()
img.save(buffered, format="PNG")
img_str = base64.b64encode(buffered.getvalue()).decode("utf-8")
return img_str
def image_uploaded(img):
img_pil = Image.fromarray(img)
img_str = image_to_base64(img_pil)
img_str = f'data:image/png;base64,{img_str}'
#img_str = f'https://upload.wikimedia.org/wikipedia/en/7/7d/Lenna_%28test_image%29.png'
# HTML content with a canvas element
html_content = f"""
<h1>Uploaded: img</h1>
<img id="image" src="{img_str}" alt="" onload="draw_image();">
<h1>Uploaded: canvas</h1>
<canvas id="canvas" width="600"></canvas>
<!-- <button onclick="draw_image();" style="background-color:#eee;padding:25px">Draw Image</button> -->
"""
return img, html_content
script = """
/* this function is executed automatically at start (not when you load canvas) */
/* so draw_image() has to be inside (not in place of `async ()`) */
async () => {
globalThis.draw_image = () => {
var image = document.getElementById("image");
var canvas = document.getElementById("canvas");
var ctx = canvas.getContext('2d');
ctx.drawImage(image, 0, 0);
}
}
"""
with gr.Blocks(js=script) as demo:
uploaded_image = gr.Image(type="numpy", show_label=False, interactive=True, show_download_button=False)
original_image = gr.Image(type="numpy", interactive=False, show_label=False, show_download_button=False)
bounding_box_editor = gr.HTML()
uploaded_image.upload(image_uploaded, inputs=uploaded_image, outputs=[original_image, bounding_box_editor])
demo.launch(share=False)
Source of information:
Gradio HTML component with javascript code don't work - 🔒 Gradio - Hugging Face Forums
gradio.HTML CAN NOT load my javascript · Issue #3446 · gradio-app/gradio
Code which add second image (image Lenna from Wikipedia) and you can use buttons to select which image put on canvas.
import gradio as gr
import numpy as np
import base64
from PIL import Image
from io import BytesIO
def image_to_base64(img):
buffered = BytesIO()
img.save(buffered, format="PNG")
img_str = base64.b64encode(buffered.getvalue()).decode("utf-8")
return img_str
def image_uploaded(img):
img_pil = Image.fromarray(img)
img_str = image_to_base64(img_pil)
img_str_1 = f'data:image/png;base64,{img_str}'
img_str_2 = f'https://upload.wikimedia.org/wikipedia/en/7/7d/Lenna_%28test_image%29.png'
# HTML content with a canvas element
html_content = f"""
<h1>Uploaded: img</h1>
<img id="image1" src="{img_str_1}" alt="" width="200" height="200"> <img id="image2" src="{img_str_2}" alt="" width="200" height="200">
<h1>Uploaded: canvas</h1>
<canvas id="canvas" width="200" height="200"></canvas>
<button onclick="draw_image('image1');" style="background-color:#eee;padding:25px">Draw Image 1</button>
<button onclick="draw_image('image2');" style="background-color:#eee;padding:25px">Draw Image 2</button>
"""
return img, html_content
script = """
/* this function is executed automatically at start (not when you load canvas) */
/* so draw_image() has to be inside (not in place of `async ()`) */
async () => {
globalThis.draw_image = (image_id) => {
var image = document.getElementById(image_id);
var canvas = document.getElementById("canvas");
var ctx = canvas.getContext('2d');
ctx.drawImage(image, 0, 0, 200, 200);
}
}
"""
with gr.Blocks(js=script) as demo:
uploaded_image = gr.Image(type="numpy", show_label=False, interactive=True, show_download_button=False)
original_image = gr.Image(type="numpy", interactive=False, show_label=False, show_download_button=False)
bounding_box_editor = gr.HTML()
uploaded_image.upload(image_uploaded, inputs=uploaded_image, outputs=[original_image, bounding_box_editor])
demo.launch(share=False)