Develop Reference

Loading video frames into javascript in google colab

I wish to present a video, frame by frame with the possibility of flipping between frames in google colab. I also wish to mark some rectangles and get the x,y,w,h of each rectangle.
As far as I could understand, the only way to click the output and gather data in google Collab is via a javascript call, which I am new to. I have split my video into a list called frames and written the following python\javascript code for my task:
def get_fxy_val(base64imgs):
js = Javascript('''
async function showImage(base64) {
var canvas = document.createElement('canvas');
var rect = {};
var drag = false;
var finish = false;
var clicks = [];
var img = document.createElement('img');
var frame = 0
var ctx = canvas.getContext('2d');
document.body.appendChild(canvas);
img = new Image();
img.onload = function () {
canvas.height = img.height;
canvas.width = img.width;
ctx.drawImage(img, 0, 0);
};
img.src = 'data:image/jpeg;base64,'+base64[frame];
var prev = document.createElement('button');
prev.innerHTML = '<';
prev.onclick = function() {
frame = frame -1;
if(frame<0){
frame = 0;
}
else{
// console.log(frame)
img.src = 'data:image/jpeg;base64,'+base64[frame];
};
};
document.body.appendChild(prev);
var next = document.createElement('button');
next.innerHTML = '>';
next.onclick = function() {
frame = frame+1;
if (frame>base64.length-1) {
// console.log(frame)
frame = base64.length-1
}
else{
img.src = 'data:image/jpeg;base64,'+base64[frame];
}
};
document.body.appendChild(next);
canvas.addEventListener('mousedown', mouseDown, false);
canvas.addEventListener('mouseup', mouseUp, false);
canvas.addEventListener('mousemove', mouseMove, false);
function mouseDown(e) {
rect.startX = e.pageX - this.offsetLeft;
rect.startY = e.pageY - this.offsetTop;
drag = true;
}
function mouseUp() {
drag = false;
ctx.clearRect(0, 0, canvas.width, canvas.height);
ctx.drawImage(img, 0, 0);
if (rect.w<0){
rect.startX = rect.startX + rect.w
rect.w = rect.w*(-1)
}
if (rect.h<0){
rect.startY = rect.startY + rect.h
rect.h = rect.h*(-1)
}
ctx.strokeRect(rect.startX, rect.startY, rect.w, rect.h);
}
function mouseMove(e) {
if (drag) {
rect.w = (e.pageX - this.offsetLeft) - rect.startX;
rect.h = (e.pageY - this.offsetTop) - rect.startY;
ctx.strokeStyle = 'red';
ctx.lineWidth = 2;
ctx.strokeRect(rect.startX, rect.startY, rect.w, rect.h);
}
}
var saver = document.createElement('button');
saver.innerHTML = 'save rectangle';
saver.onclick = function() {
clicks.push([frame, rect.startX, rect.startY, rect.w, rect.h]);
ctx.clearRect(0, 0, canvas.width, canvas.height);
ctx.drawImage(img, 0, 0);
};
document.body.appendChild(saver);
var finisher = document.createElement('button');
finisher.innerHTML = 'Done!';
finisher.onclick = function() {
};
document.body.appendChild(finisher);
await new Promise((resolve) => finisher.onclick = resolve);
document.body.removeChild(finisher)
document.body.removeChild(saver)
document.body.removeChild(next)
document.body.removeChild(prev)
document.body.removeChild(canvas)
return clicks
}
''')
display(js)
imgs = ["'"+b64img+"'" for b64img in base64imgs]
imgs = '[' + ','.join(imgs) +']'
data = eval_js(f"showImage({imgs})")
return data
I call it using:
base64imgs = [base64.b64encode(cv2.imencode('.jpg', img)[1]).decode() for img in frames]
data = get_fxy_val(base64imgs)
Everything works well for a small number of images, however for the full video Collab crushes and I cannot manage to find any logs to verify what is going on.
Two thoughts I had, which I do not know how to implement:
Reduce the images' memory size somehow. I cannot harm the aspect ratio so downsampling is not a possibility.
Load images directly from memory inside the javascript code. I have no idea how to implement this or if it is even possible.
Can I implement any of these? Is there another approach?
B.T.W, my video has images with size h,w,c = (1080, 1920, 3).

How to upload image to canvas and draw squares on it and get the squares' coordinates

I'm trying to upload an image to canvas and draw squares on it and get the squares' coordinates to send it to the backend for keys identification.
But something not right, I can't draw squares on my images.
This is my javascript which I use:
var canvas = document.getElementById('canvas');
var ctx = canvas.getContext('2d');
var rect = {};
var drag = false;
var imageObj = null;
function init() {
imageObj = new Image();
imageObj.onload = function () { ctx.drawImage(imageObj, 0, 0); };
imageObj.src =
canvas.addEventListener('mousedown', mouseDown, false);
canvas.addEventListener('mouseup', mouseUp, false);
canvas.addEventListener('mousemove', mouseMove, false);
}
function mouseDown(e) {
rect.startX = e.pageX - this.offsetLeft;
rect.startY = e.pageY - this.offsetTop;
drag = true;
}
function mouseUp() { drag = false; }
function mouseMove(e) {
if (drag) {
ctx.clearRect(0, 0, 500, 500);
ctx.drawImage(imageObj, 0, 0);
rect.w = (e.pageX - this.offsetLeft) - rect.startX;
rect.h = (e.pageY - this.offsetTop) - rect.startY;
ctx.strokeStyle = 'red';
ctx.strokeRect(rect.startX, rect.startY, rect.w, rect.h);
}
}
//
init();
</script>
<script type="text/javascript">
document.getElementById('imageLoader').onchange = function(e) {
var img = new Image();
img.onload = draw;
img.onerror = failed;
img.src = URL.createObjectURL(this.files[0]);
};
function draw() {
var canvas = document.getElementById('canvas');
canvas.width = this.width;
canvas.height = this.height;
var ctx = canvas.getContext('2d');
ctx.drawImage(this, 0,0);
}
function failed() {
console.error("The provided file couldn't be loaded as an Image media");
}
and HTML:
<canvas id="canvas" ></canvas>
<input type="file" id="imageLoader" name="imageLoader"/>

You just need to assign the image to imageObj.
function draw() {
canvas.width = this.width;
canvas.height = this.height;
ctx.drawImage(this, 0,0);
imageObj = this; //here
}
JSFiddle here

How to preview the correct image which select from canvas?

var canvas = document.getElementById('myCanvas');
var context = canvas.getContext('2d');
var rect = {};
var drag = false;
var storeRects = [];
var isMoving = false;
make_base();
init();
function make_base() {
base_image = new Image();
base_image.src = 'https://www.w3schools.com/css/img_fjords.jpg';
base_image.onload = function() {
context.drawImage(base_image, 0, 0, 800, 500);
}
}
function getMousePos(canvas, evt) {
var rect = canvas.getBoundingClientRect();
return {
x: evt.clientX - rect.left,
y: evt.clientY - rect.top
};
}
function init() {
canvas.addEventListener('mousedown', mouseDown, false);
canvas.addEventListener('mouseup', mouseUp, false);
canvas.addEventListener('mousemove', mouseMove, false);
}
function mouseDown(e) {
rect.startX = e.pageX - this.offsetLeft;
rect.startY = e.pageY - this.offsetTop;
drag = true;
}
function mouseMove(e) {
if(drag) {
isMoving = true;
rect.w = (e.pageX - this.offsetLeft) - rect.startX;
rect.h = (e.pageY - this.offsetTop) - rect.startY ;
draw();
}
}
function draw() {
context.clearRect(0, 0, canvas.width, canvas.height);
context.drawImage(base_image, 0 ,0, 800, 500);
storeRects.forEach(function(rect) {
context.beginPath();
context.rect(rect.x, rect.y,rect.w, rect.h);
context.stroke();
});
context.lineWidth="1";
context.strokeStyle = "red";
context.beginPath();
context.rect(rect.startX, rect.startY, rect.w, rect.h);
context.stroke();
}
function mouseUp() {
drag = false;
if(isMoving === true) {
storeRects.push({
x: rect.startX,
y: rect.startY,
w: rect.w,
h: rect.h,
});
var c = document.getElementById("myCanvasPreview");
var ctx = c.getContext("2d");
var pre_base_image = new Image();
pre_base_image.src = 'https://www.w3schools.com/css/img_fjords.jpg';
ctx.drawImage(pre_base_image,
(Math.min(storeRects[0].x, storeRects[0].x + Math.abs(storeRects[0].w)) / canvas.width) * pre_base_image.width,
(Math.min(storeRects[0].y, storeRects[0].y + Math.abs(storeRects[0].h)) / canvas.height) * pre_base_image.height,
(Math.abs(storeRects[0].w) / canvas.width) * pre_base_image.width ,
(Math.abs(storeRects[0].h) / canvas.height) * pre_base_image.height,
0, 0,
Math.abs(storeRects[0].w),
Math.abs(storeRects[0].h)
);
isMoving = false;
}
}
<canvas id="myCanvas" width="800" height="500" style="border:1px solid #000000;">
</canvas>
<canvas id="myCanvasPreview" width="800" height="500" style="border:1px solid #d3d3d3;">
</canvas>
When I select the person in the image, I would like to preview it on another canvas.
Now, I can preview but seems like the area is not correct.

You need to make sure that you have the correct coordinates. You must specify the top left corner (the min coordinates)
Also you need to scale to the image size.
This will fix the existing code.
ctx.drawImage(pre_base_image,
(Math.min(storeRects[0].x, storeRects[0].x + Math.abs(storeRects[0].w)) / canvas.width) * pre_base_image.width,
(Math.min(storeRects[0].y, storeRects[0].y + Math.abs(storeRects[0].h)) / canvas.height) * pre_base_image.height,
(Math.abs(storeRects[0].w) / canvas.width) * pre_base_image.width ,
(Math.abs(storeRects[0].h) / canvas.height) * pre_base_image.height,
0, 0,
Math.abs(storeRects[0].w),
Math.abs(storeRects[0].h)
);
Check your previous question as I added an answer that takes care of this problem befor you need to do the above code.

drawing a rectange over an image on canvas using dragging event

I am trying to draw a rectangle on an image using the mouse and dragging events on HTML5.
My code is shown below. When I draw the rectangle below, the actual image on the canvas disappears. Could you tell me what I am doing wrong? My intended goal is to have the rectangle on top of the image. I have attached a picture of what I actually want to see as the end result.
What am I doing wrong ?
<!DOCTYPE HTML>
<html>
<head>
<style>
body {
margin: 0px;
padding: 0px;
}
</style>
</head>
<body>
<canvas id="myCanvas" width="578" height="400"></canvas>
<script>
var canvas = document.getElementById('myCanvas');
var ctx = canvas.getContext('2d');
var imageObj = new Image();
imageObj.onload = function() {
ctx.drawImage(imageObj, 69, 50);
};
imageObj.src = 'http://www.html5canvastutorials.com/demos/assets/darth-vader.jpg';
// ctx.globalAlpha = 0.5;
rect = {},
drag = false;
var rectStartXArray = new Array() ;
var rectStartYArray = new Array() ;
var rectWArray = new Array() ;
var rectHArray = new Array() ;
function init() {
canvas.addEventListener('mousedown', mouseDown, false);
canvas.addEventListener('mouseup', mouseUp, false);
canvas.addEventListener('mousemove', mouseMove, false);
}
function mouseDown(e) {
rect.startX = e.pageX - this.offsetLeft;
rect.startY = e.pageY - this.offsetTop;
drag = true;
}
function mouseUp() {
rectStartXArray[rectStartXArray.length] = rect.startX;
rectStartYArray[rectStartYArray.length] = rect.startY;
rectWArray[rectWArray.length] = rect.w;
rectHArray[rectHArray.length] = rect.h;
drag = false;
}
function mouseMove(e) {
if (drag) {
rect.w = (e.pageX - this.offsetLeft) - rect.startX;
rect.h = (e.pageY - this.offsetTop) - rect.startY;
ctx.clearRect(0, 0, canvas.width, canvas.height);
draw();
}
//drawOldShapes();
}
function draw() {
ctx.beginPath();
ctx.fillStyle="#FF0000";
ctx.fillRect(rect.startX, rect.startY, rect.w, rect.h);
ctx.stroke();
}
function drawOldShapes(){
for(var i=0;i<rectStartXArray.length;i++)
{
if(rectStartXArray[i]!= rect.startX && rectStartYArray[i] != rect.startY && rectWArray[i] != rect.w && rectHArray[i] != rect.h)
{
ctx.beginPath();
ctx.fillStyle="#FF0000";
ctx.fillRect(rectStartXArray[i], rectStartYArray[i], rectWArray[i], rectHArray[i]);
ctx.stroke();
}
}
}
init();
</script>
</body>
</html>

You are clearing the whole canvas inside draw() by calling ctx.fillRect(rect.startX, rect.startY, rect.w, rect.h);. Remove the line and it works. Fiddle - http://jsfiddle.net/da8wv75k/

HTML5 Canvas why fillText() method clears everything after clearRect()

When i call writeTextToCanvas method before clearCanvas method, it works perfectly,
if i call clearCanvas method first than writeTextToCanvas, it doesn't works, drawing functions etc. all works after clearCanvas but fillText doesn't work and also clears all canvas when i call fillText
when i set context.globalAlpha = 0.5 before fillText, i can barely see text on the canvas but anything in canvas erased somehow
var canvas;
var context;
var selectedRole = DRAW_TOOLS.PENCIL;
function initDrawingCanvas(index) {
var question = document.getElementsByClassName('question').item(index);
canvas = question.getElementsByClassName('questionDrawingCanvas').item(0);
context = canvas.getContext("2d");
canvasWidth = canvas.width;
canvasHeight = canvas.height;
compositeOperation = context.globalCompositeOperation;
canvas.addEventListener("touchmove", onCanvasTouchMove, false);
canvas.addEventListener("touchstart", onCanvasTouchStart, false);
canvas.addEventListener("touchstop", onCanvasTouchStop, false);
}
var startX = 0;
var startY = 0;
var endX = 0;
var endY = 0;
var compositeOperation;
function onCanvasTouchMove(event){
endX = event.changedTouches[0].clientX - $(canvas).offset().left;
endY = event.changedTouches[0].clientY - $(canvas).offset().top;
if(selectedRole == DRAW_TOOLS.PENCIL){
context.beginPath();
context.moveTo(startX, startY);
context.lineTo(endX, endY);
context.strokeStyle = $('.colorPicker').css('background-color');
context.lineWidth = parseInt($('#pencilSize').attr("data-size"));
context.stroke();
}else if(selectedRole == DRAW_TOOLS.ERASER){
context.save();
context.globalCompositeOperation = "destination-out";
context.arc(endX,endY,25,0,Math.PI*2,false);
context.fill();
context.restore();
}
startX = endX;
startY = endY;
}
function onCanvasTouchStart(event){
$("#colorPicker").css("display", "none");
$("#pencilSize").css("display", "none");
$("#textColorPicker").css("display", "none");
$("#canvas_textSize").css("display", "none");
startX = event.changedTouches[0].pageX - $(canvas).offset().left;
startY = event.changedTouches[0].pageY - $(canvas).offset().top;
if(selectedRole == DRAW_TOOLS.TEXT){
showConfirmDialog('<textarea id="canvasTextArea" class="canvasTextArea"></textarea>', RESOURCES.CANVAS_TEXT_TITLE, {positiveButton: RESOURCES.OKAY, negativeButton: RESOURCES.CANCEL}, function(){
var text = $("#canvasTextArea").val();
if(isNullOrUndefined(text)) return;
writeTextToCanvas(text);
hideDialog();
}, function(){
hideDialog();
});
$("#canvasTextArea").focus();
$("#canvasTextArea").css("font-size", $('#textSizePicker').attr("data-size") + "px");
$("#canvasTextArea").css("color", $('.textColorPicker').css("background-color"));
}
}
function onCanvasTouchStop(event){
console.log(event);
}
function clearCanvas(){
context.clearRect(0, 0, canvas.width, canvas.height);
}
function getCanvasContent(){
return canvas.toDataURL();
}
function writeTextToCanvas(text) {
context.globalCompositeOperation = "source-over";
context.textBaseline = "top";
context.font = $('#textSizePicker').attr("data-size") + 'px sans-serif';
context.fillStyle = $('.textColorPicker').css("background-color");
context.fillText(text, startX, startY);
}
What should i use to clear canvas or clear some parts like eraser then i can fillText normally?

Your call to clearRect() is stopping the entire script because it throws an error (check the console). You're not passing in a rectangle to the call.
It needs to be called like:
context.clearRect(0, 0, canvas.width, canvas.height);
Here's a working example: http://jsfiddle.net/pe19L15m