Develop Reference

Loading video frames into javascript in google colab

I wish to present a video, frame by frame with the possibility of flipping between frames in google colab. I also wish to mark some rectangles and get the x,y,w,h of each rectangle.
As far as I could understand, the only way to click the output and gather data in google Collab is via a javascript call, which I am new to. I have split my video into a list called frames and written the following python\javascript code for my task:
def get_fxy_val(base64imgs):
js = Javascript('''
async function showImage(base64) {
var canvas = document.createElement('canvas');
var rect = {};
var drag = false;
var finish = false;
var clicks = [];
var img = document.createElement('img');
var frame = 0
var ctx = canvas.getContext('2d');
document.body.appendChild(canvas);
img = new Image();
img.onload = function () {
canvas.height = img.height;
canvas.width = img.width;
ctx.drawImage(img, 0, 0);
};
img.src = 'data:image/jpeg;base64,'+base64[frame];
var prev = document.createElement('button');
prev.innerHTML = '<';
prev.onclick = function() {
frame = frame -1;
if(frame<0){
frame = 0;
}
else{
// console.log(frame)
img.src = 'data:image/jpeg;base64,'+base64[frame];
};
};
document.body.appendChild(prev);
var next = document.createElement('button');
next.innerHTML = '>';
next.onclick = function() {
frame = frame+1;
if (frame>base64.length-1) {
// console.log(frame)
frame = base64.length-1
}
else{
img.src = 'data:image/jpeg;base64,'+base64[frame];
}
};
document.body.appendChild(next);
canvas.addEventListener('mousedown', mouseDown, false);
canvas.addEventListener('mouseup', mouseUp, false);
canvas.addEventListener('mousemove', mouseMove, false);
function mouseDown(e) {
rect.startX = e.pageX - this.offsetLeft;
rect.startY = e.pageY - this.offsetTop;
drag = true;
}
function mouseUp() {
drag = false;
ctx.clearRect(0, 0, canvas.width, canvas.height);
ctx.drawImage(img, 0, 0);
if (rect.w<0){
rect.startX = rect.startX + rect.w
rect.w = rect.w*(-1)
}
if (rect.h<0){
rect.startY = rect.startY + rect.h
rect.h = rect.h*(-1)
}
ctx.strokeRect(rect.startX, rect.startY, rect.w, rect.h);
}
function mouseMove(e) {
if (drag) {
rect.w = (e.pageX - this.offsetLeft) - rect.startX;
rect.h = (e.pageY - this.offsetTop) - rect.startY;
ctx.strokeStyle = 'red';
ctx.lineWidth = 2;
ctx.strokeRect(rect.startX, rect.startY, rect.w, rect.h);
}
}
var saver = document.createElement('button');
saver.innerHTML = 'save rectangle';
saver.onclick = function() {
clicks.push([frame, rect.startX, rect.startY, rect.w, rect.h]);
ctx.clearRect(0, 0, canvas.width, canvas.height);
ctx.drawImage(img, 0, 0);
};
document.body.appendChild(saver);
var finisher = document.createElement('button');
finisher.innerHTML = 'Done!';
finisher.onclick = function() {
};
document.body.appendChild(finisher);
await new Promise((resolve) => finisher.onclick = resolve);
document.body.removeChild(finisher)
document.body.removeChild(saver)
document.body.removeChild(next)
document.body.removeChild(prev)
document.body.removeChild(canvas)
return clicks
}
''')
display(js)
imgs = ["'"+b64img+"'" for b64img in base64imgs]
imgs = '[' + ','.join(imgs) +']'
data = eval_js(f"showImage({imgs})")
return data
I call it using:
base64imgs = [base64.b64encode(cv2.imencode('.jpg', img)[1]).decode() for img in frames]
data = get_fxy_val(base64imgs)
Everything works well for a small number of images, however for the full video Collab crushes and I cannot manage to find any logs to verify what is going on.
Two thoughts I had, which I do not know how to implement:
Reduce the images' memory size somehow. I cannot harm the aspect ratio so downsampling is not a possibility.
Load images directly from memory inside the javascript code. I have no idea how to implement this or if it is even possible.
Can I implement any of these? Is there another approach?
B.T.W, my video has images with size h,w,c = (1080, 1920, 3).

Recording video stream with img filter overlay

I am using a face tracking library (tracking.js) to capture a video stream and place an image on top of the face.
The image is drawn on a canvas, which has the same width and height as the video therefore, the overlay.
I am trying to take a picture and video of the stream + canvas image, however O can only get a crude implementation of the stream and image that is distorted.
Here is a CodePen
const canvas = document.getElementById('canvas');
const context = canvas.getContext('2d');
const tracker = new tracking.ObjectTracker('face');
const flowerCrownButton = document.getElementById('flower-crown');
tracker.setInitialScale(1);
tracker.setStepSize(2.7);
tracker.setEdgesDensity(.2);
const img = document.createElement("img");
img.setAttribute("id", "pic");
img.src = canvas.toDataURL();
let filterX = 0;
let filterY = 0;
let filterWidth = 0;
let filterHeight = 0;
function changePic(x, y, width, height, src) {
img.src = src;
filterX = x;
filterY = y;
filterWidth = width;
filterHeight = height;
}
function flowerCrown() {
changePic(0, -0.5, 1, 1, 'https://s3-us-west-
2. amazonaws.com / s.cdpn.io / 450347 / flower - crown.png ')
}
flowerCrownButton.addEventListener('click', flowerCrown);
//listen for track events
tracker.on('track', function(event) {
//if (event.data.length === 0) {
//alert("No objects were detected in this frame.");
//} else {
context.clearRect(0, 0, canvas.width, canvas.height)
event.data.forEach(rect => {
context.drawImage(img, rect.x + (filterX * rect.width),
rect.y + (filterY * rect.height),
rect.width * filterWidth,
rect.height * filterHeight
)
})
//}// end of else
});
//start tracking
tracking.track('#video', tracker, {
camera: true
})
const canvas2 = document.getElementById('canvas2');
const context2 = canvas2.getContext('2d');
const video = document.getElementById("video");
video.addEventListener("loadedmetadata", function() {
ratio = video.videoWidth / video.videoHeight;
w = video.videoWidth - 100;
h = parseInt(w / ratio, 10);
canvas2.width = w;
canvas2.height = h;
}, false);
function snap() {
context2.drawImage(video, 10, 5);
context2.drawImage(img, 10, 10)
}
}
Any ideas? I prefer to use the media recorder API and have tried it, but again could not get a stream or picture with the image filter overlay.
Thanks and please don't be snarky :)

Clearing drawing from Canvas permanently

I have a function that clears all drawings off of the background image in canvas when a button is clicked, but when you go to draw again, the old (previously cleared drawings) reappears. How can I make a hard delete so that I can draw again without reloading the page
'use strict';
function initCanvas() {
let bMouseIsDown = false;
let canvas = document.getElementById('cvs');
let ctx = canvas.getContext('2d');
let convert = document.getElementById('convert');
let sel = 'png';
let imgs = document.getElementById('imgs');
let imgW = 300;
let imgH = 200;
let background = new Image();
background.crossOrigin = '';
background.src = "http://i.imgur.com/yf6d9SX.jpg";
background.onload = function(){
ctx.drawImage(background,0,0,600,400);
}
bind(canvas,ctx,convert,sel,imgs,imgW,imgH,bMouseIsDown);
};
initCanvas()
function bind (canvas,ctx,convert,sel,imgs,imgW,imgH,bMouseIsDown) {
let iLastX = 0;
let iLastY = 0;
let iX;
let iY;
canvas.onmousedown = function(e) {
bMouseIsDown = true;
iLastX = e.clientX - canvas.offsetLeft + (window.pageXOffset||document.body.scrollLeft||document.documentElement.scrollLeft);
iLastY = e.clientY - canvas.offsetTop + (window.pageYOffset||document.body.scrollTop||document.documentElement.scrollTop);
}
canvas.onmouseup = function() {
bMouseIsDown = false;
iLastX = -1;
iLastY = -1;
}
canvas.onmousemove = function(e) {
if (bMouseIsDown) {
iX = e.clientX - canvas.offsetLeft + (window.pageXOffset||document.body.scrollLeft||document.documentElement.scrollLeft);
iY = e.clientY - canvas.offsetTop + (window.pageYOffset||document.body.scrollTop||document.documentElement.scrollTop);
ctx.moveTo(iLastX, iLastY);
ctx.lineTo(iX, iY);
ctx.stroke();
ctx.strokeStyle = "blue";
ctx.lineJoin = "round";
ctx.lineWidth = 5;
iLastX = iX;
iLastY = iY;
}
};
document.getElementById('clear').addEventListener('click', function() {
rerenderImg();
});
function rerenderImg() {
iY = [];
iX=[];
initCanvas()
}
};

You need to call ctx.beginPath(); before drawing on the canvas again.
The logical place to put this is just before your call to ctx.moveTo.
An explanation of why this is needed is given here.

You should actually run clearRect on the canvas:
ctx.clearRect(0, 0, canvas.width, canvas.height);

Upload two images into html5 canvas

I am trying to upload two and more images into one canvas and make some sort of collage. Pictures are the same size. However, when I try upload the second image, the first one disappears. Below there is the code and example to JSfiddle. What is wrong?
jsfiddle
<div>
<input type="file" id="imageLoader1" name="imageLoader" />
<br/>
<input type="file" id="imageLoader2" name="imageLoader" />
<br/>
<canvas id="canvas" style="background:red;"></canvas>
</div>
And JS code:
var imageLoader1 = document.getElementById('imageLoader1');
var imageLoader2 = document.getElementById('imageLoader2');
imageLoader1.addEventListener('change', handleImage, false);
imageLoader2.addEventListener('change', handleImage, false);
var canvas = document.getElementById('canvas');
var ctx = canvas.getContext('2d');
function handleImage(e) {
if (e.target == imageLoader1) {
var reader = new FileReader();
reader.onload = function (event) {
var img1 = new Image();
img1.onload = function () {
canvas.width = 1000;
canvas.height = 500;
ctx.drawImage(img1, 0, 0, img1.width, img1.height, 0, 0, img1.width * 0.4, img1.height * 0.4);
}
img1.src = event.target.result;
}
reader.readAsDataURL(e.target.files[0]);
}
if (e.target == imageLoader2) {
var reader = new FileReader();
reader.onload = function (event) {
var img2 = new Image();
img2.onload = function () {
canvas.width = 1000;
canvas.height = 500;
ctx.drawImage(img2, 0, 0, img2.width, img2.height, img2.width * 0.4, img2.height * 0.4, img2.width * 0.4, img2.height * 0.4);
}
img2.src = event.target.result;
}
reader.readAsDataURL(e.target.files[0]);
}
}
The idea was taken here: How to upload image into HTML5 canvas

When you are trying to superimpose multiple images in single canvas,You need to use two distinct paths, also you need to clearly circumscribe them with the .beginPath() method.
I have modified you code, please check. It should work for you.
function handleImage(e) {
if (e.target == imageLoader1) {
var reader = new FileReader();
reader.onload = function (event) {
var img1 = new Image();
img1.onload = function () {
canvas.width = 1000;
canvas.height = 500;
//added begin path here
ctx.beginPath();
ctx.drawImage(img1, 0, 0, img1.width, img1.height, 0, 0, img1.width * 0.4, img1.height * 0.4);
}
img1.src = event.target.result;
}
reader.readAsDataURL(e.target.files[0]);
}
if (e.target == imageLoader2) {
var reader = new FileReader();
reader.onload = function (event) {
var img2 = new Image();
img2.onload = function () {
canvas.width = 1000;
canvas.height = 500;
//added begin path here
ctx.beginPath();
ctx.drawImage(img2, 0, 0, img2.width, img2.height, img2.width * 0.4, img2.height * 0.4, img2.width * 0.4, img2.height * 0.4);
}
img2.src = event.target.result;
}
reader.readAsDataURL(e.target.files[0]);
}
}
Alternate :
Or what you can do is use globalAlpha property of canvas.
Try : ctx.globalAlpha = 0.5; .This may solve your issue.
Alternate 2:
You can use multiple canvas layer for multiple images. This will directly superimpose your images. You can superimpose as many images as you want.

I've modified original jsfiddle: click.
Here what have been changed:
var imageLoader1 = document.getElementById('imageLoader1');
var imageLoader2 = document.getElementById('imageLoader2');
imageLoader1.addEventListener('change', handleImage, false);
imageLoader2.addEventListener('change', handleImage, false);
var canvas = document.getElementById('canvas');
var ctx = canvas.getContext('2d');
let z = 0.4; // zoom
canvas.width = 1000;
canvas.height = 500;
function handleImage(e) {
if (e.target == imageLoader1) {
let img = new Image();
img.src = URL.createObjectURL(e.target.files[0]);
img.onload = function() {
ctx.drawImage(img, 0, 0, img.width, img.height, 0, 0, img.width * z, img.height * z);
}
}
if (e.target == imageLoader2) {
let img = new Image();
img.src = URL.createObjectURL(e.target.files[0]);
img.onload = function() {
ctx.drawImage(img, 0, 0, img.width, img.height, img.width * z, img.height * z, img.width * z, img.height * z);
}
}
}
I've moved canvas initialization out of load handler and simplified the loader.
This solution will not superimpose your images (but this is just a fix for original jsfiddle from the Question).

use window.URL.createObjectURL(stream) multible times

I have a SPA build with AngularJS.
ater Login i save the Streaming Object in a global variable for Future use (so i have to set the permission just once)
I use this StreamingObject at two userstories (for now)
Story one:
change profile image
Story two make a selfie and post it.
Everything works fine until i switch from Story one to Story Two.
This breaks the Stream.
I wrote this directive.
angular.module('myApp')
.controller('webcamCtrl', function ($scope, globVal) {
var width = 373;
var height = 0;
var streaming = false;
var video = null;
var canvas = null;
$scope.showPicture = false;
$scope.startup = function() {
video = document.getElementById('video');
canvas = document.getElementById('canvas');
$scope.$watch(function () {
return globVal.webcam;
}, function() {
if(globVal.webcam !== null){
video.src = globVal.webcam;
video.play();
}
});
video.addEventListener('canplay', function () {
if (!streaming) {
height = video.videoHeight / (video.videoWidth / width);
if (isNaN(height)) {
height = width / (4 / 3);
}
video.setAttribute('width', width);
video.setAttribute('height', height);
canvas.setAttribute('width', width);
canvas.setAttribute('height', height);
streaming = true;
}
}, false);
$scope.clearphoto();
};
$scope.clearphoto = function() {
var context = canvas.getContext('2d');
context.fillStyle = '#AAA';
context.fillRect(0, 0, canvas.width, canvas.height);
$scope.showPicture = false;
globVal.image = null;
};
$scope.takepicture = function() {
var context = canvas.getContext('2d');
if(!$scope.showPicture){
if (width && height) {
canvas.width = width;
canvas.height = height;
context.drawImage(video, 0, 0, width, height);
$scope.showPicture = true;
$scope.acceptpicture();
} else {
$scope.clearphoto();
}
}else {
$scope.clearphoto();
}
};
$scope.acceptpicture = function() {
if($scope.showPicture){
var data = canvas.toDataURL('image/png');
globVal.image = data;
}else{
globVal.image = null;
}
};
$scope.startup();
});
any hints?

Solved. i add a ng-if to my directive so the $scope of teh webcam can destroyed safely
<web-cam ng-if="showCam"></web-cam