Develop Reference

Loading video frames into javascript in google colab

I wish to present a video, frame by frame with the possibility of flipping between frames in google colab. I also wish to mark some rectangles and get the x,y,w,h of each rectangle.
As far as I could understand, the only way to click the output and gather data in google Collab is via a javascript call, which I am new to. I have split my video into a list called frames and written the following python\javascript code for my task:
def get_fxy_val(base64imgs):
js = Javascript('''
async function showImage(base64) {
var canvas = document.createElement('canvas');
var rect = {};
var drag = false;
var finish = false;
var clicks = [];
var img = document.createElement('img');
var frame = 0
var ctx = canvas.getContext('2d');
document.body.appendChild(canvas);
img = new Image();
img.onload = function () {
canvas.height = img.height;
canvas.width = img.width;
ctx.drawImage(img, 0, 0);
};
img.src = 'data:image/jpeg;base64,'+base64[frame];
var prev = document.createElement('button');
prev.innerHTML = '<';
prev.onclick = function() {
frame = frame -1;
if(frame<0){
frame = 0;
}
else{
// console.log(frame)
img.src = 'data:image/jpeg;base64,'+base64[frame];
};
};
document.body.appendChild(prev);
var next = document.createElement('button');
next.innerHTML = '>';
next.onclick = function() {
frame = frame+1;
if (frame>base64.length-1) {
// console.log(frame)
frame = base64.length-1
}
else{
img.src = 'data:image/jpeg;base64,'+base64[frame];
}
};
document.body.appendChild(next);
canvas.addEventListener('mousedown', mouseDown, false);
canvas.addEventListener('mouseup', mouseUp, false);
canvas.addEventListener('mousemove', mouseMove, false);
function mouseDown(e) {
rect.startX = e.pageX - this.offsetLeft;
rect.startY = e.pageY - this.offsetTop;
drag = true;
}
function mouseUp() {
drag = false;
ctx.clearRect(0, 0, canvas.width, canvas.height);
ctx.drawImage(img, 0, 0);
if (rect.w<0){
rect.startX = rect.startX + rect.w
rect.w = rect.w*(-1)
}
if (rect.h<0){
rect.startY = rect.startY + rect.h
rect.h = rect.h*(-1)
}
ctx.strokeRect(rect.startX, rect.startY, rect.w, rect.h);
}
function mouseMove(e) {
if (drag) {
rect.w = (e.pageX - this.offsetLeft) - rect.startX;
rect.h = (e.pageY - this.offsetTop) - rect.startY;
ctx.strokeStyle = 'red';
ctx.lineWidth = 2;
ctx.strokeRect(rect.startX, rect.startY, rect.w, rect.h);
}
}
var saver = document.createElement('button');
saver.innerHTML = 'save rectangle';
saver.onclick = function() {
clicks.push([frame, rect.startX, rect.startY, rect.w, rect.h]);
ctx.clearRect(0, 0, canvas.width, canvas.height);
ctx.drawImage(img, 0, 0);
};
document.body.appendChild(saver);
var finisher = document.createElement('button');
finisher.innerHTML = 'Done!';
finisher.onclick = function() {
};
document.body.appendChild(finisher);
await new Promise((resolve) => finisher.onclick = resolve);
document.body.removeChild(finisher)
document.body.removeChild(saver)
document.body.removeChild(next)
document.body.removeChild(prev)
document.body.removeChild(canvas)
return clicks
}
''')
display(js)
imgs = ["'"+b64img+"'" for b64img in base64imgs]
imgs = '[' + ','.join(imgs) +']'
data = eval_js(f"showImage({imgs})")
return data
I call it using:
base64imgs = [base64.b64encode(cv2.imencode('.jpg', img)[1]).decode() for img in frames]
data = get_fxy_val(base64imgs)
Everything works well for a small number of images, however for the full video Collab crushes and I cannot manage to find any logs to verify what is going on.
Two thoughts I had, which I do not know how to implement:
Reduce the images' memory size somehow. I cannot harm the aspect ratio so downsampling is not a possibility.
Load images directly from memory inside the javascript code. I have no idea how to implement this or if it is even possible.
Can I implement any of these? Is there another approach?
B.T.W, my video has images with size h,w,c = (1080, 1920, 3).

How to Transfer the image capture by web cam to google cloud vision api

In this I am capturing the Image from the Webcam.But I fail to transfer the image to the google cloud vision api.I wanna do the text detection also when the capturing the image and image transfer through the api.
Can anybody help out with that problem?
This is Webcam code
<script>
(function() {
var width = 320; // We will scale the photo width to this
var height = 0; // This will be computed based on the input stream
var streaming = false;
var video = null;
var canvas = null;
var photo = null;
var startbutton = null;
var data=null;
function startup() {
video = document.getElementById('video');
canvas = document.getElementById('canvas');
photo = document.getElementById('photo');
startbutton = document.getElementById('startbutton');
navigator.mediaDevices.getUserMedia({
video: true,
audio: false
})
.then(function(stream) {
video.srcObject = stream;
video.play();
})
.catch(function(err) {
console.log("An error occurred: " + err);
});
video.addEventListener('canplay', function(ev) {
if (!streaming) {
height = video.videoHeight / (video.videoWidth / width);
if (isNaN(height)) {
height = width / (4 / 3);
}
video.setAttribute('width', width);
video.setAttribute('height', height);
canvas.setAttribute('width', width);
canvas.setAttribute('height', height);
streaming = true;
}
}, false);
startbutton.addEventListener('click', function(ev) {
takepicture();
ev.preventDefault();
}, false);
clearphoto();
quickstart()
}
function clearphoto() {
var context = canvas.getContext('2d');
context.fillStyle = "#AAA";
context.fillRect(0, 0, canvas.width, canvas.height);
data = canvas.toDataURL('image/png');
photo.setAttribute('src', data);
}
function takepicture() {
var context = canvas.getContext('2d');
if (width && height) {
canvas.width = width;
canvas.height = height;
context.drawImage(video, 0, 0, width, height);
data = canvas.toDataURL('image/png');
photo.setAttribute('src', data);
} else {
clearphoto();
}
}
window.addEventListener('load', startup, false);
})();
</script>
I wanna add that code in the webcam portion.
// Imports the Google Cloud client library
async function quickstart() {
process.env.GOOGLE_APPLICATION_CREDENTIALS = "/home/manu/Cap/Real_Time/VisionAPI_DEMO/ServiceAccountToken.json"
const vision = require('#google-cloud/vision');
const client = new vision.ImageAnnotatorClient();
// Performs text detection on the local file
const [result] = await client.textDetection("image/test.jpg");
const detections = result.textAnnotations;
const [ text, ...others ] = detections
console.log(`Text: ${ text.description }`);
}
quickstart()

Recording video stream with img filter overlay

I am using a face tracking library (tracking.js) to capture a video stream and place an image on top of the face.
The image is drawn on a canvas, which has the same width and height as the video therefore, the overlay.
I am trying to take a picture and video of the stream + canvas image, however O can only get a crude implementation of the stream and image that is distorted.
Here is a CodePen
const canvas = document.getElementById('canvas');
const context = canvas.getContext('2d');
const tracker = new tracking.ObjectTracker('face');
const flowerCrownButton = document.getElementById('flower-crown');
tracker.setInitialScale(1);
tracker.setStepSize(2.7);
tracker.setEdgesDensity(.2);
const img = document.createElement("img");
img.setAttribute("id", "pic");
img.src = canvas.toDataURL();
let filterX = 0;
let filterY = 0;
let filterWidth = 0;
let filterHeight = 0;
function changePic(x, y, width, height, src) {
img.src = src;
filterX = x;
filterY = y;
filterWidth = width;
filterHeight = height;
}
function flowerCrown() {
changePic(0, -0.5, 1, 1, 'https://s3-us-west-
2. amazonaws.com / s.cdpn.io / 450347 / flower - crown.png ')
}
flowerCrownButton.addEventListener('click', flowerCrown);
//listen for track events
tracker.on('track', function(event) {
//if (event.data.length === 0) {
//alert("No objects were detected in this frame.");
//} else {
context.clearRect(0, 0, canvas.width, canvas.height)
event.data.forEach(rect => {
context.drawImage(img, rect.x + (filterX * rect.width),
rect.y + (filterY * rect.height),
rect.width * filterWidth,
rect.height * filterHeight
)
})
//}// end of else
});
//start tracking
tracking.track('#video', tracker, {
camera: true
})
const canvas2 = document.getElementById('canvas2');
const context2 = canvas2.getContext('2d');
const video = document.getElementById("video");
video.addEventListener("loadedmetadata", function() {
ratio = video.videoWidth / video.videoHeight;
w = video.videoWidth - 100;
h = parseInt(w / ratio, 10);
canvas2.width = w;
canvas2.height = h;
}, false);
function snap() {
context2.drawImage(video, 10, 5);
context2.drawImage(img, 10, 10)
}
}
Any ideas? I prefer to use the media recorder API and have tried it, but again could not get a stream or picture with the image filter overlay.
Thanks and please don't be snarky :)

How to create a JavaScript Audio Visualizer?

I've been searching around looking for some solutions but haven't really found much. I'd like something really simple along the lines of the image below. Has anyone ever used one in a project? Any advice or any API's I could use? Thanks.

Here is base:
You need a canvas
You need canvas context
You need audio context
var canvas = document.createElement("canvas");
canvas.width = 500;
canvas.height = 180;
var ctx = canvas.getContext("2d");
ctx.fillStyle = "black";
ctx.strokeStyle = "white";
ctx.lineCap = "round";
var auctx;
window.onload = () => {
document.body.appendChild(canvas);
auctx = new(window.AudioContext || window.webkitAudioContext)();
startAudio();
}
var buffer, src, analyser, buffLen;
var barWidth, dataArray;
function startAudio() {
var url = "https://cf-media.sndcdn.com/cTGZiRbnSouE.128.mp3?Policy=eyJTdGF0ZW1lbnQiOlt7IlJlc291cmNlIjoiKjovL2NmLW1lZGlhLnNuZGNkbi5jb20vY1RHWmlSYm5Tb3VFLjEyOC5tcDMiLCJDb25kaXRpb24iOnsiRGF0ZUxlc3NUaGFuIjp7IkFXUzpFcG9jaFRpbWUiOjE1MTk5NTQ5NjB9fX1dfQ__&Signature=JmNkAHzih0~f3lQVwvPXFeTIUVuMXbwlbqizsXbCtc6lFIxjRlqa3wUGp5-xAkt7AUlhiYxu~Wscc6MfQTTc527DHJURMpdqvdXv61ll-WJqoV1V-tpWSa~qR-NEAWGCGBvrge0BkRRAsOHFljeLNCvO3DjzH7lSTPMlV-MtbFV2k-PiY0vrY1LuicAOcfEtXYTiMBkg-rhzkeHFcNHYt2Nb2hmIvmWFI1cFG74FBIXTnVPAg2Yo0r-LeiirWvSgewkIu~zPzaVYjnPaN1y-ZGnPBFiBSC1mpVhtB5wkhTXF5LFthkGUHnUK2ybESr-1uOH9GLye-7dxdIXx~A1LDA__&Key-Pair-Id=APKAJAGZ7VMH2PFPW6UQ"; // nice url
var request = new XMLHttpRequest();
request.open('GET', url, true);
request.responseType = 'arraybuffer';
request.onload = function() {
auctx.decodeAudioData(request.response, function(buffer) {
buffer = buffer;
src = auctx.createBufferSource();
src.buffer = buffer;
src.loop = false;
src.connect(auctx.destination);
src.start(0);
analyser = auctx.createAnalyser();
src.connect(analyser);
analyser.connect(auctx.destination);
analyser.fftSize = 256;
buffLen = analyser.frequencyBinCount;
dataArray = new Uint8Array(buffLen);
barWidth = (500 - 2 * buffLen - 4) / buffLen * 2.5;
ctx.lineWidth = barWidth;
draw();
});
}
request.send();
}
function draw() {
ctx.fillRect(0, 0, 500, 180);
analyser.getByteFrequencyData(dataArray);
for (var i = 0; i < buffLen; i++) {
ctx.beginPath();
ctx.moveTo(4 + 2 * i * barWidth + barWidth / 2, 178 - barWidth / 2);
ctx.lineTo(4 + 2 * i * barWidth + barWidth / 2, 178 - dataArray[i] * 0.65 - barWidth / 2);
ctx.stroke();
}
requestAnimationFrame(draw);
}
canvas {
background: black;
}
This code should work. You can add some images and tweak settings.

I think that this is maybe what you’re looking for. Sorry I’m a bit late.
// AUDIO CONTEXT
window.AudioContext = (window.AudioContext ||
window.webkitAudioContext ||
window.mozAudioContext ||
window.oAudioContext ||
window.msAudioContext);
if (!AudioContext) alert('This site cannot be run in your Browser. Try a recent Chrome or Firefox. ');
var audioContext = new AudioContext();
var currentBuffer = null;
// CANVAS
var canvasWidth = window.innerWidth, canvasHeight = 120 ;
var newCanvas = createCanvas (canvasWidth, canvasHeight);
var context = null;
window.onload = appendCanvas;
function appendCanvas() { document.body.appendChild(newCanvas);
context = newCanvas.getContext('2d'); }
// the function that loads the sound file
//NOTE this program for some reason won’t load sound files from like a weebly website so you’ll have to add the files to your github or whatever and use that raw audio file
function loadMusic(url) {
var req = new XMLHttpRequest();
req.open( "GET", url, true );
req.responseType = "arraybuffer";
req.onreadystatechange = function (e) {
if (req.readyState == 4) {
if(req.status == 200)
audioContext.decodeAudioData(req.response,
function(buffer) {
currentBuffer = buffer;
displayBuffer(buffer);
}, onDecodeError);
else
alert('error during the load.Wrong url or cross origin issue');
}
} ;
req.send();
}
function onDecodeError() { alert('error while decoding your file.'); }
// MUSIC DISPLAY
function displayBuffer(buff /* is an AudioBuffer */) {
var drawLines = 500;
var leftChannel = buff.getChannelData(0); // Float32Array describing left channel
var lineOpacity = canvasWidth / leftChannel.length ;
context.save();
context.fillStyle = '#080808' ;
context.fillRect(0,0,canvasWidth,canvasHeight );
context.strokeStyle = '#46a0ba';
context.globalCompositeOperation = 'lighter';
context.translate(0,canvasHeight / 2);
//context.globalAlpha = 0.6 ; // lineOpacity ;
context.lineWidth=1;
var totallength = leftChannel.length;
var eachBlock = Math.floor(totallength / drawLines);
var lineGap = (canvasWidth/drawLines);
context.beginPath();
for(var i=0;i<=drawLines;i++){
var audioBuffKey = Math.floor(eachBlock * i);
var x = i*lineGap;
var y = leftChannel[audioBuffKey] * canvasHeight / 2;
context.moveTo( x, y );
context.lineTo( x, (y*-1) );
}
context.stroke();
context.restore();
}
// Creates the Canvas
function createCanvas ( w, h ) {
var newCanvas = document.createElement('canvas');
newCanvas.width = w; newCanvas.height = h;
return newCanvas;
};
// The program runs the url you put into the line below
loadMusic('||YOUR LINK||');
Happy Coding!!
If you by chance have a better solution on this, may you send me the code because I’m also having a bit of trouble with this. I’m trying to create one like soundcloud without using external libraries.
EDIT 1
So i thought that it would be nice if i give an example of what it would look like in action so, here —>
// AUDIO CONTEXT
window.AudioContext = (window.AudioContext ||
window.webkitAudioContext ||
window.mozAudioContext ||
window.oAudioContext ||
window.msAudioContext);
if (!AudioContext) alert('This site cannot be run in your Browser. Try a recent Chrome or Firefox. ');
var audioContext = new AudioContext();
var currentBuffer = null;
// CANVAS
var canvasWidth = window.innerWidth, canvasHeight = 120 ;
var newCanvas = createCanvas (canvasWidth, canvasHeight);
var context = null;
window.onload = appendCanvas;
function appendCanvas() { document.body.appendChild(newCanvas);
context = newCanvas.getContext('2d'); }
// the function that loads the sound file
//NOTE this program for some reason won’t load sound files from like a weebly website so you’ll have to add the files to your github or whatever and use that raw audio file
function loadMusic(url) {
var req = new XMLHttpRequest();
req.open( "GET", url, true );
req.responseType = "arraybuffer";
req.onreadystatechange = function (e) {
if (req.readyState == 4) {
if(req.status == 200)
audioContext.decodeAudioData(req.response,
function(buffer) {
currentBuffer = buffer;
displayBuffer(buffer);
}, onDecodeError);
else
alert('error during the load.Wrong url or cross origin issue');
}
} ;
req.send();
}
function onDecodeError() { alert('error while decoding your file.'); }
// MUSIC DISPLAY
function displayBuffer(buff /* is an AudioBuffer */) {
var drawLines = 500;
var leftChannel = buff.getChannelData(0); // Float32Array describing left channel
var lineOpacity = canvasWidth / leftChannel.length ;
context.save();
context.fillStyle = '#080808' ;
context.fillRect(0,0,canvasWidth,canvasHeight );
context.strokeStyle = '#46a0ba';
context.globalCompositeOperation = 'lighter';
context.translate(0,canvasHeight / 2);
//context.globalAlpha = 0.6 ; // lineOpacity ;
context.lineWidth=1;
var totallength = leftChannel.length;
var eachBlock = Math.floor(totallength / drawLines);
var lineGap = (canvasWidth/drawLines);
context.beginPath();
for(var i=0;i<=drawLines;i++){
var audioBuffKey = Math.floor(eachBlock * i);
var x = i*lineGap;
var y = leftChannel[audioBuffKey] * canvasHeight / 2;
context.moveTo( x, y );
context.lineTo( x, (y*-1) );
}
context.stroke();
context.restore();
}
// Creates the Canvas
function createCanvas ( w, h ) {
var newCanvas = document.createElement('canvas');
newCanvas.width = w; newCanvas.height = h;
return newCanvas;
};
// The program runs the url you put into the line below
loadMusic('https://raw.githubusercontent.com/lightning417techa/Music/master/images/lil%20dicky%20-%20freaky%20friday%20(lyrics)%20ft.%20chris%20brown.mp3');
Note: that’s the kind of file that this thing needs, it’s annoying at times.

how to add a background image to signature pad

How do I export together with the background image together?
Demo :http://59.127.247.144/AmhsGlassrecord/
I want to get results :
enter image description here
Javascript
var wrapper = document.getElementById("signature-pad"),
clearButton = wrapper.querySelector("[data-action=clear]"),
saveButton = wrapper.querySelector("[data-action=save]"),
canvas = wrapper.querySelector("canvas"),
signaturePad;
function resizeCanvas() {
var ratio = Math.max(window.devicePixelRatio || 1, 1);
canvas.width = canvas.offsetWidth * ratio;
canvas.height = canvas.offsetHeight * ratio;
canvas.getContext("2d").scale(ratio, ratio);
}
window.onresize = resizeCanvas;
resizeCanvas();
signaturePad = new SignaturePad(canvas);
var signaturePad = new SignaturePad(canvas);
signaturePad.minWidth = 1;
signaturePad.maxWidth = 1;
signaturePad.penColor = "rgb(66, 133, 244)";
clearButton.addEventListener("click", function (event) {
signaturePad.clear();
});
saveButton.addEventListener("click", function (event) {
if (signaturePad.isEmpty()) {
alert("Please do not blank.");
} else {
//document.getElementById("hfSign").value = signaturePad.toDataURL();
window.open(signaturePad.toDataURL());
}
});

You'll need to draw the image on the canvas with Javascript, CSS will not work.
The background image that you have in the signature pad comes from another domain, it needs to be in your domain otherwise you cannot export it or you'll get the tainted canvas error.
To draw the image use the following code (but be sure that the image is hosted in the same domain of your app) :
Give an ID to your canvas, not only the wrapper :
var canvas = document.getElementById("ID-of-your-canvas"),
ctx = canvas.getContext("2d");
var background = new Image();
// The image needs to be in your domain.
background.src = "http://dkpopnews.fooyoh.com/files/attach/images4/14989425/2015/1/14995985/thumbnail_725x300_ratio.jpg";
// Make sure the image is loaded first otherwise nothing will draw.
background.onload = function() {
ctx.drawImage(background, 0, 0);
};
function action() {
// draw the image on the canvas
ctx.drawImage(background, 0, 0);
//// Then continue with your code
var wrapper = document.getElementById("signature-pad"),
clearButton = wrapper.querySelector("[data-action=clear]"),
saveButton = wrapper.querySelector("[data-action=save]"),
canvas = wrapper.querySelector("canvas"),
signaturePad;
function resizeCanvas() {
var ratio = Math.max(window.devicePixelRatio || 1, 1);
canvas.width = canvas.offsetWidth * ratio;
canvas.height = canvas.offsetHeight * ratio;
canvas.getContext("2d").scale(ratio, ratio);
}
window.onresize = resizeCanvas;
resizeCanvas();
signaturePad = new SignaturePad(canvas);
var signaturePad = new SignaturePad(canvas);
signaturePad.minWidth = 1;
signaturePad.maxWidth = 1;
signaturePad.penColor = "rgb(66, 133, 244)";
clearButton.addEventListener("click", function(event) {
signaturePad.clear();
});
saveButton.addEventListener("click", function(event) {
if (signaturePad.isEmpty()) {
alert("Please do not blank.");
} else {
//document.getElementById("hfSign").value = signaturePad.toDataURL();
window.open(signaturePad.toDataURL());
}
});
}
That would work at first with the image. Note that you need to draw again the image when the user clicks on the clear button.
The following demo works using instead of an image a color instead :
var canvas = document.getElementById("ID-of-your-canvas"),
ctx = canvas.getContext("2d");
function setBackground(color){
ctx.fillStyle = color;
ctx.fillRect(0, 0, canvas.width, canvas.height);
}
//// Then continue with your code
var wrapper = document.getElementById("signature-pad"),
clearButton = wrapper.querySelector("[data-action=clear]"),
saveButton = wrapper.querySelector("[data-action=save]"),
canvas = wrapper.querySelector("canvas"),
signaturePad;
function resizeCanvas() {
var ratio = Math.max(window.devicePixelRatio || 1, 1);
canvas.width = canvas.offsetWidth * ratio;
canvas.height = canvas.offsetHeight * ratio;
canvas.getContext("2d").scale(ratio, ratio);
}
window.onresize = resizeCanvas;
resizeCanvas();
signaturePad = new SignaturePad(canvas);
var signaturePad = new SignaturePad(canvas);
signaturePad.minWidth = 1;
signaturePad.maxWidth = 1;
signaturePad.penColor = "rgb(66, 133, 244)";
setBackground("#0e2630");
clearButton.addEventListener("click", function(event) {
signaturePad.clear();
setBackground("#0e2630");
});
saveButton.addEventListener("click", function(event) {
if (signaturePad.isEmpty()) {
alert("Please do not blank.");
} else {
//document.getElementById("hfSign").value = signaturePad.toDataURL();
window.open(signaturePad.toDataURL());
}
});
And the result :

You can just set backgroundColor for signaturePad.
var signaturePad = new SignaturePad(canvas,
{
backgroundColor: "rgba(0, 0, 0, 1)"
});