I am trying to run u2net model in browser, I have converted the pytorch u2netp model into ONNX model and wrote the following code to run it but the results very poor. I followed the same preprocessing steps as that of python script but did not get the results. I was not able to find onnx functions to perform the preprocessing so I have used for loops to change the values in each channel.
<!DOCTYPE html>
<html>
<header>
<title>ONNX Runtime JavaScript examples: Quick Start - Web (using script tag)</title>
<input id="image-selector" type="file" style="top:10px;left:10px" >
<button id="predict-button" class="btn btn-dark float-right" style="top:10px;left:70px" >Predict</button>
<img id="selected-image" src="" />
<canvas id="canvas" width =320px height=320px ></canvas>
</header>
<body>
<!-- import ONNXRuntime Web from CDN -->
<script src="https://code.jquery.com/jquery-3.3.1.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/onnxruntime-web/dist/ort.min.js"></script>
<script>
$("#image-selector").change(function () {
let reader = new FileReader();
reader.onload = function () {
let dataURL = reader.result;
$("#selected-image").attr("src", dataURL);
}
let file = $("#image-selector").prop("files")[0];
reader.readAsDataURL(file);
});
$("#predict-button").click(async function (){
const canvas = document.getElementById("canvas");
const ctx = canvas.getContext("2d");
const session = await ort.InferenceSession.create('./u2netp.onnx').then(console.log("model loaded"));
const inputNames = session.inputNames;
const outputNames = session.outputNames;
console.log('inputNames', inputNames)
console.log('outputNames', outputNames)
let image = $("#selected-image").get(0);
console.log("image.naturalHeight", image.naturalHeight)
console.log("image.naturalWidth", image.naturalWidth)
var oc = document.createElement('canvas'),
octx = oc.getContext('2d');
oc.width = 320;
oc.height = 320;
octx.drawImage(image, 0, 0, oc.width, oc.height);
var input_imageData = octx.getImageData(0, 0, 320, 320);
var floatArr = new Float32Array(320 * 320 * 3)
var j = 0
for (let i = 1; i < input_imageData.data.length+1; i ++) {
if(i % 4 != 0){
floatArr[j] = (input_imageData.data[i-1].toFixed(2))/255; // red color
j = j + 1;
}
}
console.log("floatArr1", floatArr)
for (let i = 1; i < floatArr.length+1; i += 3) {
floatArr[i-1] = (floatArr[i-1] - 0.485)/0.229 // red color
floatArr[i] = (floatArr[i] - 0.456)/0.224 // green color
floatArr[i+1] = (floatArr[i+1] - 0.406)/0.225 // blue color
}
console.log("floatArr2", floatArr)
const input = new ort.Tensor('float32', floatArr, [1, 3, 320, 320])
a = inputNames[0]
console.log("a", a)
const feeds = {"input.1": input};
console.log("feeds", feeds)
const results = await session.run(feeds).then();
const pred = Object.values(results)[0]
console.log('pred', pred)
console.log('pred.data.length', pred.data.length)
console.log('pred.data[0]', Math.round(pred.data[0]*255))
var myImageData = ctx.createImageData(320, 320);
for (let i = 0; i < pred.data.length*4; i += 4) {
var pixelIndex = i;
if(i != 0){
t = i/4;
}
else{
t = 0;
}
myImageData.data[pixelIndex ] = Math.round(pred.data[t]*255); // red color
myImageData.data[pixelIndex + 1] = Math.round(pred.data[t]*255); // green color
myImageData.data[pixelIndex + 2] = Math.round(pred.data[t]*255); // blue color
myImageData.data[pixelIndex + 3] = 255;
}
ctx.putImageData(myImageData, 10, 10);
console.log("myImageData", myImageData)
});
</script>
</body>
</html>
Needed to transpose the image data array before passing it to the model.
<!DOCTYPE html>
<html>
<header>
<title>ONNX Runtime JavaScript examples: Quick Start - Web (using script tag)</title>
<input id="image-selector" type="file" style="top:10px;left:10px" >
<button id="predict-button" class="btn btn-dark float-right" style="top:10px;left:70px" >Predict</button>
<img id="selected-image" src="" />
<canvas id="canvas" width =320px height=320px ></canvas>
</header>
<body>
<!-- import ONNXRuntime Web from CDN -->
<script src="https://code.jquery.com/jquery-3.3.1.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/onnxruntime-web/dist/ort.min.js"></script>
<script>
$("#image-selector").change(function () {
let reader = new FileReader();
reader.onload = function () {
let dataURL = reader.result;
$("#selected-image").attr("src", dataURL);
}
let file = $("#image-selector").prop("files")[0];
reader.readAsDataURL(file);
});
// async function main() {
$("#predict-button").click(async function (){
const canvas = document.getElementById("canvas");
const ctx = canvas.getContext("2d");
const session = await ort.InferenceSession.create('./u2netp.onnx').then(console.log("model loaded"));
// input_name = getInputs();
const inputNames = session.inputNames;
const outputNames = session.outputNames;
let image = $("#selected-image").get(0);
var oc = document.createElement('canvas'),
octx = oc.getContext('2d');
oc.width = 320;
oc.height = 320;
octx.drawImage(image, 0, 0, oc.width, oc.height);
var input_imageData = octx.getImageData(0, 0, 320, 320);
console.log("input_imageData",input_imageData.data)
var floatArr = new Float32Array(320 * 320 * 3)
var floatArr1 = new Float32Array(320 * 320 * 3)
var floatArr2 = new Float32Array(320 * 320 * 3)
var j = 0
for (let i = 1; i < input_imageData.data.length+1; i ++) {
if(i % 4 != 0){
floatArr[j] = (input_imageData.data[i-1].toFixed(2))/255; // red color
j = j + 1;
}
}
console.log("floatArr", floatArr)
for (let i = 1; i < floatArr.length+1; i += 3) {
floatArr1[i-1] = (floatArr[i-1] - 0.485)/0.229 // red color
floatArr1[i] = (floatArr[i] - 0.456)/0.224 // green color
floatArr1[i+1] = (floatArr[i+1] - 0.406)/0.225 // blue color
}
var k = 0
for (let i = 0; i < floatArr.length; i += 3) {
floatArr2[k] = floatArr[i] // red color
k = k + 1
}
console.log("k", k)
var l = 102400
for (let i = 1; i < floatArr.length; i += 3) {
floatArr2[l] = floatArr[i] // red color
l = l + 1
}
console.log("l", l)
var m = 204800
for (let i = 2; i < floatArr.length; i += 3) {
floatArr2[m] = floatArr[i] // red color
m = m + 1
}
const input = new ort.Tensor('float32', floatArr2, [1, 3, 320, 320])
a = inputNames[0]
const feeds = {"input.1": input};
const results = await session.run(feeds).then();
const pred = Object.values(results)[0]
var myImageData = ctx.createImageData(320, 320);
for (let i = 0; i < pred.data.length*4; i += 4) {
var pixelIndex = i;
if(i != 0){
t = i/4;
}
else{
t = 0;
}
myImageData.data[pixelIndex ] = Math.round(pred.data[t]*255); // red color
myImageData.data[pixelIndex + 1] = Math.round(pred.data[t]*255); // green color
myImageData.data[pixelIndex + 2] = Math.round(pred.data[t]*255); // blue color
myImageData.data[pixelIndex + 3] = 255;
}
ctx.putImageData(myImageData, 10, 10);
});
</script>
</body>
</html>
Related
I am building a website which makes a random list of numbers and quick sorts it with a visualization. When I run the function in the script section of my HTML file, it does not work correctly, however it works fine when I run it as a js file. Is there any reason for this? The quick sort function is at the bottom.
<!DOCTYPE html>
<html>
<H1>
Representation
</H1>
<h2></h2>
<body>
<button id='Quick Sort' onclick="quicksort()">Quick Sort</button>
<canvas id="myCanvas" width="1400" height="650" style="border:1px solid #2a1baf;"></canvas>
<input type="range" min="4" max="1000" value="1" class="slider" id="slider">
<p>Amount of Rectangles: <span id="choice"></span></p>
<script>
var c = document.getElementById("myCanvas");
var ctx = c.getContext("2d");
var setup = true;
var slider = document.getElementById("slider");
var slideText = document.getElementById("choice");
var size = 4;
slideText.innerHTML = size;
if (setup) {
var arr = reset(size);
setup = false;
}
var button = document.getElementById('Quick Sort');
slider.oninput = function() {
slideText.innerHTML = this.value;
size = this.value;
ctx.clearRect(0, 0, c.width, c.height);
arr = reset(size);
}
function reset(size) {
nums = [];
ar = [];
for (i = 1; i <= size; i++) {
nums.push(i);
}
while (nums.length > 0) {
const random = Math.floor(Math.random() * nums.length);
ar.push(nums[random]);
nums.splice(random, 1);
}
ctx.beginPath();
for (var i = 0; i < size; i++) {
ctx.rect(i * (1400 / size), 0, 1400 / size, 600 / (size) * ar[i]);
}
ctx.stroke();
return ar;
}
function bin_search(arr, val, start, end) {
if (start == end) {
if (arr[start] > val) {
return start;
}
else {
return start + 1;
}
}
if (start > end) {
return start;
}
var mid = Math.floor((start + end) / 2);
if (arr[mid] < val) {
return bin_search(arr, val, mid + 1, end);
}
else if (arr[mid] > val) {
return bin_search(arr, val, start, mid - 1);
}
else {
return mid;
}
}
function quicksort() {
for (var i = 1; i < arr.length; i++) {
val = arr[i];
j = bin_search(arr, val, 0, i - 1);
slice2 = arr.slice(j, i);
slice3 = arr.slice(i + 1);
arr = arr.slice(0, j);
arr.push(val);
arr = arr.concat(slice2);
arr = arr.concat(slice3);
ctx.beginPath();
for (var i = 0; i < size; i++) {
ctx.rect(i * (1400 / size), 0, 1400 / size, 600 / (size) * ar[i]);
}
ctx.stroke();
}
}
</script>
</body>
</html>
Your problem is due to var declaration - using let for the for loops would fix your issue
like:
function quicksort() {
for (let i = 1; i < arr.length; i++) {
val = arr[i];
j = bin_search(arr, val, 0, i - 1);
slice2 = arr.slice(j, i);
slice3 = arr.slice(i + 1);
arr = arr.slice(0, j);
arr.push(val);
arr = arr.concat(slice2);
arr = arr.concat(slice3);
ctx.beginPath();
for (let i = 0; i < size; i++) {
ctx.rect(i * (1400 / size), 0, 1400 / size, 600 / (size) * ar[i]);
}
ctx.stroke();
}
}
I will not comment that many variables are missing their declaration
I am learning to use neural networks, and have encountered a problem.
I can not figure out how to convert data for a neural network.
As I understand it, I need to normalize the data, after normalization and learning, the answer is always averaged.
https://jsfiddle.net/eoy7krzj/
<html>
<head>
<script src="https://cdn.rawgit.com/BrainJS/brain.js/5797b875/browser.js"></script>
</head>
<body>
<div>
<button onclick="train()">train</button><button onclick="Generate.next(); Generate.draw();">generate</button><button onclick="calculate()">calculate</button>
</div>
<canvas id="generate" style="border: 1px solid #000"></canvas>
</body>
<script type="text/javascript">
var trainData = [];
function randomInteger(min, max) {
var rand = min - 0.5 + Math.random() * (max - min + 1)
//rand = Math.round(rand);
return rand;
}
function getRandomColor() {
var letters = '0123456789ABCDEF';
var color = '#';
for (var i = 0; i < 6; i++) {
color += letters[Math.floor(Math.random() * 16)];
}
return color;
}
var Generate = new function(){
var canvas = document.getElementById('generate');
var ctx = canvas.getContext('2d');
var elem = {
input: [],
output: []
}
var size = {
width: 240,
height: 140
}
canvas.width = 500;
canvas.height = 250;
this.next = function(){
this.build();
trainData.push({
input: elem.input,
output: elem.output
});
}
this.clear = function(){
ctx.clearRect(0, 0, canvas.width, canvas.height);
}
this.draw = function(){
this.clear();
this.item(elem.input, function(item){
ctx.strokeStyle = "green";
ctx.strokeRect(item[0], item[1], item[2], item[3]);
})
this.item(elem.output, function(item){
ctx.strokeStyle = "blue";
ctx.strokeRect(item[0], item[1], item[2], item[3]);
})
}
this.item = function(where, call){
for (var i = 0; i < where.length; i+=4) {
var input = [
where[i],
where[i+1],
where[i+2],
where[i+3],
];
this.denormalize(input);
call(input)
}
}
this.normalize = function(input){
input[0] = input[0] / 500;
input[1] = input[1] / 250;
input[2] = input[2] / 500;
input[3] = input[3] / 250;
}
this.denormalize = function(input){
input[0] = input[0] * 500;
input[1] = input[1] * 250;
input[2] = input[2] * 500;
input[3] = input[3] * 250;
}
this.empty = function(add){
var data = [];
for (var i = 0; i < add; i++) {
data = data.concat([0,0,0,0]);
}
return data;
}
this.build = function(){
var output = [];
var input = [];
size.width = randomInteger(100,500);
size.height = randomInteger(50,250);
var lines = 1;//Math.round(size.height / 100);
var line_size = 0;
var line_offset = 0;
for(var i = 0; i < lines; i++){
line_size = randomInteger(30,Math.round(size.height / lines));
var columns = Math.round(randomInteger(1,3));
var columns_width = 0;
var columns_offset = 0;
for(var c = 0; c < columns; c++){
columns_width = randomInteger(30,Math.round(size.width / columns));
var item = [
columns_offset + 10,
line_offset + 10,
columns_width - 20,
line_size - 20
];
this.normalize(item);
input = input.concat(item);
columns_offset += columns_width;
}
var box = [
0,
line_offset,
columns_offset,
line_size
]
this.normalize(box);
output = output.concat(box);
line_offset += line_size + 10;
}
elem.input = input.concat(this.empty(5 - Math.round(input.length / 4)));
elem.output = output.concat(this.empty(2 - Math.round(output.length / 4)));
}
this.get = function(){
return elem.input;
}
this.calculate = function(result, stat){
console.log('brain:',result);
this.item(result, function(item){
ctx.strokeStyle = "red";
ctx.strokeRect(item[0], item[1], item[2], item[3]);
})
}
this.train = function(){
for(var i = 0; i < 40; i++){
this.next();
}
}
}
Generate.train();
Generate.log = true;
var net,stat;
function train(){
net = new brain.NeuralNetwork({ hiddenLayers: [8,4]});
stat = net.train(trainData,{log: true, iterations: 250,learningRate: 0.01,errorThresh:0.05});
console.log('stat:',stat)
}
function calculate(){
Generate.calculate(net.run(Generate.get()))
}
</script>
</html>
My goal is to train the network to find the elements and show their sizes.
Procedure:
Click to train
Click generate
Click to calculate
As a result, the network shows average results and red indicates that the network has found.
Maybe not so I transfer the data, can on another it is necessary?
I have been trying to make an animated bubble sort in JS but every time I run the code it goes from a random order to sorted instantaneously. How do I make it redraw and stall for a set amount of time after every time I swap two things in the array of colour codes.
<html>
<body>
<canvas id="myCanvas" width="300" height="700" style="border:1px solid #d3d3d3;">
</canvas>
<script>
function swap(nums, pos1,pos2) {
var a = nums[pos1];
nums[pos1] = nums[pos2];
nums[pos2] = a;
}
function shuffle(nums) {
for (var i = 0; i < 1000; i++) {
var num1 = Math.floor((Math.random())*26);
var num2 = Math.floor((Math.random())*26);
swap(nums,num1,num2);
}
}
function sleep(seconds)
{
var e = new Date().getTime() + (seconds * 1000);
while (new Date().getTime() <= e) {}
}
function drawIt(nums) {
var c = document.getElementById("myCanvas");
var ctx = c.getContext("2d");
for (var i = 0; i < nums.length; i++) {
var c = document.getElementById("myCanvas");
var ctx = c.getContext("2d");
ctx.beginPath();
ctx.rect(10, i * 20 + 10, 50, 20);
ctx.stroke();
ctx.fillStyle = "rgba(" + nums[i] + ", " + nums[i] + ", " + nums[i] + ", 1)";
ctx.fill();
}
}
var nums = new Array();
for (var i = 0; i<=250; i+=10) {
nums[i/10] = i;
}
shuffle(nums);
drawIt(nums);
for (var i = 0; i < nums.length-1; i++) {
for (var j = i+1; j < nums.length; j++) {
if (nums[i] > nums[j]) {
swap(nums, i,j);
drawIt(nums);
}
}
}
</script>
</body>
</html>
I have tried to use setTimeout() but nothing worked.
EDIT:
Is this what you meant because this still isn't working.
<html>
<body>
<canvas id="myCanvas" width="300" height="800" style="border:1px solid #d3d3d3;">
</canvas>
<script>
function swap(nums, pos1,pos2) {
var a = nums[pos1];
nums[pos1] = nums[pos2];
nums[pos2] = a;
}
function shuffle(nums) {
for (var i = 0; i < 1000; i++) {
var num1 = Math.floor((Math.random())*26);
var num2 = Math.floor((Math.random())*26);
swap(nums,num1,num2);
}
}
var counter1 = 0;
var counter2 = 1;
function drawIt(nums) {
if (nums[counter1] > nums[counter2]) {
swap(nums, counter1,counter2);
}
var c = document.getElementById("myCanvas");
var ctx = c.getContext("2d");
for (var i = 0; i < nums.length; i++) {
var c = document.getElementById("myCanvas");
var ctx = c.getContext("2d");
ctx.beginPath();
ctx.lineWidth = "1";
ctx.strokeStyle = "rgba(" + nums[i] + ", " + nums[i] + ", " + nums[i] + ", 1)";
ctx.rect(10, i * 15 + 10, 50, 15);
ctx.stroke();
ctx.beginPath();
ctx.rect(10, i * 15 + 10, 50, 15);
ctx.stroke();
ctx.fillStyle = "rgba(" + nums[i] + ", " + nums[i] + ", " + nums[i] + ", 1)";
ctx.fill();
}
if (counter2 < nums.length) {
counter2++;
}
else {
counter1++;
counter2 = counter1+1;
}
}
var nums = new Array();
for (var i = 0; i<=250; i+=5) {
nums[i/5] = i;
}
shuffle(nums);
drawIt(nums);
for (var i = 0; i < (nums.length*(nums.length-1)) / 2; i++) {
setTimeout(drawIt(nums), 2000);
}
</script>
</body>
</html>
Unfortunately there is no way to force a draw, or DOM update, while your thread is still running. The way to solve this is to make your sort algorithm re entrant, so that it can be called from a timer, or via setTimeout. Each time the timer is triggered, you perform one iteration of the sort, and then draw the result.
This is a bit annoying sometimes when studying algorithms by visualizing them, as step visualization cannot be achieved without modifying the algorithm.
EDIT: Added working code snippet that implements a "reentrant" sort.
<html>
<body>
<canvas id="myCanvas" width="300" height="700" style="border:1px solid #d3d3d3;">
</canvas>
<script>
function swap(nums, pos1,pos2) {
var a = nums[pos1];
nums[pos1] = nums[pos2];
nums[pos2] = a;
}
function shuffle(nums) {
for (var i = 0; i < 1000; i++) {
var num1 = Math.floor((Math.random())*26);
var num2 = Math.floor((Math.random())*26);
swap(nums,num1,num2);
}
}
function sleep(seconds)
{
var e = new Date().getTime() + (seconds * 1000);
while (new Date().getTime() <= e) {}
}
function drawIt(nums) {
var c = document.getElementById("myCanvas");
var ctx = c.getContext("2d");
for (var i = 0; i < nums.length; i++) {
var c = document.getElementById("myCanvas");
var ctx = c.getContext("2d");
ctx.beginPath();
ctx.rect(10, i * 20 + 10, 50, 20);
ctx.stroke();
ctx.fillStyle = "rgba(" + nums[i] + ", " + nums[i] + ", " + nums[i] + ", 1)";
ctx.fill();
}
}
var nums = new Array();
for (var i = 0; i<=250; i+=10) {
nums[i/10] = i;
}
shuffle(nums);
function sort()
{
var i=0;
var j=0;
var sortComplete=false;
var reentrantSort = function()
{
if (nums[i] > nums[j]){
swap(nums, i, j);
drawIt(nums)
}
j++;
if (j == nums.length){
j=0;i++;
if (i == nums.length ) {
i=0;j=0;
sortComplete=true; //Sorting is done
}
}
if (!sortComplete){ // If sort still not complete
setTimeout(reentrantSort,40); // Run next iteration in 40 ms.
}
};
reentrantSort();
}
sort();
</script>
</body>
</html>
Ideally you want to separate concerns - i.e. sorting an array and rendering it's state at a given point are different problems - you could achieve this with an ES2015 generator. In the following example I've taken a generic bubble sort function and added the function* signature and a yield statement - without modifying the algorithm this effectively allows you to step through each iteration in sequence.
function* bubbleSequence(arr){
for(let i = 0; i < arr.length; ++i)
for(let j = 0, end = arr.length - i; j < end; ++j)
if(arr[j] > arr[j+1]){
let temp = arr[j];
arr[j] = arr[j+1];
arr[j+1] = temp;
yield j+1; // <--- pauses execution
}
}
function draw(arr, swapIdx){
let ctx = document.getElementById('my-canvas').getContext('2d');
arr.forEach((n, i) => {
ctx.beginPath();
ctx.rect(i * 11 + 10, 10, 10, 40);
ctx.fillStyle = `rgba(${n},${n},${n},1)`;
ctx.fill();
ctx.beginPath();
ctx.moveTo(i * 11 + 10, 53);
ctx.lineTo(i * 11 + 20, 53);
ctx.lineWidth = 2;
ctx.strokeStyle = i === swapIdx || (i+1) === swapIdx ? '#f00' : '#333';
ctx.stroke();
});
}
var nums = [...Array(51)].map((_, i) => i * 5)
.sort(_ => (Math.random() * 3 | 0) - 1);
var sorter = bubbleSequence(nums);
var interval = setInterval(_ => {
let next = sorter.next();
if(next.done) clearInterval(interval);
window.requestAnimationFrame(_ => draw(nums, next.value));
}, 80);
<canvas id="my-canvas" width=600 height=180></canvas>
The code below creates up the elements for a grid, and arranges it using the transformOut method. This part works fine, but I then want the grid to collapse in to the centre on mousedown, and spring back out again on mouseup. However, all subsequent calls to either the transformIn or transformOut function result in the items going in and out. A working example is here:
http://codepen.io/timsig/pen/JdXYwE
Code as follows, thanks for any help.
define('main', function (require, exports, module) {
var Engine = require('famous/core/Engine');
var View = require('famous/core/View');
var Surface = require('famous/core/Surface');
var Modifier = require('famous/core/Modifier');
var StateModifier = require('famous/modifiers/StateModifier');
var EventHandler = require('famous/core/EventHandler');
var PhysicsEngine = require('famous/physics/PhysicsEngine');
var Transitionable = require('famous/transitions/Transitionable');
var SpringTransition= require('famous/transitions/SpringTransition');
var Particle = require('famous/physics/bodies/Particle');
var Drag = require('famous/physics/forces/Drag');
var RepulsionForce = require('famous/physics/forces/Repulsion');
var Wall = require('famous/physics/constraints/Wall');
var Random = require('famous/math/Random');
var Transform = require('famous/core/Transform');
Transitionable.registerMethod('spring', SpringTransition);
var context = Engine.createContext();
var cols = 5;
var rows = 5;
var gridSize = Math.min(window.innerWidth, window.innerHeight) / 1.5;
var itemSize = gridSize / (cols + 1);
var gridItems = [];
var transformOutArray = [itemSize / 2 - gridSize / 2,
(itemSize / 2 - gridSize / 2) / 2,
0,
(gridSize / 2 - itemSize / 2) / 2,
gridSize / 2 - itemSize / 2];
var transformInArray = Array.prototype.slice.call(transformOutArray);
transformInArray.reverse();
var cameraView = new View();
var camera = new Modifier({
origin: [0.5, 0.5],
align: [0.5, 0.5]
});
context.add(camera).add(cameraView);
function createGridItems(){
for (var r = 0; r < rows; r += 1){
for (var c = 0; c < cols; c += 1){
var gridItem = new Surface({
size: [itemSize, itemSize],
properties:{
backgroundColor: '#aa62bb'
},
content: r + "," + c
});
gridItem.mod = new StateModifier({
origin: [0.5, 0.5],
align: [0.5, 0.5],
transform: Transform.identity
});
gridItem.idx = gridItems.length;
gridItem.transformOutrs = transformOutArray[r];
gridItem.transformOutcs = transformOutArray[c];
gridItem.transformInrs = transformInArray[r];
gridItem.transformIncs = transformInArray[c];
gridItems.push(gridItem);
cameraView.add(gridItem.mod).add(gridItem);
}
}
}
function transformOut(){
console.log('transform out')
for (var i = 0; i < gridItems.length; i+=1){
var index = i;
var gridItem = gridItems[index];
var tran = Transform.translate(gridItem.transformOutrs, gridItem.transformOutcs);
gridItem.mod.setTransform(tran, {
method: 'spring',
dampingRatio: 0.5,
period: 600
});
}
}
function transformIn(){
console.log('transform in');
for (var j = 0; j < gridItems.length; j+=1){
var index = j;
var gridItem = gridItems[index];
var tran = Transform.translate(gridItem.transformInrs, gridItem.transformIncs);
gridItem.mod.setTransform(tran, {
method: 'spring',
dampingRatio: 0.5,
period: 600
});
}
}
createGridItems();
console.log (transformOutArray);
console.log (transformInArray);
transformOut();
Engine.on('mousedown', transformIn);
Engine.on('mouseup', transformOut);
});
Setting the Transform back to the identity will return items back to their default on transform. Currently they are transitioning back to original once the new transform is set, then apply the new transform. You are only reversing their order with the new transforms.
function transformIn(){
console.log('transform in');
for (var j = 0; j < gridItems.length; j+=1){
var index = j;
var gridItem = gridItems[index];
gridItem.mod.setTransform(Transform.identity, {
method: 'spring',
dampingRatio: 0.5,
period: 600
});
}
}
Example snippet:
define('main', function (require, exports, module) {
var Engine = require('famous/core/Engine');
var View = require('famous/core/View');
var Surface = require('famous/core/Surface');
var Modifier = require('famous/core/Modifier');
var StateModifier = require('famous/modifiers/StateModifier');
var EventHandler = require('famous/core/EventHandler');
var PhysicsEngine = require('famous/physics/PhysicsEngine');
var Transitionable = require('famous/transitions/Transitionable');
var SpringTransition= require('famous/transitions/SpringTransition');
var Particle = require('famous/physics/bodies/Particle');
var Drag = require('famous/physics/forces/Drag');
var RepulsionForce = require('famous/physics/forces/Repulsion');
var Wall = require('famous/physics/constraints/Wall');
var Random = require('famous/math/Random');
var Transform = require('famous/core/Transform');
Transitionable.registerMethod('spring', SpringTransition);
var context = Engine.createContext();
var cols = 5;
var rows = 5;
var gridSize = Math.min(window.innerWidth, window.innerHeight) / 1.5;
var itemSize = gridSize / (cols + 1);
var gridItems = [];
var transformOutArray = [itemSize / 2 - gridSize / 2,
(itemSize / 2 - gridSize / 2) / 2,
0,
(gridSize / 2 - itemSize / 2) / 2,
gridSize / 2 - itemSize / 2];
var transformInArray = Array.prototype.slice.call(transformOutArray);
transformInArray.reverse();
var cameraView = new View();
var camera = new Modifier({
origin: [0.5, 0.5],
align: [0.5, 0.5]
});
context.add(camera).add(cameraView);
function createGridItems(){
for (var r = 0; r < rows; r += 1){
for (var c = 0; c < cols; c += 1){
var gridItem = new Surface({
size: [itemSize, itemSize],
properties:{
backgroundColor: '#aa62bb'
},
content: r + "," + c
});
gridItem.mod = new StateModifier({
origin: [0.5, 0.5],
align: [0.5, 0.5],
transform: Transform.identity
});
gridItem.idx = gridItems.length;
gridItem.transformOutrs = transformOutArray[r];
gridItem.transformOutcs = transformOutArray[c];
gridItem.transformInrs = transformInArray[r];
gridItem.transformIncs = transformInArray[c];
gridItems.push(gridItem);
cameraView.add(gridItem.mod).add(gridItem);
}
}
}
function transformOut(){
console.log('transform out');
for (var i = 0; i < gridItems.length; i+=1){
var index = i;
var gridItem = gridItems[index];
var tran = Transform.translate(gridItem.transformOutrs, gridItem.transformOutcs);
gridItem.mod.setTransform(tran, {
method: 'spring',
dampingRatio: 0.5,
period: 600
});
}
}
function transformIn(){
console.log('transform in');
for (var j = 0; j < gridItems.length; j+=1){
var index = j;
var gridItem = gridItems[index];
gridItem.mod.setTransform(Transform.identity, {
method: 'spring',
dampingRatio: 0.5,
period: 600
});
}
}
createGridItems();
console.log (transformOutArray);
console.log (transformInArray);
transformOut();
Engine.on('mousedown', transformIn);
Engine.on('mouseup', transformOut);
});
require(['main']);
<script src="http://requirejs.org/docs/release/2.1.16/minified/require.js"></script>
<script src="http://code.famo.us/lib/requestAnimationFrame.js"></script>
<script src="http://code.famo.us/lib/classList.js"></script>
<script src="http://code.famo.us/lib/functionPrototypeBind.js"></script>
<link rel="stylesheet" type="text/css" href="http://code.famo.us/famous/0.3.5/famous.css" />
<script src="http://code.famo.us/famous/0.3.5/famous.min.js"></script>
NOTE: setTransform is now deprecated in Famo.us
Deprecated: Prefer transformFrom with static Transform, or use a TransitionableTransform.
I'm a university masters degree student in Computer Graphics, I'm having difficulty using three.js to access the image data(pixels) of a texture created with a EffectComposer.
The first composer (composer) is using a line detection shader to find road lines in a lane, and put the result in a renderTarget (rt_Binary). My second composer (fcomposer2) uses a shader that paints an area green if is within a certain space.
The plan was to render the composer first and after analysing the rt_Binary image i could determine the limits.
I found some functions that allow me to get the imagedata (getImageData(image) and getPixel(imagedata, x, y)) but they only work on these occasions:
// before image
var imagedata = getImageData(videoTexture.image);
// processed image
var imagedata2 = getImageData(renderer.domElement);
If a put the first composer to render to screen, i get the correct values for the limits, but when i put the second composer, i get the wrong values for the limits.
Is there any way to get the imageData from a renderTarget? is so, how?
Edit1:
Here's the code for script that i use for the html:
<html xmlns="http://www.w3.org/1999/xhtml"><head>
<title>Tests WebGL</title>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
<script src="http://ajax.googleapis.com/ajax/libs/jquery/1.9.1/jquery.min.js"></script>
<script src="three.js/build/three.js"></script>
<script src="js/CopyShader.js"></script>
<script src="js/EffectComposer.js"></script>
<script src="js/MaskPass.js" ></script>
<script src="js/RenderPass.js" ></script>
<script src="js/ShaderPass.js"></script>
<script src="js/stats.min.js" ></script>
<!-- Shaders -->
<script src="js/shaders/KernelShader.js" ></script>
<script src="js/shaders/SimpleShader.js"></script>
<script src="js/shaders/MyShader.js"></script>
<script src="js/shaders/BinaryShader.js"></script>
<script type="text/javascript">
var scene, fscene, sceneF;
var camera;
var renderer, rt_Binary;
var composer;
var stats;
var fmaterial;
var videoTexture;
var videoWidth = 480;
var videoHeight = 270;
var rendererWidth = videoWidth;
var rendererHeight = videoHeight;
var x_max = 345;//videoWidth*0.72; //
var x_min = 120;//videoWidth*0.25; //
var y_max = 189;//videoHeight*0.7 ;
var y_min = 148;//videoHeight*0.55;
// var ml=0.0, mr=0.0, mm=0.0;
// var bl=0.0, br=0.0, bm=0.0;
var yMaxL = 0, yMinL = 0, yMaxR = 0, yMinR = 0;
var xMaxL = 0, xMinL = 0, xMaxR = 0, xMinR = 0;
var frame = 0;
// init the scene
window.onload = function() {
renderer = new THREE.WebGLRenderer(
{
antialias: true, // to get smoother output
preserveDrawingBuffer: true // to allow screenshot
});
renderer.setClearColor(0xffffff, 1);
renderer.autoClear = false;
renderer.setSize(rendererWidth, rendererHeight);
document.getElementById('container').appendChild(renderer.domElement);
//add stats
stats = new Stats();
stats.domElement.style.position = 'absolute';
stats.domElement.style.top = '0px';
document.getElementById('container').appendChild(stats.domElement);
// create Main scene
scene = new THREE.Scene();
camera = new THREE.PerspectiveCamera(35, rendererWidth / rendererHeight, 1, 10000);
camera.position.set(0, 1, 6);
scene.add(camera);
// define video element
video = document.createElement('video');
// video.src = 'GOPR0007.webm';
video.src = 'output.webm';
video.width = videoWidth;
video.height = videoHeight;
video.autoplay = true;
video.loop = true;
//create 3d object and apply video texture to it
var videoMesh = new THREE.Object3D();
scene.add(videoMesh);
videoTexture = new THREE.Texture(video);
var geom = new THREE.PlaneGeometry(1, 1);
material = new THREE.MeshBasicMaterial({map: videoTexture});
var mesh = new THREE.Mesh(geom, material);
videoMesh.add(mesh);
var renderTargetParameters = { minFilter: THREE.LinearFilter, magFilter: THREE.LinearFilter, format: THREE.RGBFormat, stencilBufer: false };
rt_Binary = new THREE.WebGLRenderTarget( videoWidth, videoHeight, renderTargetParameters );
// Composers
// composer = new THREE.EffectComposer(renderer, renderTarget2);
composer = new THREE.EffectComposer(renderer, rt_Binary );
composer.addPass(new THREE.RenderPass(scene, camera));
var simple = new SimpleShader.Class(videoWidth, videoHeight);
var simEffect = new THREE.ShaderPass(simple.shader);
composer.addPass(simEffect);
var ef = new BinaryShader.Class(videoWidth, videoHeight, 1.1, [-2,-2,-2,0,0,0,2,2,2]);
var effect = new THREE.ShaderPass(ef.shader);
composer.addPass(effect);
var copyPass = new THREE.ShaderPass(THREE.CopyShader);
// copyPass.renderToScreen = true;
composer.addPass(copyPass);
//New scene
sceneF = new THREE.Scene();
sceneF.add(camera);
var videoMesh2 = new THREE.Object3D();
sceneF.add(videoMesh2);
var geomF = new THREE.PlaneGeometry(1, 1);
var materialF = new THREE.MeshBasicMaterial({map: videoTexture});
var meshF = new THREE.Mesh(geomF, materialF);
sceneF.add(meshF);
fcomposer2 = new THREE.EffectComposer(renderer );
fcomposer2.addPass(new THREE.RenderPass(sceneF, camera));
fcomposer2.addPass(simEffect);
var ef1 = new MyShader.Class(videoWidth, videoHeight, [yMaxL,yMinL,xMaxL,xMinL,yMaxR,yMinR,xMaxR,xMinR], videoTexture);
var effect1 = new THREE.ShaderPass(ef1.shader);
fcomposer2.addPass(effect1);
var copyPass2 = new THREE.ShaderPass(THREE.CopyShader);
copyPass2.renderToScreen = true;
fcomposer2.addPass(copyPass2);
animate();
}
// animation loop
function animate() {
// loop on request animation loop
// - it has to be at the begining of the function
requestAnimationFrame(animate);
// do the render
render();
stats.update();
if ((frame % 50) == 0) {
console.log("frame ", frame, " ");
console.log("yMaxL: ", yMaxL, " ");
console.log("yMinL: ", yMinL, " ");
console.log("xMaxL: ", xMaxL, " ");
console.log("xMinL: ", xMinL, " ");
console.log("yMaxR: ", yMaxR, " ");
console.log("yMinR: ", yMinR, " ");
console.log("xMaxR: ", xMaxR, " ");
console.log("xMinR: ", xMinR, " ");
manipulatePixels();
}
frame = frame + 1;
yMaxL = 0, yMinL = 0, yMaxR = 0, yMinR = 0;
xMaxL = 0, xMinL = 0, xMaxR = 0, xMinR = 0;
}
// render the scene
function render() {
if (video.readyState === video.HAVE_ENOUGH_DATA) {
videoTexture.needsUpdate = true;
}
// actually render the scene
renderer.clear();
composer.render();
var left_x = new Array();
var left_y = new Array();
var l = 0;
var right_x = new Array();
var right_y = new Array();
var r = 0;
if (frame == 200) {
var imagedata2 = getImageData(renderer.domElement);
var middle = imagedata2.width / 2;
for (var x=x_min; x < x_max; x=x+1) {
for (var y=y_min; y < y_max; y=y+1) {
var pixel = getPixel(imagedata2, x, y);
if (pixel.g > 0)
{
//console.log(pixel);
if (x < middle) {
left_x[l] = x;
left_y[l] = y;
l++;
}
else {
right_x[r] = x;
right_y[r] = y;
r++;
}
}
}
}
lineEquation(left_x, left_y, right_x, right_y);
}
fcomposer2.render();
}
function lineEquation(left_x,left_y,right_x,right_y) {
var newYMAX = left_y[0];
var newYMIN = left_y[0];
var maximosL = new Array();
var minimosL = new Array();
//left
for (var i=1; i < left_y.length; i++) {
if (left_y[i]>newYMAX) newYMAX = left_y[i];
else {
if (left_y[i]<newYMIN) newYMIN = left_y[i];
}
}
yMaxL = newYMAX;
yMinL = newYMIN;
// yMaxL = ymaxL/videoHeight;
// yMinL = yminL/videoHeight;
var pmin=0, pmax=0;
for (var i=0; i < left_y.length; i++) {
if (left_y[i] === newYMAX) {
// console.log(left_y[i]);
// console.log(left_x[i]);
maximosL[pmax] = left_x[i];
pmax++;
}
}
for (var j=0; j < left_y.length; j++) {
if (left_y[j] === newYMIN) {
// console.log(left_y[j]);
// console.log(left_x[j]);
minimosL[pmin] = left_x[j];
pmin++;
}
}
// console.log(maximosL);
// console.log(minimosL);
var sumMAX = 0, sumMIN = 0;
for (var i=0; i< maximosL.length; i++) {
sumMAX = sumMAX + maximosL[i];
}
for (var j=0; j< minimosL.length; j++) {
sumMIN = sumMIN + minimosL[j];
}
xMaxL = sumMAX/maximosL.length;
xMinL = sumMIN/minimosL.length;
// xMaxL /= videoWidth;
// xMinL /= videoWidth;
//right
var maximosR = new Array();
var minimosR = new Array();
newYMAX = right_y[0];
newYMIN = right_y[0];
pmin=0; pmax=0;
for (var i=0; i < right_y.length; i++) {
if (right_y[i]> newYMAX) newYMAX = right_y[i];
else {
if (right_y[i]< newYMIN) newYMIN = right_y[i];
}
}
yMaxR = newYMAX;
yMinR = newYMIN;
// yMaxR = ymaxR/videoHeight;
// yMinR = yminR/videoHeight;
for (var i=0; i < right_y.length; i++) {
if (right_y[i] === newYMAX)
{maximosR[pmax] = right_x[i]; pmax++;}
if (right_y[i] === newYMIN)
{minimosR[pmin] = right_x[i]; pmin++;}
}
// console.log(maximosR);
// console.log(minimosR);
xMaxR=0;
for (var i=0; i< maximosR.length; i++) {
xMaxR += maximosR[i];
}
xMinR=0;
for (var i=0; i< minimosR.length; i++) {
xMinR += minimosR[i];
}
// console.log(xMaxR);
// console.log(xMinR);
xMaxR /= maximosR.length;
xMinR /= minimosR.length;
// console.log(xMaxR);
// console.log(xMinR);
// xMinR /= videoWidth;
// xMaxR /= videoWidth;
}
function manipulatePixels() {
// imagem antes
var imagedata = getImageData(videoTexture.image);
// imagem processada
var imagedata2 = getImageData(renderer.domElement);
// console.log(getPixel(imagedata, 480 - 1, 270 - 1));
// console.log(getPixel(imagedata2, 480 - 1, 270 - 1));
}
function getImageData(image) {
var canvas = document.createElement('canvas');
canvas.width = image.width;
canvas.height = image.height;
var context = canvas.getContext('2d');
context.drawImage(image, 0, 0);
return context.getImageData(0, 0, image.width, image.height);
}
function getPixel(imagedata, x, y) {
var position = (x + imagedata.width * y) * 4, data = imagedata.data;
return {r: data[ position ], g: data[ position + 1 ], b: data[ position + 2 ], a: data[ position + 3 ]};
}
function findLineByLeastSquares(values_x, values_y) {
var sum_x = 0;
var sum_y = 0;
var sum_xy = 0;
var sum_xx = 0;
/*
* We'll use those variables for faster read/write access.
*/
var x = 0;
var y = 0;
var values_length = values_x.length;
if (values_length != values_y.length) {
throw new Error('The parameters values_x and values_y need to have same size!');
}
/*
* Nothing to do.
*/
if (values_length === 0) {
return [ [], [] ];
}
/*
* Calculate the sum for each of the parts necessary.
*/
for (var v = 0; v < values_length; v++) {
x = values_x[v];
y = values_y[v];
sum_x += x;
sum_y += y;
sum_xx += (x*x);
sum_xy += (x*y);
}
console.log (sum_x);
console.log(sum_y);
console.log(sum_xx);
console.log(sum_xy);
console.log(values_length);
/*
* Calculate m and b for the formular:
* y = x * m + b
*/
var m = (sum_x*sum_y - values_length*sum_xy) / (sum_x*sum_x - values_length*sum_xx);
var b = (sum_y - (m*sum_x))/values_length;
//console.log([m,b]);
return [m, b];
}
//resize method
/**window.addEventListener('resize', onWindowResize, false);
function onWindowResize() {
camera.aspect = window.innerWidth / window.innerHeight;
camera.updateProjectionMatrix();
renderer.setSize(window.innerWidth, window.innerHeight);
} */
</script>
Edit2 : Some images of what i'm trying to do: Image 1 shows the results from composer on the console, the limits i get from the lineEquation function are the correct ones for what i intend to do, but in Image 2 shows the results from fcomposer2 (fixed area) and on the console, the limits are the wrong ones.
![Image1]: http://prntscr.com/1ays73
![Image2]: http://prntscr.com/1ays0j
Edit3 :
By "access" i mean to be able to read the values of the pixels from the texture created by the binaryShader.
For example, in image1 the lines are painted in blue/green tone, I wanted to search the position of the pixels (x,y) in the image that the renderTarget would save. If i could find those pixels, i could adapt the green area in image2 to fit between the road lines.
This processing is need to make the green area overlap the current driving lane the user is currently on, if i can't get those points, i can't identify a lane.
I got it to work. Apparently i forgot to declare the fcomposer2 in the beginning of the script.
Thanks for the responses/comments, and sorry for the inconvenience.