Emulating palette based graphics in WebGL v.s. Canvas 2D - javascript
Currently, I'm using 2D canvas context to draw an image generated (from pixel to pixel, but refreshed as a whole buffer in once after a generated frame) from JavaScript at about a 25fps rate. The generated image is always one byte (integer / typed array) per pixel and a fixed palette is used to generate RGB final result. Scaling is also needed to adopt to the size of the canvas (ie: going to fullscreen) and/or at user request (zoom in/out buttons).
The 2D context of canvas is OK for this purpose, however I'm curious if WebGL can provide better result and/or better performance. Please note: I don't want to put pixels via webGL, I want to put pixels into my buffer (which is basically Uint8Array), and use that buffer (in once) to refresh the context. I don't know too much about WebGL, but using the needed generated image as some kind of texture would work somehow for example? Then I would need to refresh the texture at about 25fps rate, I guess.
It would be really fantastic, if WebGL support the colour space conversion somehow. With 2D context, I need to convert 1 byte / pixel buffer into RGBA for the imagedata in JavaScript for every pixel ... Scaling (for 2D context) is done now by altering the height/width style of the canvas, so browsers scales the image then. However I guess it can be slower than what WebGL can do with hw support, and also (I hope) WebGL can give greater flexibility to control the scaling, eg with the 2D context, browsers will do antialiasing even if I don't want to do (eg: integer zooming factor), and maybe that's a reason it can be quite slow sometimes.
I've already tried to learn several WebGL tutorials but all of them starts with objects, shapes, 3D cubes, etc, I don't need any - classical - object to render only what 2D context can do as well - in the hope that WebGL can be a faster solution for the very same task! Of course if there is no win here with WebGL at all, I would continue to use 2D context.
To be clear: this is some kind of computer hardware emulator done in JavaScript, and its output (what would be seen on a PAL TV connected to it) is rendered via a canvas context. The machine has fixed palette with 256 elements, internally it only needs one byte for a pixel to define its colour.
You can use a texture as your palette and a different texture as your image. You then get a value from the image texture and use it too look up a color from the palette texture.
The palette texture is 256x1 RGBA pixels. Your image texture is any size you want but just a single channel ALPHA texture. You can then look up a value from the image
float index = texture2D(u_image, v_texcoord).a * 255.0;
And use that value to look up a color in the palette
gl_FragColor = texture2D(u_palette, vec2((index + 0.5) / 256.0, 0.5));
Your shaders might be something like this
Vertex Shader
attribute vec4 a_position;
varying vec2 v_texcoord;
void main() {
gl_Position = a_position;
// assuming a unit quad for position we
// can just use that for texcoords. Flip Y though so we get the top at 0
v_texcoord = a_position.xy * vec2(0.5, -0.5) + 0.5;
}
Fragment shader
precision mediump float;
varying vec2 v_texcoord;
uniform sampler2D u_image;
uniform sampler2D u_palette;
void main() {
float index = texture2D(u_image, v_texcoord).a * 255.0;
gl_FragColor = texture2D(u_palette, vec2((index + 0.5) / 256.0, 0.5));
}
Then you just need a palette texture.
// Setup a palette.
var palette = new Uint8Array(256 * 4);
// I'm lazy so just setting 4 colors in palette
function setPalette(index, r, g, b, a) {
palette[index * 4 + 0] = r;
palette[index * 4 + 1] = g;
palette[index * 4 + 2] = b;
palette[index * 4 + 3] = a;
}
setPalette(1, 255, 0, 0, 255); // red
setPalette(2, 0, 255, 0, 255); // green
setPalette(3, 0, 0, 255, 255); // blue
// upload palette
...
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, 256, 1, 0, gl.RGBA,
gl.UNSIGNED_BYTE, palette);
And your image. It's an alpha only image so just 1 channel.
// Make image. Just going to make something 8x8
var image = new Uint8Array([
0,0,1,1,1,1,0,0,
0,1,0,0,0,0,1,0,
1,0,0,0,0,0,0,1,
1,0,2,0,0,2,0,1,
1,0,0,0,0,0,0,1,
1,0,3,3,3,3,0,1,
0,1,0,0,0,0,1,0,
0,0,1,1,1,1,0,0,
]);
// upload image
....
gl.texImage2D(gl.TEXTURE_2D, 0, gl.ALPHA, 8, 8, 0, gl.ALPHA,
gl.UNSIGNED_BYTE, image);
You also need to make sure both textures are using gl.NEAREST for filtering since one represents indices and the other a palette and filtering between values in those cases makes no sense.
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
Here's a working example:
var canvas = document.getElementById("c");
var gl = canvas.getContext("webgl");
// Note: createProgramFromScripts will call bindAttribLocation
// based on the index of the attibute names we pass to it.
var program = twgl.createProgramFromScripts(
gl,
["vshader", "fshader"],
["a_position", "a_textureIndex"]);
gl.useProgram(program);
var imageLoc = gl.getUniformLocation(program, "u_image");
var paletteLoc = gl.getUniformLocation(program, "u_palette");
// tell it to use texture units 0 and 1 for the image and palette
gl.uniform1i(imageLoc, 0);
gl.uniform1i(paletteLoc, 1);
// Setup a unit quad
var positions = [
1, 1,
-1, 1,
-1, -1,
1, 1,
-1, -1,
1, -1,
];
var vertBuffer = gl.createBuffer();
gl.bindBuffer(gl.ARRAY_BUFFER, vertBuffer);
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array(positions), gl.STATIC_DRAW);
gl.enableVertexAttribArray(0);
gl.vertexAttribPointer(0, 2, gl.FLOAT, false, 0, 0);
// Setup a palette.
var palette = new Uint8Array(256 * 4);
// I'm lazy so just setting 4 colors in palette
function setPalette(index, r, g, b, a) {
palette[index * 4 + 0] = r;
palette[index * 4 + 1] = g;
palette[index * 4 + 2] = b;
palette[index * 4 + 3] = a;
}
setPalette(1, 255, 0, 0, 255); // red
setPalette(2, 0, 255, 0, 255); // green
setPalette(3, 0, 0, 255, 255); // blue
// make palette texture and upload palette
gl.activeTexture(gl.TEXTURE1);
var paletteTex = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, paletteTex);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, 256, 1, 0, gl.RGBA, gl.UNSIGNED_BYTE, palette);
// Make image. Just going to make something 8x8
var image = new Uint8Array([
0,0,1,1,1,1,0,0,
0,1,0,0,0,0,1,0,
1,0,0,0,0,0,0,1,
1,0,2,0,0,2,0,1,
1,0,0,0,0,0,0,1,
1,0,3,3,3,3,0,1,
0,1,0,0,0,0,1,0,
0,0,1,1,1,1,0,0,
]);
// make image textures and upload image
gl.activeTexture(gl.TEXTURE0);
var imageTex = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, imageTex);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.ALPHA, 8, 8, 0, gl.ALPHA, gl.UNSIGNED_BYTE, image);
gl.drawArrays(gl.TRIANGLES, 0, positions.length / 2);
canvas { border: 1px solid black; }
<script src="https://twgljs.org/dist/twgl.min.js"></script>
<script id="vshader" type="whatever">
attribute vec4 a_position;
varying vec2 v_texcoord;
void main() {
gl_Position = a_position;
// assuming a unit quad for position we
// can just use that for texcoords. Flip Y though so we get the top at 0
v_texcoord = a_position.xy * vec2(0.5, -0.5) + 0.5;
}
</script>
<script id="fshader" type="whatever">
precision mediump float;
varying vec2 v_texcoord;
uniform sampler2D u_image;
uniform sampler2D u_palette;
void main() {
float index = texture2D(u_image, v_texcoord).a * 255.0;
gl_FragColor = texture2D(u_palette, vec2((index + 0.5) / 256.0, 0.5));
}
</script>
<canvas id="c" width="256" height="256"></canvas>
To animate just update the image and then re-upload it into the texture
gl.texImage2D(gl.TEXTURE_2D, 0, gl.ALPHA, 8, 8, 0, gl.ALPHA,
gl.UNSIGNED_BYTE, image);
Example:
var canvas = document.getElementById("c");
var gl = canvas.getContext("webgl");
// Note: createProgramFromScripts will call bindAttribLocation
// based on the index of the attibute names we pass to it.
var program = twgl.createProgramFromScripts(
gl,
["vshader", "fshader"],
["a_position", "a_textureIndex"]);
gl.useProgram(program);
var imageLoc = gl.getUniformLocation(program, "u_image");
var paletteLoc = gl.getUniformLocation(program, "u_palette");
// tell it to use texture units 0 and 1 for the image and palette
gl.uniform1i(imageLoc, 0);
gl.uniform1i(paletteLoc, 1);
// Setup a unit quad
var positions = [
1, 1,
-1, 1,
-1, -1,
1, 1,
-1, -1,
1, -1,
];
var vertBuffer = gl.createBuffer();
gl.bindBuffer(gl.ARRAY_BUFFER, vertBuffer);
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array(positions), gl.STATIC_DRAW);
gl.enableVertexAttribArray(0);
gl.vertexAttribPointer(0, 2, gl.FLOAT, false, 0, 0);
// Setup a palette.
var palette = new Uint8Array(256 * 4);
// I'm lazy so just setting 4 colors in palette
function setPalette(index, r, g, b, a) {
palette[index * 4 + 0] = r;
palette[index * 4 + 1] = g;
palette[index * 4 + 2] = b;
palette[index * 4 + 3] = a;
}
setPalette(1, 255, 0, 0, 255); // red
setPalette(2, 0, 255, 0, 255); // green
setPalette(3, 0, 0, 255, 255); // blue
// make palette texture and upload palette
gl.activeTexture(gl.TEXTURE1);
var paletteTex = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, paletteTex);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, 256, 1, 0, gl.RGBA, gl.UNSIGNED_BYTE, palette);
// Make image. Just going to make something 8x8
var width = 8;
var height = 8;
var image = new Uint8Array([
0,0,1,1,1,1,0,0,
0,1,0,0,0,0,1,0,
1,0,0,0,0,0,0,1,
1,0,2,0,0,2,0,1,
1,0,0,0,0,0,0,1,
1,0,3,3,3,3,0,1,
0,1,0,0,0,0,1,0,
0,0,1,1,1,1,0,0,
]);
// make image textures and upload image
gl.activeTexture(gl.TEXTURE0);
var imageTex = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, imageTex);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.ALPHA, width, height, 0, gl.ALPHA, gl.UNSIGNED_BYTE, image);
var frameCounter = 0;
function render() {
++frameCounter;
// skip 3 of 4 frames so the animation is not too fast
if ((frameCounter & 3) == 0) {
// rotate the image left
for (var y = 0; y < height; ++y) {
var temp = image[y * width];
for (var x = 0; x < width - 1; ++x) {
image[y * width + x] = image[y * width + x + 1];
}
image[y * width + width - 1] = temp;
}
// re-upload image
gl.activeTexture(gl.TEXTURE0);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.ALPHA, width, height, 0, gl.ALPHA,
gl.UNSIGNED_BYTE, image);
gl.drawArrays(gl.TRIANGLES, 0, positions.length / 2);
}
requestAnimationFrame(render);
}
render();
canvas { border: 1px solid black; }
<script src="https://twgljs.org/dist/twgl.min.js"></script>
<script id="vshader" type="whatever">
attribute vec4 a_position;
varying vec2 v_texcoord;
void main() {
gl_Position = a_position;
// assuming a unit quad for position we
// can just use that for texcoords. Flip Y though so we get the top at 0
v_texcoord = a_position.xy * vec2(0.5, -0.5) + 0.5;
}
</script>
<script id="fshader" type="whatever">
precision mediump float;
varying vec2 v_texcoord;
uniform sampler2D u_image;
uniform sampler2D u_palette;
void main() {
float index = texture2D(u_image, v_texcoord).a * 255.0;
gl_FragColor = texture2D(u_palette, vec2((index + 0.5) / 256.0, 0.5));
}
</script>
<canvas id="c" width="256" height="256"></canvas>
Of course that assumes your goal is to do the animation on the CPU by manipulating pixels. Otherwise you can use any normal webgl techniques to manipulate texture coordinates or whatever.
You can also update the palette similarly for palette animation. Just modify the palette and re-upload it
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, 256, 1, 0, gl.RGBA,
gl.UNSIGNED_BYTE, palette);
Example:
var canvas = document.getElementById("c");
var gl = canvas.getContext("webgl");
// Note: createProgramFromScripts will call bindAttribLocation
// based on the index of the attibute names we pass to it.
var program = twgl.createProgramFromScripts(
gl,
["vshader", "fshader"],
["a_position", "a_textureIndex"]);
gl.useProgram(program);
var imageLoc = gl.getUniformLocation(program, "u_image");
var paletteLoc = gl.getUniformLocation(program, "u_palette");
// tell it to use texture units 0 and 1 for the image and palette
gl.uniform1i(imageLoc, 0);
gl.uniform1i(paletteLoc, 1);
// Setup a unit quad
var positions = [
1, 1,
-1, 1,
-1, -1,
1, 1,
-1, -1,
1, -1,
];
var vertBuffer = gl.createBuffer();
gl.bindBuffer(gl.ARRAY_BUFFER, vertBuffer);
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array(positions), gl.STATIC_DRAW);
gl.enableVertexAttribArray(0);
gl.vertexAttribPointer(0, 2, gl.FLOAT, false, 0, 0);
// Setup a palette.
var palette = new Uint8Array(256 * 4);
// I'm lazy so just setting 4 colors in palette
function setPalette(index, r, g, b, a) {
palette[index * 4 + 0] = r;
palette[index * 4 + 1] = g;
palette[index * 4 + 2] = b;
palette[index * 4 + 3] = a;
}
setPalette(1, 255, 0, 0, 255); // red
setPalette(2, 0, 255, 0, 255); // green
setPalette(3, 0, 0, 255, 255); // blue
// make palette texture and upload palette
gl.activeTexture(gl.TEXTURE1);
var paletteTex = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, paletteTex);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, 256, 1, 0, gl.RGBA, gl.UNSIGNED_BYTE, palette);
// Make image. Just going to make something 8x8
var width = 8;
var height = 8;
var image = new Uint8Array([
0,0,1,1,1,1,0,0,
0,1,0,0,0,0,1,0,
1,0,0,0,0,0,0,1,
1,0,2,0,0,2,0,1,
1,0,0,0,0,0,0,1,
1,0,3,3,3,3,0,1,
0,1,0,0,0,0,1,0,
0,0,1,1,1,1,0,0,
]);
// make image textures and upload image
gl.activeTexture(gl.TEXTURE0);
var imageTex = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, imageTex);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.ALPHA, width, height, 0, gl.ALPHA, gl.UNSIGNED_BYTE, image);
var frameCounter = 0;
function render() {
++frameCounter;
// skip 3 of 4 frames so the animation is not too fast
if ((frameCounter & 3) == 0) {
// rotate the 3 palette colors
var tempR = palette[4 + 0];
var tempG = palette[4 + 1];
var tempB = palette[4 + 2];
var tempA = palette[4 + 3];
setPalette(1, palette[2 * 4 + 0], palette[2 * 4 + 1], palette[2 * 4 + 2], palette[2 * 4 + 3]);
setPalette(2, palette[3 * 4 + 0], palette[3 * 4 + 1], palette[3 * 4 + 2], palette[3 * 4 + 3]);
setPalette(3, tempR, tempG, tempB, tempA);
// re-upload palette
gl.activeTexture(gl.TEXTURE1);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, 256, 1, 0, gl.RGBA,
gl.UNSIGNED_BYTE, palette);
gl.drawArrays(gl.TRIANGLES, 0, positions.length / 2);
}
requestAnimationFrame(render);
}
render();
canvas { border: 1px solid black; }
<script src="https://twgljs.org/dist/twgl.min.js"></script>
<script id="vshader" type="whatever">
attribute vec4 a_position;
varying vec2 v_texcoord;
void main() {
gl_Position = a_position;
// assuming a unit quad for position we
// can just use that for texcoords. Flip Y though so we get the top at 0
v_texcoord = a_position.xy * vec2(0.5, -0.5) + 0.5;
}
</script>
<script id="fshader" type="whatever">
precision mediump float;
varying vec2 v_texcoord;
uniform sampler2D u_image;
uniform sampler2D u_palette;
void main() {
float index = texture2D(u_image, v_texcoord).a * 255.0;
gl_FragColor = texture2D(u_palette, vec2((index + 0.5) / 256.0, 0.5));
}
</script>
<canvas id="c" width="256" height="256"></canvas>
Slightly related is this tile shader example
http://blog.tojicode.com/2012/07/sprite-tile-maps-on-gpu.html
Presumably you're building up a javascript array that's around 512 x 512 (PAL size)...
A WebGL fragment shader could definitely do your palette conversion pretty nicely. The recipe would go something like this:
Set up WebGL with a "geometry" of just two triangles that span your viewport. (GL is all triangles.) This is the biggest bother, if you're not already GL fluent. But it's not that bad. Spend some quality time with http://learningwebgl.com/blog/?page_id=1217 . But it will be ~100 lines of stuff. Price of admission.
Build your in-memory frame buffer 4 times bigger. (I think textures always have to be RGBA?) And populate every fourth byte, the R component, with your pixel values. Use new Float32Array to allocate it. You can use values 0-255, or divide it down to 0.0 to 1.0. We'll pass this to webgl as a texture. This one changes every frame.
Build a second texture that's 256 x 1 pixels, which is your palette lookup table. This one never changes (unless the palette can be modified?).
In your fragment shader, use your emulated frame buffer texture as a lookup into your palette. The first pixel in the palette is accessed at location (0.5/256.0, 0.5), middle of the pixel.
On each frame, resubmit the emulated frame buffer texture and redraw. Pushing pixels to the GPU is expensive... but a PAL-sized image is pretty small by modern standards.
Bonus step: You could enhance the fragment shader to imitate scanlines, interlace video, or other cute emulation artifacts (phosphor dots?) on modern high-resolution displays, all at no cost to your javascript!
This is just a sketch. But it will work. WebGL is a pretty low-level API, and quite flexible, but well worth the effort (if you like that kind of thing, which I do. :-) ).
Again, http://learningwebgl.com/blog/?page_id=1217 is well-recommended for overall WebGL guidance.
Related
WEBGL-FBO: Why is the DEPTH_COMPONENT texture blank after rendercall
To The Readers: "The solution by gman works for the question, but my own Problem was a in a diffrent part of the Code". I just wanted to render my Scene to a Framebuffer with a DepthComponent_texuture. And wanted to render this texture to the screen, so that i olny see the DepthComponent on my Screen. I want to get into Shadowmapping and throught this would be a good exercise. Stuff i did: -set read/drawBuffer(s) to gl.NONE because i dont hava a colorAttachment (probably dosent even matter). -when creating the depth texture i tried gl.texImage2D but could not get it to work. -When i am using a ColorAttachment isted of a DepthAttachment, i get a normal result(The color Attachment_texture just has the sceen)[obviously here i also added a DepthRenderbuffer(DEPTH24STENCTIL8)]. //creating the Framebuffer id = gl.createFramebuffer(); gl.bindFramebuffer(gl.FRAMEBUFFER,id); atex = gl.createTexture(); gl.bindTexture(gl.TEXTURE_2D, atex); gl.texStorage2D(gl.TEXTURE_2D,1,gl.DEPTH_COMPONENT16,width,height); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.DEPTH_ATTACHMENT, gl.TEXTURE_2D, atex, 0); gl.drawBuffers([gl.NONE]); gl.readBuffer(gl.NONE); gl.bindFramebuffer(gl.FRAMEBUFFER,null); //---------------- //In the renderLoop //renders the scene with the pbrShader to the Framebuffer //The FragmentShader of the PBRShader is empty cause i dont have any //ColorAttachments and the Vertex is just transforming the position and //passing the uvs[...] render(scene,pbrShader,framebuffer); //renders a texture to the screen using a very simple vertex shader and //a Fragment Shader that just takes the texture and mapps it to the Screen renderScreen(framebuffer.atex); //Some more Code for clarity: renderScene(scene,shader,fbo=null){ if(fbo!=null){ fbo.activate(); } this.gl.fClear(); let batches = new Map(); let modals = scene.getEntitiesByType("modal"); modals.forEach(modal=>{ if(batches.has(modal.mesh)){ batches.get(modal.mesh).push(modal); }else{ batches.set(modal.mesh,[modal]); } }) shader.activate().prepareScene(scene); batches.forEach((batch,mesh)=>{ shader.prepareVAO(mesh); batch.forEach(modal=>{ shader.prepareInstance(modal); shader.drawVAO(mesh); }); shader.unbindVAO(mesh); }); shader.deactivate(); if(fbo!=null){ fbo.deactivate(); } }
Here's working example of using depth textures in WebGL2 const vs = `#version 300 es layout(location = 0) in vec4 a_position; out vec2 v_texcoord; void main() { gl_Position = a_position; v_texcoord = a_position.xy * 0.5 + 0.5; } `; const fs = `#version 300 es precision mediump float; in vec2 v_texcoord; uniform sampler2D u_sampler; out vec4 outColor; void main() { vec4 color = texture(u_sampler, v_texcoord); outColor = vec4(color.r, 0, 0, 1); } `; function main() { var gl = document.querySelector("canvas").getContext('webgl2'); if (!gl) { return alert("no WebGL2"); } var program = twgl.createProgram(gl, [vs, fs]); gl.useProgram(program); var verts = [ 1, 1, 1, -1, 1, 0, -1, -1, -1, 1, 1, 1, -1, -1, -1, 1, -1, 0, ]; var vertBuffer = gl.createBuffer(); gl.bindBuffer(gl.ARRAY_BUFFER, vertBuffer); gl.bufferData(gl.ARRAY_BUFFER, new Float32Array(verts), gl.STATIC_DRAW); gl.enableVertexAttribArray(0); gl.vertexAttribPointer(0, 3, gl.FLOAT, false, 0, 0); // create a depth texture. var depthTex = gl.createTexture(); gl.bindTexture(gl.TEXTURE_2D, depthTex); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST); const mipLevel = 0; const depthTexWidth = 16; const depthTexHeight = 16; gl.texImage2D( gl.TEXTURE_2D, mipLevel, gl.DEPTH_COMPONENT24, depthTexWidth, depthTexHeight, 0, gl.DEPTH_COMPONENT, gl.UNSIGNED_INT, null); // Create a framebuffer and attach the textures. var fb = gl.createFramebuffer(); gl.bindFramebuffer(gl.FRAMEBUFFER, fb); gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.DEPTH_ATTACHMENT, gl.TEXTURE_2D, depthTex, 0); // use some other texture to render with while we render to the depth texture. const tex = gl.createTexture(); gl.bindTexture(gl.TEXTURE_2D, tex); gl.texImage2D( gl.TEXTURE_2D, 0, gl.RGBA8, 1, // width 1, // height 0, gl.RGBA, gl.UNSIGNED_BYTE, new Uint8Array([0, 0, 255, 255])); // Turn on depth testing so we can write to the depth texture. gl.enable(gl.DEPTH_TEST); // note: we don't need to set u_sampler because uniforms // default to 0 so it will use texture unit 0 which // is the also the default active texture unit // Render to the depth texture gl.bindFramebuffer(gl.FRAMEBUFFER, fb); gl.viewport(0, 0, depthTexWidth, depthTexHeight); gl.clear(gl.DEPTH_BUFFER_BIT); gl.drawArrays(gl.TRIANGLES, 0, 6); // Now draw with the texture to the canvas gl.bindFramebuffer(gl.FRAMEBUFFER, null); gl.viewport(0, 0, gl.canvas.width, gl.canvas.height); gl.bindTexture(gl.TEXTURE_2D, depthTex); gl.drawArrays(gl.TRIANGLES, 0, 6); } main(); <script src="https://twgljs.org/dist/4.x/twgl.min.js"></script> <canvas></canvas>
How to read and modify pixels according to a color table in WebGL [duplicate]
Currently, I'm using 2D canvas context to draw an image generated (from pixel to pixel, but refreshed as a whole buffer in once after a generated frame) from JavaScript at about a 25fps rate. The generated image is always one byte (integer / typed array) per pixel and a fixed palette is used to generate RGB final result. Scaling is also needed to adopt to the size of the canvas (ie: going to fullscreen) and/or at user request (zoom in/out buttons). The 2D context of canvas is OK for this purpose, however I'm curious if WebGL can provide better result and/or better performance. Please note: I don't want to put pixels via webGL, I want to put pixels into my buffer (which is basically Uint8Array), and use that buffer (in once) to refresh the context. I don't know too much about WebGL, but using the needed generated image as some kind of texture would work somehow for example? Then I would need to refresh the texture at about 25fps rate, I guess. It would be really fantastic, if WebGL support the colour space conversion somehow. With 2D context, I need to convert 1 byte / pixel buffer into RGBA for the imagedata in JavaScript for every pixel ... Scaling (for 2D context) is done now by altering the height/width style of the canvas, so browsers scales the image then. However I guess it can be slower than what WebGL can do with hw support, and also (I hope) WebGL can give greater flexibility to control the scaling, eg with the 2D context, browsers will do antialiasing even if I don't want to do (eg: integer zooming factor), and maybe that's a reason it can be quite slow sometimes. I've already tried to learn several WebGL tutorials but all of them starts with objects, shapes, 3D cubes, etc, I don't need any - classical - object to render only what 2D context can do as well - in the hope that WebGL can be a faster solution for the very same task! Of course if there is no win here with WebGL at all, I would continue to use 2D context. To be clear: this is some kind of computer hardware emulator done in JavaScript, and its output (what would be seen on a PAL TV connected to it) is rendered via a canvas context. The machine has fixed palette with 256 elements, internally it only needs one byte for a pixel to define its colour.
You can use a texture as your palette and a different texture as your image. You then get a value from the image texture and use it too look up a color from the palette texture. The palette texture is 256x1 RGBA pixels. Your image texture is any size you want but just a single channel ALPHA texture. You can then look up a value from the image float index = texture2D(u_image, v_texcoord).a * 255.0; And use that value to look up a color in the palette gl_FragColor = texture2D(u_palette, vec2((index + 0.5) / 256.0, 0.5)); Your shaders might be something like this Vertex Shader attribute vec4 a_position; varying vec2 v_texcoord; void main() { gl_Position = a_position; // assuming a unit quad for position we // can just use that for texcoords. Flip Y though so we get the top at 0 v_texcoord = a_position.xy * vec2(0.5, -0.5) + 0.5; } Fragment shader precision mediump float; varying vec2 v_texcoord; uniform sampler2D u_image; uniform sampler2D u_palette; void main() { float index = texture2D(u_image, v_texcoord).a * 255.0; gl_FragColor = texture2D(u_palette, vec2((index + 0.5) / 256.0, 0.5)); } Then you just need a palette texture. // Setup a palette. var palette = new Uint8Array(256 * 4); // I'm lazy so just setting 4 colors in palette function setPalette(index, r, g, b, a) { palette[index * 4 + 0] = r; palette[index * 4 + 1] = g; palette[index * 4 + 2] = b; palette[index * 4 + 3] = a; } setPalette(1, 255, 0, 0, 255); // red setPalette(2, 0, 255, 0, 255); // green setPalette(3, 0, 0, 255, 255); // blue // upload palette ... gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, 256, 1, 0, gl.RGBA, gl.UNSIGNED_BYTE, palette); And your image. It's an alpha only image so just 1 channel. // Make image. Just going to make something 8x8 var image = new Uint8Array([ 0,0,1,1,1,1,0,0, 0,1,0,0,0,0,1,0, 1,0,0,0,0,0,0,1, 1,0,2,0,0,2,0,1, 1,0,0,0,0,0,0,1, 1,0,3,3,3,3,0,1, 0,1,0,0,0,0,1,0, 0,0,1,1,1,1,0,0, ]); // upload image .... gl.texImage2D(gl.TEXTURE_2D, 0, gl.ALPHA, 8, 8, 0, gl.ALPHA, gl.UNSIGNED_BYTE, image); You also need to make sure both textures are using gl.NEAREST for filtering since one represents indices and the other a palette and filtering between values in those cases makes no sense. gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST); Here's a working example: var canvas = document.getElementById("c"); var gl = canvas.getContext("webgl"); // Note: createProgramFromScripts will call bindAttribLocation // based on the index of the attibute names we pass to it. var program = twgl.createProgramFromScripts( gl, ["vshader", "fshader"], ["a_position", "a_textureIndex"]); gl.useProgram(program); var imageLoc = gl.getUniformLocation(program, "u_image"); var paletteLoc = gl.getUniformLocation(program, "u_palette"); // tell it to use texture units 0 and 1 for the image and palette gl.uniform1i(imageLoc, 0); gl.uniform1i(paletteLoc, 1); // Setup a unit quad var positions = [ 1, 1, -1, 1, -1, -1, 1, 1, -1, -1, 1, -1, ]; var vertBuffer = gl.createBuffer(); gl.bindBuffer(gl.ARRAY_BUFFER, vertBuffer); gl.bufferData(gl.ARRAY_BUFFER, new Float32Array(positions), gl.STATIC_DRAW); gl.enableVertexAttribArray(0); gl.vertexAttribPointer(0, 2, gl.FLOAT, false, 0, 0); // Setup a palette. var palette = new Uint8Array(256 * 4); // I'm lazy so just setting 4 colors in palette function setPalette(index, r, g, b, a) { palette[index * 4 + 0] = r; palette[index * 4 + 1] = g; palette[index * 4 + 2] = b; palette[index * 4 + 3] = a; } setPalette(1, 255, 0, 0, 255); // red setPalette(2, 0, 255, 0, 255); // green setPalette(3, 0, 0, 255, 255); // blue // make palette texture and upload palette gl.activeTexture(gl.TEXTURE1); var paletteTex = gl.createTexture(); gl.bindTexture(gl.TEXTURE_2D, paletteTex); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST); gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, 256, 1, 0, gl.RGBA, gl.UNSIGNED_BYTE, palette); // Make image. Just going to make something 8x8 var image = new Uint8Array([ 0,0,1,1,1,1,0,0, 0,1,0,0,0,0,1,0, 1,0,0,0,0,0,0,1, 1,0,2,0,0,2,0,1, 1,0,0,0,0,0,0,1, 1,0,3,3,3,3,0,1, 0,1,0,0,0,0,1,0, 0,0,1,1,1,1,0,0, ]); // make image textures and upload image gl.activeTexture(gl.TEXTURE0); var imageTex = gl.createTexture(); gl.bindTexture(gl.TEXTURE_2D, imageTex); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST); gl.texImage2D(gl.TEXTURE_2D, 0, gl.ALPHA, 8, 8, 0, gl.ALPHA, gl.UNSIGNED_BYTE, image); gl.drawArrays(gl.TRIANGLES, 0, positions.length / 2); canvas { border: 1px solid black; } <script src="https://twgljs.org/dist/twgl.min.js"></script> <script id="vshader" type="whatever"> attribute vec4 a_position; varying vec2 v_texcoord; void main() { gl_Position = a_position; // assuming a unit quad for position we // can just use that for texcoords. Flip Y though so we get the top at 0 v_texcoord = a_position.xy * vec2(0.5, -0.5) + 0.5; } </script> <script id="fshader" type="whatever"> precision mediump float; varying vec2 v_texcoord; uniform sampler2D u_image; uniform sampler2D u_palette; void main() { float index = texture2D(u_image, v_texcoord).a * 255.0; gl_FragColor = texture2D(u_palette, vec2((index + 0.5) / 256.0, 0.5)); } </script> <canvas id="c" width="256" height="256"></canvas> To animate just update the image and then re-upload it into the texture gl.texImage2D(gl.TEXTURE_2D, 0, gl.ALPHA, 8, 8, 0, gl.ALPHA, gl.UNSIGNED_BYTE, image); Example: var canvas = document.getElementById("c"); var gl = canvas.getContext("webgl"); // Note: createProgramFromScripts will call bindAttribLocation // based on the index of the attibute names we pass to it. var program = twgl.createProgramFromScripts( gl, ["vshader", "fshader"], ["a_position", "a_textureIndex"]); gl.useProgram(program); var imageLoc = gl.getUniformLocation(program, "u_image"); var paletteLoc = gl.getUniformLocation(program, "u_palette"); // tell it to use texture units 0 and 1 for the image and palette gl.uniform1i(imageLoc, 0); gl.uniform1i(paletteLoc, 1); // Setup a unit quad var positions = [ 1, 1, -1, 1, -1, -1, 1, 1, -1, -1, 1, -1, ]; var vertBuffer = gl.createBuffer(); gl.bindBuffer(gl.ARRAY_BUFFER, vertBuffer); gl.bufferData(gl.ARRAY_BUFFER, new Float32Array(positions), gl.STATIC_DRAW); gl.enableVertexAttribArray(0); gl.vertexAttribPointer(0, 2, gl.FLOAT, false, 0, 0); // Setup a palette. var palette = new Uint8Array(256 * 4); // I'm lazy so just setting 4 colors in palette function setPalette(index, r, g, b, a) { palette[index * 4 + 0] = r; palette[index * 4 + 1] = g; palette[index * 4 + 2] = b; palette[index * 4 + 3] = a; } setPalette(1, 255, 0, 0, 255); // red setPalette(2, 0, 255, 0, 255); // green setPalette(3, 0, 0, 255, 255); // blue // make palette texture and upload palette gl.activeTexture(gl.TEXTURE1); var paletteTex = gl.createTexture(); gl.bindTexture(gl.TEXTURE_2D, paletteTex); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST); gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, 256, 1, 0, gl.RGBA, gl.UNSIGNED_BYTE, palette); // Make image. Just going to make something 8x8 var width = 8; var height = 8; var image = new Uint8Array([ 0,0,1,1,1,1,0,0, 0,1,0,0,0,0,1,0, 1,0,0,0,0,0,0,1, 1,0,2,0,0,2,0,1, 1,0,0,0,0,0,0,1, 1,0,3,3,3,3,0,1, 0,1,0,0,0,0,1,0, 0,0,1,1,1,1,0,0, ]); // make image textures and upload image gl.activeTexture(gl.TEXTURE0); var imageTex = gl.createTexture(); gl.bindTexture(gl.TEXTURE_2D, imageTex); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST); gl.texImage2D(gl.TEXTURE_2D, 0, gl.ALPHA, width, height, 0, gl.ALPHA, gl.UNSIGNED_BYTE, image); var frameCounter = 0; function render() { ++frameCounter; // skip 3 of 4 frames so the animation is not too fast if ((frameCounter & 3) == 0) { // rotate the image left for (var y = 0; y < height; ++y) { var temp = image[y * width]; for (var x = 0; x < width - 1; ++x) { image[y * width + x] = image[y * width + x + 1]; } image[y * width + width - 1] = temp; } // re-upload image gl.activeTexture(gl.TEXTURE0); gl.texImage2D(gl.TEXTURE_2D, 0, gl.ALPHA, width, height, 0, gl.ALPHA, gl.UNSIGNED_BYTE, image); gl.drawArrays(gl.TRIANGLES, 0, positions.length / 2); } requestAnimationFrame(render); } render(); canvas { border: 1px solid black; } <script src="https://twgljs.org/dist/twgl.min.js"></script> <script id="vshader" type="whatever"> attribute vec4 a_position; varying vec2 v_texcoord; void main() { gl_Position = a_position; // assuming a unit quad for position we // can just use that for texcoords. Flip Y though so we get the top at 0 v_texcoord = a_position.xy * vec2(0.5, -0.5) + 0.5; } </script> <script id="fshader" type="whatever"> precision mediump float; varying vec2 v_texcoord; uniform sampler2D u_image; uniform sampler2D u_palette; void main() { float index = texture2D(u_image, v_texcoord).a * 255.0; gl_FragColor = texture2D(u_palette, vec2((index + 0.5) / 256.0, 0.5)); } </script> <canvas id="c" width="256" height="256"></canvas> Of course that assumes your goal is to do the animation on the CPU by manipulating pixels. Otherwise you can use any normal webgl techniques to manipulate texture coordinates or whatever. You can also update the palette similarly for palette animation. Just modify the palette and re-upload it gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, 256, 1, 0, gl.RGBA, gl.UNSIGNED_BYTE, palette); Example: var canvas = document.getElementById("c"); var gl = canvas.getContext("webgl"); // Note: createProgramFromScripts will call bindAttribLocation // based on the index of the attibute names we pass to it. var program = twgl.createProgramFromScripts( gl, ["vshader", "fshader"], ["a_position", "a_textureIndex"]); gl.useProgram(program); var imageLoc = gl.getUniformLocation(program, "u_image"); var paletteLoc = gl.getUniformLocation(program, "u_palette"); // tell it to use texture units 0 and 1 for the image and palette gl.uniform1i(imageLoc, 0); gl.uniform1i(paletteLoc, 1); // Setup a unit quad var positions = [ 1, 1, -1, 1, -1, -1, 1, 1, -1, -1, 1, -1, ]; var vertBuffer = gl.createBuffer(); gl.bindBuffer(gl.ARRAY_BUFFER, vertBuffer); gl.bufferData(gl.ARRAY_BUFFER, new Float32Array(positions), gl.STATIC_DRAW); gl.enableVertexAttribArray(0); gl.vertexAttribPointer(0, 2, gl.FLOAT, false, 0, 0); // Setup a palette. var palette = new Uint8Array(256 * 4); // I'm lazy so just setting 4 colors in palette function setPalette(index, r, g, b, a) { palette[index * 4 + 0] = r; palette[index * 4 + 1] = g; palette[index * 4 + 2] = b; palette[index * 4 + 3] = a; } setPalette(1, 255, 0, 0, 255); // red setPalette(2, 0, 255, 0, 255); // green setPalette(3, 0, 0, 255, 255); // blue // make palette texture and upload palette gl.activeTexture(gl.TEXTURE1); var paletteTex = gl.createTexture(); gl.bindTexture(gl.TEXTURE_2D, paletteTex); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST); gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, 256, 1, 0, gl.RGBA, gl.UNSIGNED_BYTE, palette); // Make image. Just going to make something 8x8 var width = 8; var height = 8; var image = new Uint8Array([ 0,0,1,1,1,1,0,0, 0,1,0,0,0,0,1,0, 1,0,0,0,0,0,0,1, 1,0,2,0,0,2,0,1, 1,0,0,0,0,0,0,1, 1,0,3,3,3,3,0,1, 0,1,0,0,0,0,1,0, 0,0,1,1,1,1,0,0, ]); // make image textures and upload image gl.activeTexture(gl.TEXTURE0); var imageTex = gl.createTexture(); gl.bindTexture(gl.TEXTURE_2D, imageTex); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST); gl.texImage2D(gl.TEXTURE_2D, 0, gl.ALPHA, width, height, 0, gl.ALPHA, gl.UNSIGNED_BYTE, image); var frameCounter = 0; function render() { ++frameCounter; // skip 3 of 4 frames so the animation is not too fast if ((frameCounter & 3) == 0) { // rotate the 3 palette colors var tempR = palette[4 + 0]; var tempG = palette[4 + 1]; var tempB = palette[4 + 2]; var tempA = palette[4 + 3]; setPalette(1, palette[2 * 4 + 0], palette[2 * 4 + 1], palette[2 * 4 + 2], palette[2 * 4 + 3]); setPalette(2, palette[3 * 4 + 0], palette[3 * 4 + 1], palette[3 * 4 + 2], palette[3 * 4 + 3]); setPalette(3, tempR, tempG, tempB, tempA); // re-upload palette gl.activeTexture(gl.TEXTURE1); gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, 256, 1, 0, gl.RGBA, gl.UNSIGNED_BYTE, palette); gl.drawArrays(gl.TRIANGLES, 0, positions.length / 2); } requestAnimationFrame(render); } render(); canvas { border: 1px solid black; } <script src="https://twgljs.org/dist/twgl.min.js"></script> <script id="vshader" type="whatever"> attribute vec4 a_position; varying vec2 v_texcoord; void main() { gl_Position = a_position; // assuming a unit quad for position we // can just use that for texcoords. Flip Y though so we get the top at 0 v_texcoord = a_position.xy * vec2(0.5, -0.5) + 0.5; } </script> <script id="fshader" type="whatever"> precision mediump float; varying vec2 v_texcoord; uniform sampler2D u_image; uniform sampler2D u_palette; void main() { float index = texture2D(u_image, v_texcoord).a * 255.0; gl_FragColor = texture2D(u_palette, vec2((index + 0.5) / 256.0, 0.5)); } </script> <canvas id="c" width="256" height="256"></canvas> Slightly related is this tile shader example http://blog.tojicode.com/2012/07/sprite-tile-maps-on-gpu.html
Presumably you're building up a javascript array that's around 512 x 512 (PAL size)... A WebGL fragment shader could definitely do your palette conversion pretty nicely. The recipe would go something like this: Set up WebGL with a "geometry" of just two triangles that span your viewport. (GL is all triangles.) This is the biggest bother, if you're not already GL fluent. But it's not that bad. Spend some quality time with http://learningwebgl.com/blog/?page_id=1217 . But it will be ~100 lines of stuff. Price of admission. Build your in-memory frame buffer 4 times bigger. (I think textures always have to be RGBA?) And populate every fourth byte, the R component, with your pixel values. Use new Float32Array to allocate it. You can use values 0-255, or divide it down to 0.0 to 1.0. We'll pass this to webgl as a texture. This one changes every frame. Build a second texture that's 256 x 1 pixels, which is your palette lookup table. This one never changes (unless the palette can be modified?). In your fragment shader, use your emulated frame buffer texture as a lookup into your palette. The first pixel in the palette is accessed at location (0.5/256.0, 0.5), middle of the pixel. On each frame, resubmit the emulated frame buffer texture and redraw. Pushing pixels to the GPU is expensive... but a PAL-sized image is pretty small by modern standards. Bonus step: You could enhance the fragment shader to imitate scanlines, interlace video, or other cute emulation artifacts (phosphor dots?) on modern high-resolution displays, all at no cost to your javascript! This is just a sketch. But it will work. WebGL is a pretty low-level API, and quite flexible, but well worth the effort (if you like that kind of thing, which I do. :-) ). Again, http://learningwebgl.com/blog/?page_id=1217 is well-recommended for overall WebGL guidance.
How to draw image in WebGL using another canvas buffer Data?
I am trying to draw image to webgl canvas from a 2d canvas. If I use: gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, image); , it works and renders the image successfully, but if I use : gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, c.width, c.height, 0, gl.RGBA, gl.UNSIGNED_BYTE, dataTypedArray); , it just shows a black screen. Here's my Code : Vertex Shader attribute vec2 a_position; uniform vec2 u_resolution; uniform mat3 u_matrix; varying vec2 v_texCoord; void main() { gl_Position = vec4(u_matrix * vec3(a_position, 1), 1); v_texCoord = a_position; } Fragment Shader precision mediump float; // our texture uniform sampler2D u_image; // the texCoords passed in from the vertex shader. varying vec2 v_texCoord; void main() { gl_FragColor = texture2D(u_image, v_texCoord); } Javascript window.onload = main; var buffer = null; function main() { var image = new Image(); image.src = "images/GL.jpg" image.onload = function() { render(image); } } function render(image) { var c = document.getElementById("c"); c.width = window.innerWidth*0.90; c.height = window.innerHeight*0.90; var context = c.getContext('2d'); context.drawImage(image, 0, 0); var imageData = context.getImageData(0,0,image.width,image.height); buffer = imageData.data.buffer; // ArrayBuffer var canvas = document.getElementById("canvas"); canvas.width = window.innerWidth*0.90; canvas.height = window.innerHeight*0.90; //Get A WebGL context var gl = getWebGLContext(canvas); if (!gl) { return; } // setup GLSL program var program = createProgramFromScripts(gl, ["2d-vertex-shader", "2d-fragment-shader"]); gl.useProgram(program); // look up where the vertex data needs to go. var positionLocation = gl.getAttribLocation(program, "a_position"); // look up uniform locations var u_imageLoc = gl.getUniformLocation(program, "u_image"); var u_matrixLoc = gl.getUniformLocation(program, "u_matrix"); // provide texture coordinates for the rectangle. var positionBuffer = gl.createBuffer(); gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer); gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([ 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 1.0, 1.0, 0.0, 1.0, 1.0]), gl.STATIC_DRAW); gl.enableVertexAttribArray(positionLocation); gl.vertexAttribPointer(positionLocation, 2, gl.FLOAT, false, 0, 0); var texture = gl.createTexture(); gl.bindTexture(gl.TEXTURE_2D, texture); // Set the parameters so we can render any size image. gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST); // Upload the image into the texture. var dataTypedArray = new Uint8Array(buffer); //gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, image); textureFromPixelArray(gl, buffer, gl.RGBA, canvas.width, canvas.height); var dstX = 20; var dstY = 30; var dstWidth = canvas.width; var dstHeight = canvas.height; // convert dst pixel coords to clipspace coords var clipX = dstX / gl.canvas.width * 2 - 1; var clipY = dstY / gl.canvas.height * -2 + 1; var clipWidth = dstWidth / gl.canvas.width * 2; var clipHeight = dstHeight / gl.canvas.height * -2; // build a matrix that will stretch our // unit quad to our desired size and location gl.uniformMatrix3fv(u_matrixLoc, false, [ clipWidth, 0, 0, 0, clipHeight, 0, clipX, clipY, 1, ]); // Draw the rectangle. gl.drawArrays(gl.TRIANGLES, 0, 6); } function textureFromPixelArray(gl, dataArray, type, width, height) { var dataTypedArray = new Uint8Array(dataArray); // Don't need to do this if the data is already in a typed array var texture = gl.createTexture(); gl.bindTexture(gl.TEXTURE_2D, texture); gl.texImage2D(gl.TEXTURE_2D, 0, type, width, height, 0, type, gl.UNSIGNED_BYTE, dataTypedArray); // Other texture setup here, like filter modes and mipmap generation console.log(dataTypedArray); return texture; }
So first off, you can pass a canvas directly to gl.texImage2D. There's no good reason to first call ctx.getImageData and get the data out. Just call gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, someCanvas); Second, looking at your code you first create a texture, then set filtering. You then call textureFromPixelArray, WHICH CREATES A NEW TEXTURE, that texture does not have filtering set so if it's not a power-of-2 then it won't render. Just a guess but did you check the JavaScript console? I'm just guessing it probably printed a warning about your texture not being renderable. On top of that, even though textureFromPixelArray creates a new texture the code ignores the return value. To make the code work as is I think you want to change it to this // not needed -- var texture = gl.createTexture(); // not needed -- gl.bindTexture(gl.TEXTURE_2D, texture); // moved from below // Upload the image into the texture. var dataTypedArray = new Uint8Array(buffer); //gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, image); var texture = textureFromPixelArray(gl, buffer, gl.RGBA, canvas.width, canvas.height); // Set the parameters so we can render any size image. gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
Webgl apply 1D texture on a line
I have an webgl application with a lot of segments (gl.LINES). Each segment needs to apply a specific 1D texture, or if you prefer, each pixels of a segment needs to have a specific color. This will be use as an alternative to an histogram on the segment. I made a test texture of 4 pixels from red to black. I would have liked to see a gradient from red to black, but instead the output is a red only line. Here is my getTextureFromPixelArray method: Scene.getTextureFromPixelArray = function(data, width, height) { if (_.isArray(data) && data.length) { var gl = this._context; data = new Uint8Array(data); var texture = gl.createTexture(); gl.bindTexture(gl.TEXTURE_2D, texture); gl.pixelStorei(gl.UNPACK_ALIGNMENT, 1); gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, width, height, 0, gl.RGBA, gl.UNSIGNED_BYTE, data); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); gl.bindTexture(gl.TEXTURE_2D, null); return texture; } else { this.log("Unable to create texture"); } return null; }; Here is my code for generating a test texture 1D : var verticles = []; verticles.push(this.vertex1.position[0]); verticles.push(this.vertex1.position[1]); verticles.push(this.vertex2.position[0]); verticles.push(this.vertex2.position[1]); var color = []; var textureWidth = 4; var textureHeight = 1; var textureCoords = []; for (var i=0;i<textureHeight;i++) { for (var j=0;j<textureWidth;j++) { color.push(Math.round(255 - j*255/(textureWidth-1))); // Red color.push(0); // Green color.push(0); // Blue color.push(255); // Alpha } } textureCoords.push(0); textureCoords.push(0.5); textureCoords.push(1); textureCoords.push(0.5); var vertexPositionAttributeLocation = gl.getAttribLocation(shaderProgram, "aVertexPosition"); var textureCoordAttributeLocation = gl.getAttribLocation(shaderProgram, "aTextureCoord"); // Set vertex buffers gl.enableVertexAttribArray(vertexPositionAttributeLocation); var verticlesBuffer = gl.createBuffer(); gl.bindBuffer(gl.ARRAY_BUFFER, verticlesBuffer); gl.bufferData(gl.ARRAY_BUFFER, new Float32Array(verticles), gl.STATIC_DRAW); gl.vertexAttribPointer(vertexPositionAttributeLocation, 2, gl.FLOAT, false, 0, 0); var texture = this.getTextureFromPixelArray(textureData, textureWidth, textureHeight||1); var vertexTextureCoordBuffer = gl.createBuffer(); gl.bindBuffer(gl.ARRAY_BUFFER, vertexTextureCoordBuffer); gl.bufferData(gl.ARRAY_BUFFER, new Float32Array(textureCoords), gl.STATIC_DRAW); gl.vertexAttribPointer(textureCoordAttributeLocation, 2, gl.FLOAT, false, 0, 0); gl.activeTexture(gl.TEXTURE0); gl.bindTexture(gl.TEXTURE_2D, texture); gl.uniform1i(gl.getUniformLocation(shaderProgram, "uSampler"), 0); // Draw the segment gl.drawArrays(gl.LINES, 0, verticles.length/2); Here is my shaders: Scene.fragmentShaderSrc = "\ precision mediump float;\ varying highp vec2 vTextureCoord;\ uniform sampler2D uSampler;\ void main(void) {gl_FragColor = texture2D(uSampler, vTextureCoord);}\ "; Scene.vertexShaderSrc = "\ attribute vec2 aVertexPosition;\ attribute vec2 aTextureCoord;\ uniform mat4 uPMatrix;\ uniform mat4 uModelView;\ varying highp vec2 vTextureCoord;\ void main(void) {\ gl_Position = uPMatrix * uModelView * vec4(aVertexPosition,0,1);\ vTextureCoord = aTextureCoord;\ }\ "; Can someone see where I made an error?
I needed to enable the texture coordinate attribute with : gl.enableVertexAttribArray(textureCoordAttributeLocation); before var vertexTextureCoordBuffer = gl.createBuffer(); gl.bindBuffer(gl.ARRAY_BUFFER, vertexTextureCoordBuffer); gl.bufferData(gl.ARRAY_BUFFER, new Float32Array(textureCoords), gl.STATIC_DRAW); gl.vertexAttribPointer(textureCoordAttributeLocation, 2, gl.FLOAT, false, 0, 0);
WebGL texture creation trouble
I'm coding a ray tracing program and the main idea is save the triangles and the indices of .obj model into 2 textures: a) Vertex texture b) Indices texture Im doing this to do a ray-triangle intersection in my GLSL fragment shader, browsers are telling me that I've this problem with textures : a) Mozilla Firefox: Error #1 (with vertexes texture) : Error: WebGL: texImage2D: invalid type 0x1406. Error #2 (with indices texture) : Error: WebGL: texImage2D: invalid type 0x1403. b) Google Chrome : Error with both textures : webgl invalid_enum teximage2d invalid texture type Here is the code : Obj.prototype.initTextures = function(){ this.vertexTexture = gl.createTexture(); gl.activeTexture(gl.TEXTURE0) gl.bindTexture(gl.TEXTURE_2D, this.vertexTexture); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST); gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGB, this.vertexSize, 1, 0, gl.RGB, gl.FLOAT, new Float32Array(this.vertexArray)); gl.bindTexture(gl.TEXTURE_2D, null); this.trianglesTexture = gl.createTexture(); gl.activeTexture(gl.TEXTURE1); gl.bindTexture(gl.TEXTURE_2D, this.trianglesTexture); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST); gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGB, this.faceSize, 1, 0, gl.RGB, gl.UNSIGNED_SHORT, new Uint16Array(this.faceArray)); gl.bindTexture(gl.TEXTURE_2D, null); } For obvious reasons this.vertexArray and this.faceArray have the data with them.
gl.RGB / gl.FLOAT texture are only allowed if you check for and enable floating point textures. var floatTextures = gl.getExtension('OES_texture_float'); if (!floatTextures) { alert('no floating point texture support'); return; } gl.RGB / gl.UNSIGNED_SHORT textures do not exist period on WebGL. You can try encoding your unsigned short values as R * 256 + G * 65536 or something along those lines. Or just use floats here too. Note: Filtering floats, as in gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR), is a separate extension OES_texture_float_linear but in your case since you're only using gl.NEAREST you don't need to check for that extension As for putting vertex data in textures you might find you have to do some work to pull out the correct values. To index a value in a texture you need to compute a texture coordinate that will access the correct texel. To do that you need to access from the middle of the first texel to the middle of the last texel. In other words if we had 3 values (and therefore 3 texels) we'd have something like this ------3x1 ----- texels ---------- [ ][ ][ ] 0.0 |<----------------------------->| 1.0 If we just did index / numValues to compute a texture coordinate we'd get [ ][ ][ ] | | | 0.0 0.333 0.666 Which is right between texels so we add a halfTexel to get this [ ][ ][ ] | | | 0.167 0.5 0.833 Hope that made sense. Here's a snippet var gl = document.querySelector("canvas").getContext("webgl"); var ext = gl.getExtension("OES_texture_float"); if (!ext) { alert("need OES_texture_float extension cause I'm lazy"); //return; } if (gl.getParameter(gl.MAX_VERTEX_TEXTURE_IMAGE_UNITS) < 2) { alert("need to be able to access textures from vertex shaders"); //return; } var m4 = twgl.m4; var v3 = twgl.v3; var program = twgl.createProgramFromScripts(gl, ["vshader", "fshader"]); var positionIndexLoc = gl.getAttribLocation(program, "a_positionIndex"); var normalIndexLoc = gl.getAttribLocation(program, "a_normalIndex"); var colorLoc = gl.getUniformLocation(program, "u_color"); var mvpMatrixLoc = gl.getUniformLocation(program, "u_mvpMatrix"); var mvMatrixLoc = gl.getUniformLocation(program, "u_mvMatrix"); var lightDirLoc = gl.getUniformLocation( program, "u_lightDirection"); var u_positionsLoc = gl.getUniformLocation( program, "u_positions"); var u_normalsLoc = gl.getUniformLocation( program, "u_normals"); gl.uniform1i(u_positionsLoc, 0); // use texture unit 0 for positions gl.uniform1i(u_normalsLoc, 1); // use texture unit 1 for normals // Cube data var positions = [ -1, -1, -1, // 0 lbb +1, -1, -1, // 1 rbb 2---3 -1, +1, -1, // 2 ltb /| /| +1, +1, -1, // 3 rtb 6---7 | -1, -1, +1, // 4 lbf | | | | +1, -1, +1, // 5 rbf | 0-|-1 -1, +1, +1, // 6 ltf |/ |/ +1, +1, +1, // 7 rtf 4---5 ]; var positionIndices = [ 3, 7, 5, 3, 5, 1, // right 6, 2, 0, 6, 0, 4, // left 6, 7, 3, 6, 3, 2, // top 0, 1, 5, 0, 5, 4, // bottom 7, 6, 4, 7, 4, 5, // front 2, 3, 1, 2, 1, 0, // back ]; var normals = [ +1, 0, 0, -1, 0, 0, 0, +1, 0, 0, -1, 0, 0, 0, +1, 0, 0, -1, ] var normalIndices = [ 0, 0, 0, 0, 0, 0, // right 1, 1, 1, 1, 1, 1, // left 2, 2, 2, 2, 2, 2, // top 3, 3, 3, 3, 3, 3, // bottom 4, 4, 4, 4, 4, 4, // front 5, 5, 5, 5, 5, 5, // back ]; function degToRad(deg) { return deg * Math.PI / 180; } function uploadIndices(loc, data, indices) { // scale indices into texture coordinates var scaledIndices = new Float32Array(indices.length); // to index the value in the texture we need to // compute a texture coordinate that will access // the correct texel. To do that we need access from // the middle of the first texel to the middle of the // last texel. // // In other words if we had 3 values (and therefore // 3 texels) we'd have something like this // // ------3x1 ----- texels ---------- // [ ][ ][ ] // 0.0 |<----------------------------->| 1.0 // // If we just did index / numValues we'd get // // [ ][ ][ ] // | | | // 0.0 0.333 0.666 // // Which is right between texels so we add a // a halfTexel to get this // // [ ][ ][ ] // | | | // 0.167 0.5 0.833 var size = data.length / 3; var texel = 1 / size; var halfTexel = texel / 2; for (var ii = 0; ii < indices.length; ++ii) { scaledIndices[ii] = indices[ii] / size + halfTexel; } var buffer = gl.createBuffer(); gl.bindBuffer(gl.ARRAY_BUFFER, buffer); gl.bufferData(gl.ARRAY_BUFFER, scaledIndices, gl.STATIC_DRAW); gl.enableVertexAttribArray(loc); gl.vertexAttribPointer(loc, 1, gl.FLOAT, false, 0, 0); } function uploadTexture(unit, data) { gl.activeTexture(gl.TEXTURE0 + unit); var tex = gl.createTexture(); gl.bindTexture(gl.TEXTURE_2D, tex); gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGB, data.length / 3, 1, 0, gl.RGB, gl.FLOAT, new Float32Array(data)); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); } uploadIndices(positionIndexLoc, positions, positionIndices); uploadIndices(normalIndexLoc, normals, normalIndices); uploadTexture(0, positions); uploadTexture(1, normals); var xRot = 30; var yRot = 20; var zRot = 0; var lightDir = v3.normalize([-0.2, -0.1, 0.5]); function draw() { xRot += 0; yRot += 1; twgl.resizeCanvasToDisplaySize(gl.canvas); gl.viewport(0, 0, gl.canvas.width, gl.canvas.height); gl.enable(gl.DEPTH_TEST); gl.enable(gl.CULL_FACE); gl.clearColor(0, 0, 0, 1); gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT); gl.useProgram(program); var projection = m4.perspective( degToRad(45), gl.canvas.clientWidth / gl.canvas.clientHeight, 0.1, 100.0); var world = m4.identity(); world = m4.translate(world, [0.0, 0.0, -5.0]); world = m4.rotateX(world, degToRad(xRot)); world = m4.rotateY(world, degToRad(yRot)); world = m4.rotateZ(world, degToRad(zRot)); var mvp = m4.multiply(projection, world); gl.uniformMatrix4fv(mvMatrixLoc, false, world); gl.uniformMatrix4fv(mvpMatrixLoc, false, mvp); gl.uniform4f(colorLoc, 0.5, 0.8, 1, 1); gl.uniform3fv(lightDirLoc, lightDir); gl.drawArrays(gl.TRIANGLES, 0, 36); requestAnimationFrame(draw); } draw(); body { margin: 0; } canvas { width: 100vw; height: 100vh; display: block; } <script src="//twgljs.org/dist/2.x/twgl-full.min.js"></script> <script id="vshader" type="whatever"> attribute float a_positionIndex; attribute float a_normalIndex; attribute vec4 a_pos; uniform sampler2D u_positions; uniform sampler2D u_normals; uniform mat4 u_mvpMatrix; uniform mat4 u_mvMatrix; varying vec3 v_normal; void main() { vec3 position = texture2D( u_positions, vec2(a_positionIndex, 0.5)).rgb; vec3 normal = texture2D( u_normals, vec2(a_normalIndex, 0.5)).rgb; gl_Position = u_mvpMatrix * vec4(position, 1); v_normal = (u_mvMatrix * vec4(normal, 0)).xyz; } </script> <script id="fshader" type="whatever"> precision mediump float; uniform vec4 u_color; uniform vec3 u_lightDirection; varying vec3 v_normal; void main() { float light = dot( normalize(v_normal), u_lightDirection) * 0.5 + 0.5; gl_FragColor = vec4(u_color.rgb * light, u_color.a); } </script> <canvas></canvas>