Develop Reference

How to let Pix2Pix with p5.js run?

I would like to try the ml5.js Pix2Pix example for p5.js. If I just copy the code, update the paths, and try to let it run on my local server, it doesn't work.
Same here:
// Copyright (c) 2019 ml5
//
// This software is released under the MIT License.
// https://opensource.org/licenses/MIT
/* ===
ml5 Example
Pix2pix Edges2Pikachu example with p5.js using callback functions
This uses a pre-trained model on Pikachu images
For more models see: https://github.com/ml5js/ml5-data-and-training/tree/master/models/pix2pix
=== */
// The pre-trained Edges2Pikachu model is trained on 256x256 images
// So the input images can only be 256x256 or 512x512, or multiple of 256
const SIZE = 256;
let inputImg, inputCanvas, outputContainer, statusMsg, pix2pix, clearBtn, transferBtn, modelReady = false,
isTransfering = false;
function setup() {
// Create a canvas
inputCanvas = createCanvas(SIZE, SIZE);
inputCanvas.class('border-box').parent('canvasContainer');
// Display initial input image
inputImg = loadImage('https://ml5js.github.io/ml5-examples/javascript/Pix2Pix/Pix2Pix_promise/images/input.png', drawImage);
// Selcect output div container
outputContainer = select('#output');
statusMsg = select('#status');
// Select 'transfer' button html element
transferBtn = select('#transferBtn');
// Select 'clear' button html element
clearBtn = select('#clearBtn');
// Attach a mousePressed event to the 'clear' button
clearBtn.mousePressed(function() {
clearCanvas();
});
// Set stroke to black
stroke(0);
pixelDensity(1);
// Create a pix2pix method with a pre-trained model
pix2pix = ml5.pix2pix('https://github.com/ml5js/ml5-library/blob/main/examples/p5js/Pix2Pix/Pix2Pix_callback/models/edges2pikachu.pict', modelLoaded);
}
// Draw on the canvas when mouse is pressed
function draw() {
if (mouseIsPressed) {
line(mouseX, mouseY, pmouseX, pmouseY);
}
}
// Whenever mouse is released, transfer the current image if the model is loaded and it's not in the process of another transformation
function mouseReleased() {
if (modelReady && !isTransfering) {
transfer()
}
}
// A function to be called when the models have loaded
function modelLoaded() {
// Show 'Model Loaded!' message
statusMsg.html('Model Loaded!');
// Set modelReady to true
modelReady = true;
// Call transfer function after the model is loaded
transfer();
// Attach a mousePressed event to the transfer button
transferBtn.mousePressed(function() {
transfer();
});
}
// Draw the input image to the canvas
function drawImage() {
image(inputImg, 0, 0);
}
// Clear the canvas
function clearCanvas() {
background(255);
}
function transfer() {
// Set isTransfering to true
isTransfering = true;
// Update status message
statusMsg.html('Applying Style Transfer...!');
// Select canvas DOM element
const canvasElement = select('canvas').elt;
// Apply pix2pix transformation
pix2pix.transfer(canvasElement, function(err, result) {
if (err) {
console.log(err);
}
if (result && result.src) {
// Set isTransfering back to false
isTransfering = false;
// Clear output container
outputContainer.html('');
// Create an image based result
createImg(result.src).class('border-box').parent('output');
// Show 'Done!' message
statusMsg.html('Done!');
}
});
}
<script src="https://cdnjs.cloudflare.com/ajax/libs/p5.js/0.9.0/p5.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/p5.js/0.9.0/addons/p5.dom.min.js"></script>
<script src="https://unpkg.com/ml5#latest/dist/ml5.min.js" type="text/javascript"></script>
<h1>Pix2Pix Edges2Pichaku Example</h1>
<p>1. Wait until the model is loaded</p>
<p>2. Press your mouse to draw a Pikachu on the left side of the canvas.</p>
<p>3. A colored Pikachu image will automatically appear on the right side of the canvas in ~2 seconds. You could also click the "Transfer" button to generate an new image.</p>
<p>4. You could click the "Clear" button to clear the canvas and draw again.</p>
<p id="status">Loading Model... Please wait...</p>
<div class="flex">
<div>
<div id="canvasContainer"></div>
<div id="btnContainer" class="flex flex-space-between">
<button id="clearBtn">Clear</button><br />
<button id="transferBtn" class="btn">Transfer</button>
</div>
</div>
<div id="transferContainer">
</div>
<div id="output"></div>
</div>
Here would be also a jsFiddle:
https://jsfiddle.net/L6oaydrm/
Has anyone an idea how to let it run? Would be very thankful.

I think I was able to get the 'callback' example to work locally with some tinkering:
Download files from the example: https://github.com/ml5js/ml5-library/tree/main/examples/p5js/Pix2Pix/Pix2Pix_callback
Adjust the index.html to load ml5.min.js from the unpkg.com URL in your code.
Create a new function:
function startTransfer(){
// Create a pix2pix method with a pre-trained model
pix2pix = ml5.pix2pix('./models/edges2pikachu.pict', modelLoaded);
}
Replace all calls to transfer() except the first one in modelLoaded() with startTransfer().
Start a simple local web server; for me: python -m http.server worked.
The example appeared to work. I could draw on the canvas, and the ML model would redraw the Pikachu image factoring in the new lines I added. Note, sometimes the initial transfer is run before the template image (input.png) is loaded, and the result is a garbled yellow / red pixels; clicking 'Transfer' fixes this.
Basically, it always will reload the model into the ml5 library; I don't know of the performance implications of this, but it was redrawing relatively quickly in my browser. The file will be cached in the browser, so that isn't a concern, but I'm not sure of the internals of the ml5.js lib and what ml5.pix2pix(...) does.
I've put my revised code (including some other tweaks to the JS) up on https://jsfiddle.net/lecrte/jvohcw8r/16/ ... but it won't work there because the assets aren't available relative to the HTML file, and we can't load the edges2pikachu.pict direct from github.com due to CORS issues.

A-frame - Is there a way to make an image show up on only one side of a shape?

I'm very new to A-frame and doing a project for my class that requires me to take a 2D image and use it in a 3D scene in whatever way I can come up with. For my project, I want the image to be displayed on multiple boxes, but only on the side that the camera sees first. Anytime I link an image to a shape, it will cover the whole shape as a texture. Is there a way to specify where it can show up on the shape?

There is a component which allows you to use different materials for each side of a cube.
If You're interested in doing it by yourself then you could write a component which would
load the texture
create an array of six materials (one for each side) by only one textured
use the array as the material of the cube
Like this:
<script src="https://aframe.io/releases/1.2.0/aframe.min.js"></script>
<script>
AFRAME.registerComponent("foo", {
schema: {
img: {type: "selector"}
},
init: function() {
// wait until the entity is loaded
this.el.addEventListener("loaded", evt => {
// grab the mesh
const mesh = this.el.getObject3D("mesh");
// load the texture
this.el.sceneEl.systems.material.loadTexture(this.data.img, {
src: this.data.img
},
// when loaded, create 6 materials for each side
function textureLoaded(texture) {
var cubeMaterial = [
//left
new THREE.MeshBasicMaterial({map: texture}),
//right
new THREE.MeshBasicMaterial({color: 'orange'}),
// top
new THREE.MeshBasicMaterial({color: 'green'}),
// bottom
new THREE.MeshBasicMaterial({color: 'blue'}),
// front
new THREE.MeshBasicMaterial({olor: 'pink'}),
//back
new THREE.MeshBasicMaterial({color: 'yellow'})
];
// apply the new material + clean up the old one
var oldMaterial = mesh.material;
mesh.material = cubeMaterial;
oldMaterial.dispose()
});
});
}
})
</script>
<a-scene>
<a-assets>
<img id="lobster" src="https://i.imgur.com/wjobVTN.jpeg" crossorigin="anonymous" />
</a-assets>
<a-box position="-1 0.5 -3" rotation="45 -45 0" foo="img: #lobster"></a-box>
</a-scene>
if you try to find out on which side of the cube is your camera, then you can change the array, or add / remove the map attribute.

Aframe Dynamic Canvas as Texture

I was trying to use a canvas as texture in my aframe project. I found some instructions here. It mentioned:
The texture will automatically refresh itself as the canvas changes.
However, I gave it a try today and the canvas could only be changed / updated in init function. Afterwards the update to canvas cannot be reflected. Here is my implementation:
module.exports = {
'canvas_component': {
schema: {
canvasId: { type: 'string' }
},
init: function () {
this.canvas = document.getElementById(this.data.canvasId);
this.ctx = this.canvas.getContext('2d');
this.ctx.fillStyle = "#FF0000";
this.ctx.fillRect(20, 20, 150, 100);
setTimeout(() => {
this.ctx.fillStyle = "#FFFF00";
this.ctx.fillRect(20, 20, 150, 100);
}, 2000);
}
}
The color change of of the texture was never changed. Is there anything I missed? Thank you so much for any advice.

I could never get it to work with those instructions (never checked out if bug or improper use though), but you can achieve the same with Three.js:
// assuming this is inside an aframe component
init: function() {
// we'll update this manually
this.texture = null
let canvas = document.getElementById("source-canvas");
// wait until the element is ready
this.el.addEventListener('loaded', e => {
// create the texture
this.texture = new THREE.CanvasTexture(canvas);
// get the references neccesary to swap the texture
let mesh = this.el.getObject3D('mesh')
mesh.material.map = this.texture
// if there was a map before, you should dispose it
})
},
tick: function() {
// if the texture is created - update it
if (this.texture) this.texture.needsUpdate = true
}
Check it out in this glitch.
Instead using the tick function, you could update the texture whenever you get any callback from changing the canvas (mouse events, source change).

The docs are out of date, I've made a pull request to update them. Here is the code that shows how to do it now:
src: https://github.com/aframevr/aframe/issues/4878
which points to: https://github.com/aframevr/aframe/blob/b164623dfa0d2548158f4b7da06157497cd4ea29/examples/test/canvas-texture/components/canvas-updater.js
We can quickly turn that into a component like this, for example:
/* global AFRAME */
AFRAME.registerComponent('live-canvas', {
dependencies: ['geometry', 'material'],
schema: {
src: { type: "string", default: "#id"}
},
init() {
if (!document.querySelector(this.data.src)) {
console.error("no such canvas")
return
}
this.el.setAttribute('material',{src:this.data.src})
},
tick() {
var el = this.el;
var material;
material = el.getObject3D('mesh').material;
if (!material.map) {
console.error("no material map")
this.el.removeAttribute('live-canvas')
return;
}
material.map.needsUpdate = true;
}
});
(remember to declate your components before your scene...)
usage:
<body>
<canvas id="example-canvas"></canvas>
<a-scene>
<a-box live-canvas="src:#example-canvas;"></a-box>
</a-scene>
</body>
live glitch code demo here:
https://glitch.com/edit/#!/live-canvas-demo?path=index.html%3A58%3A43
You can of course be more efficient than a tick handler if you just intentionally run the equivalent code manually whenever you update the canvas yourself, if that makes more sense / isn't happening frame-by-frame.

Click event does not provide the mesh name in gltf model

Is there a simple way to figure out the name of the child mesh that was clicked in a gltf model? I am on 0.8.0 of aframe.
I tried the following -
HTML source:
<a-scene embedded="" raycaster-autorefresh cursor="rayOrigin: mouse">
<a-assets>
<a-asset-item id="male" src="../images/trying/scene.gltf">
</a-asset-item>
</a-assets>
<a-entity id="camera" camera="" position="0 1.6 0" look-controls wasd-controls>
</a-entity>
<a-entity gltf-model="#male" position="-14 -30 -125" rotation= "0 160 0" material-displacements></a-entity>
</a-scene>
Component source:
// first component
AFRAME.registerComponent('raycaster-autorefresh', {
init: function () {
var el = this.el;
this.el.addEventListener('object3dset', function () {
var cursorEl = el.querySelector('[raycaster]');
cursorEl.components.raycaster.refreshObjects();
});
}
});
// second component
var lastIndex = -1;
var colors = ['red', 'green', 'blue', 'black'];
AFRAME.registerComponent('material-displacements', {
init: function () {
this.el.addEventListener('object3dset', () => {
this.el.addEventListener('click', (evt) => {
console.log(evt.detail.intersection.object.name);
});
});
},
update: function () {
lastIndex = (lastIndex + 1) % colors.length;
console.log(lastIndex);
this.material = new THREE.MeshStandardMaterial({color: colors[lastIndex]});
const mesh = this.el.getObject3D('mesh');
console.log("mesh is", mesh);
console.log("material is", this.material);
if (mesh) {
mesh.traverse((node) => {
if (node.name==="bed_Wood_0") {
if (node.isMesh) node.material = this.material;
}
});
}
}
});
But, it only provides the name of the first child mesh. Similarly, evt.detail.intersection.point only provides the same x, y and z coordinates irrespective of where I click on the model.
My requirement is to be able to figure out which child mesh was clicked inside the gltf model

Similarly, evt.detail.intersection.point only provides the same x, y and z coordinates irrespective of where I click on the model.
This is a bug — update to A-Frame 0.8.2 per issue #3467. There should also be no need for raycaster-autorefresh in A-Frame 0.8+.
For the remaining issue of getting names from nodes attached to the one that was clicked, consider this code:
this.el.addEventListener('click', (evt) => {
var object = evt.detail.intersection.object;
// name of entity to which component is attached
console.log(this.el.getObject3D('mesh').name);
// name of object directly clicked
console.log(object.name);
// name of object's parent
console.log(object.parent.name);
// name of object and its children
object.traverse((node) => console.log(node.name));
});
All of the above will give names of nodes around the mesh that was clicked. Without a demo I can't guess which of them you expect to have clicked, or what nodes exist, so I think you'll need to experiment from there. Also see docs for THREE.Object3D.

Exception when running three.js example using iewebgl

I'm trying to use the iewebgl and having trouble running one of the examples from three.js, the webgl_loader_obj. I am getting the following error:
SCRIPT445: Object doesn't support this action
iewebgl.html, line 134 character 5
which points to this line
// texture
var manager = new THREE.LoadingManager(); <!-- here -->
manager.onProgress = function ( item, loaded, total ) {
console.log( item, loaded, total );
};
I also tried commenting out the texture and model portions and loading the object without the manager but I would receive the following error:
SCRIPT445: Object doesn't support this action
OBJLoader.js, line 19 character 3
which points to this line
THREE.OBJLoader.prototype = {
constructor: THREE.OBJLoader,
load: function ( url, onLoad, onProgress, onError ) {
var scope = this;
var loader = new THREE.XHRLoader( scope.manager ); <!-- here-->
loader.setCrossOrigin( this.crossOrigin );
I tried both creating canvas and getting WebGL context from JavaScript and creating WebGL context with helper.
I'm using ie 10 with r.46 of three.js. When I use r.61 of three.js I get the following exception
SCRIPT5007: Unable to get property 'getExtension' of undefined or null reference
three.min.js, line 8322 character 2
which is
} catch (Zb) {
console.error(Zb)
}
Na = j.getExtension("OES_texture_float"); <!-- here -->
j.getExtension("OES_texture_float_linear");
va = j.getExtension("OES_standard_derivatives");
Any idea what may be causing it?

I was able to solve my issue using the Creating canvas and getting WebGL context from JavaScript example
<div id="container">
<script id="WebGLCanvasCreationScript" type="text/javascript">WebGLHelper.CreateGLCanvasInline('renderCanvas', onLoad)</script>
</div>
I then placed both the init() and animate() calls in the following block leaving out the global vars
var container, stats;
var camera, cameraTarget, scene, renderer;
function onLoad() {
<!-- require(["js/Three.js"], function () { -->
init();
animate();
<!-- }); -->
}
I didn't use the require code, but included it in case anyone else runs into a similar problem and finds they need it.
Finally, had to change the render code to the following. (I left the original commented)
var externalCanvas = document.getElementById('renderCanvas');
renderer = new THREE.WebGLRenderer({ 'canvas': externalCanvas, 'clearColor': 0xffffff });
<!-- renderer = new THREE.WebGLRenderer(); -->
<!-- renderer = new THREE.WebGLRenderer( { antialias: true, alpha: false } ); -->

If you use the un-minified version of three.js, I think you'll see the problem is the _gl object (j in the minified version) is null. It also looks to me as if three.js will throw an exception, log the message "Error creating WebGL context" and then proceed to try to use the _gl object. :)
Did you see that message in your console? Here's the un-minified source where I think your error occurs.
function initGL() {
try {
var attributes = {
alpha: _alpha,
premultipliedAlpha: _premultipliedAlpha,
antialias: _antialias,
stencil: _stencil,
preserveDrawingBuffer: _preserveDrawingBuffer
};
_gl = _canvas.getContext( 'webgl', attributes ) || _canvas.getContext( 'experimental-webgl', attributes );
if ( _gl === null ) {
throw 'Error creating WebGL context.';
}
} catch ( error ) {
console.error( error );
}
_glExtensionTextureFloat = _gl.getExtension( 'OES_texture_float' );
_glExtensionTextureFloatLinear = _gl.getExtension( 'OES_texture_float_linear' );
_glExtensionStandardDerivatives = _gl.getExtension( 'OES_standard_derivatives' );
Is WebGL supported on your IE? Do other webgl tests work? I don't have IE Win8.1 preview but I am curious to know if three.js works with it.