I am trying to apply transfer learning by using a knnClassifier and the mobileNet image recognition model in Tensorflow.js I am, however, receiving the following error:
Size(28672) must match the product of shape 28,3072
I don't know how to tackle this issue, I've tried creating tensor3D, resizing using bilinear and nearest neighbor but to no avail. I was wondering if someone here could check this out.
Note that my idea here is to train images from certain folders and assign them to their class using the add example of the knnClassifier. I have a function that reads the image from a path, and an async function that trains the model and makes a prediction from an image.
................................................................................................
const tf = require('#tensorflow/tfjs');
//MobileNet : pre-trained model for TensorFlow.js
const mobilenet = require('#tensorflow-models/mobilenet');
//The module provides native TensorFlow execution
//in backend JavaScript applications under the Node.js runtime.
const tfnode = require('#tensorflow/tfjs-node');
const knnClassifier = require('./node_modules/#tensorflow-models/knn-classifier/dist/knn-classifier');
var glob = require('glob')
//The fs module provides an API for interacting with the file system.
const fs = require('fs');
const readImage = path => {
//reads the entire contents of a file.
//readFileSync() is synchronous and blocks execution until finished.
const imageBuffer = fs.readFileSync(path);
//Given the encoded bytes of an image,
//it returns a 3D or 4D tensor of the decoded image. Supports BMP, GIF, JPEG and PNG formats.
var tfimage = tfnode.node.decodeImage(imageBuffer);
// const t3d = tf.tensor3d(Array.from(tfimage.dataSync()),[tfimage.shape[0], tfimage.shape[1], 1])
const smalImg = tf.image.resizeNearestNeighbor(tfimage, [32, 32]);
const resized = tf.cast(smalImg, 'float32');
// t3d.reshape([32,32,3])
// var smalImg = tf.image.resizeBilinear(tfimage, [368, 432]);
// const resized = tf.cast(smalImg, 'float32');
return resized;
}
var mainDirectory = "./img_samples/";
const imageClassification = async path => {
const classifier = await knnClassifier.create();
const image = await readImage(path);
// Load the model.
const model = await mobilenet.load();
// Classify the image.
const predictions = await model.classify(image);
// print results on terminal
console.log('Classification Results:', predictions);
var folders = fs.readdirSync(mainDirectory);
var filesPerClass = [];
for(var i=0;i<folders.length;i++){
files = fs.readdirSync(mainDirectory+folders[i]);
var files_complete = [];
for(var j=0;j<files.length;j++){
files_complete.push(mainDirectory+folders[i]+"/"+files[j]);
}
filesPerClass.push(files_complete);
}
for(var i=0;i<filesPerClass.length;i++){
for(var j=0;j<filesPerClass[i].length;j++){
imageSample = readImage(filesPerClass[i][j]);
console.log(imageSample);
activation = await model.infer(imageSample, 'conv_preds'); //main directory
classifier.addExample(activation,i);
}
}
console.log(readImage('./hospitalTest.jpg'))
const predictionsTest = await classifier.predictClass(readImage('./hospitalTest.jpg'));
console.log('classficationTest:',predictionsTest);
}
if (process.argv.length !== 3) throw new Error('Incorrect arguments: node classify.js <IMAGE_FILE>');
imageClassification(process.argv[2]);
Since the knn classifier is trained using an output from a node of mobilenet, the prediction needs to be done likewise
outputMobilenet = await model.infer(readImage('./hospitalTest.jpg'), 'conv_preds')
predicted = await classifier.predictClass(outputMobilenet)
Related
So I followed some tutorial on converting a tensorflow model (downloaded from tensorflow hub) to a tfjs model with binary and json files. When I try to use loadLayersModel() it throws a error. When I try to use loadGraphModel(), it loads and runs but the predictions dont seem to work or be meaningful in anyway. How can I tell which method to load models? I need some direction on how to troubleshoot or some recommended workflow particularly in tensorflowJS. I am using this in React Native with expo project. not sure if that matters.
TensorflowHub : https://tfhub.dev/google/aiy/vision/classifier/food_V1/1
my code:
import * as tf from '#tensorflow/tfjs'
import { bundleResourceIO, decodeJpeg } from '#tensorflow/tfjs-react-native'
import * as FileSystem from 'expo-file-system';
const modelJSON = require('./TFJS/model.json')
const modelWeights = require('./TFJS/group1-shard1of1.bin')
export const loadModel = async () => {
const model = await tf.loadLayersModel(
bundleResourceIO(modelJSON, modelWeights)
).catch((e) => {
console.log("[LOADING ERROR] info:", e)
})
return model
}
export const transformImageToTensor = async (uri)=>{
//read the image as base64
const img64 = await FileSystem.readAsStringAsync(uri, {encoding:FileSystem.EncodingType.Base64})
const imgBuffer = tf.util.encodeString(img64, 'base64').buffer
const raw = new Uint8Array(imgBuffer)
let imgTensor = decodeJpeg(raw)
const scalar = tf.scalar(255)
//resize the image
imgTensor = tf.image.resizeNearestNeighbor(imgTensor, [192, 192])
//normalize; if a normalization layer is in the model, this step can be skipped
const tensorScaled = imgTensor.div(scalar)
//final shape of the rensor
const img = tf.reshape(tensorScaled, [1,192,192,3])
return img
}
export const getPredictions = async (image)=>{
await tf.ready()
const model = await loadModel()
const tensor_image = await transformImageToTensor(image)
// const predictions = await makePredictions(1, model, tensor_image)
const prediction = await model.predict(tensor_image)
console.log(prediction)
console.log(prediction.datasync()[0])
return prediction
}
If its a layers model, you you 'loadLayersModel and if its a graph model, you use 'loadGraphModel - they are NOT interchangable and you CANNOT load a model using different method.
So if it loads using loadGraphModel, it is a graph model - as simple as that.
the predictions dont seem to work or be meaningful in anyway
This does not help - what do you expect and what do you actually get?
This code works well in Vanilla JS, but when in a component of React audioContext.decodeAudioData(buffer) returns the error:
"Uncaught (in promise) DOMException: The buffer passed to decodeAudioData contains an unknown content type." (It needs an arrayBuffer)
When logging the buffer I pass to the decoder, it is indeed an ArrayBuffer. I have tried creating the ArrayBuffer in different ways, such as using different file types but nothing has worked.
I built this following documentation and some examples, so far I think it might be an issue wit React.
If someone has a workaround or a solution, It would be awesome. Thanks for taking the time.
import { audioContext, primaryGainControl } from "../App";
const Channel = () => {
/*console.log(audioContext);
console.log(primaryGainControl);*/
const [volume, setVolume] = useState(80);
const channelGain = audioContext.createGain();
channelGain.gain.setValueAtTime(0.8, 0);
channelGain.connect(primaryGainControl);
//Global variables
var startTime = 0;
var offsetToResume = 0;
var pauseTime = 0;
var isPlaying = false;
var songSource = audioContext.createBufferSource();
var rate = 1;
var isPlaying = false;
var FILE_URL = "271417__soneproject__hats-6.wav";
const playAudio = async () => {
if (isPlaying) return;
channelGain.gain.setValueAtTime(volume / 100, audioContext.currentTime);
const response = await fetch(FILE_URL);
const buffer = await response.arrayBuffer();
const songBuffer = await audioContext.decodeAudioData(buffer);
//Here is where the error occurs ^ ^
songSource = audioContext.createBufferSource();
songSource.buffer = songBuffer;
songSource.connect(channelGain);
startTime = audioContext.currentTime;
songSource.start(audioContext.currentTime, offsetToResume);
isPlaying = true;
};
...
I have a couple of functions I am running to download a zip file from Nexus, then unzip/extract the contents of the zip file, finally a search for a specific file type. All function work, however the synchronous search for some reason is not producing any results. If I simply run the download and extract functions in 1 script, then execute the search in another script I get my expected results. I am almost positive it is due to the search being synchronous whereas the download and extract are both async. Is there a quick way to add the find function at the end after the download & extract functions have run? Below is the code:
//npm modules
const fs = require('fs-extra');
const download = require('download');
const unzipper = require('unzipper');
const path = require('path');
//Custom Variables
const artifact = 'SOME_FILE.zip';
const repo = "SOME NEXUS REPOSITORY";
const url = "http://SOME URL/repository/";
const directory = 'SOME DIRECTORY';
//Get Artifact and Extract it to local directory function
const getArtifact = async () => {
const getArtifact = await download(url+repo, "./unzip")
const file = await fs.writeFileSync(directory+artifact, await download(url+repo))
const readStream = await fs.createReadStream(directory + artifact).pipe(unzipper.Extract({path:
directory}))
}
//Find function which should run after download and extract have been fulfilled
const findFile = function (dir, pattern) {
var results = [];
fs.readdirSync(dir).forEach(function (dirInner) {
dirInner = path.resolve(dir, dirInner);
var stat = fs.statSync(dirInner);
console.log(stat)
if (stat.isDirectory()) {
results = results.concat(findFile(dirInner, pattern));
}
if (stat.isFile() && dirInner.endsWith(pattern)) {
results.push(dirInner);
}
});
console.log(results)
return results;
};
//clear contents of directory before new download and extract
fs.emptyDirSync(directory)
//call download and extract function
getArtifact()
When I run "findFile" after the download & extract by itself in a separate script I get expected array output. However, when I try to incorporate (see below) this into the same script I get the an empty array:
getArtifact().then(function findFile (dir, pattern) {
var results = [];
fs.readdirSync(directory).forEach(function (dirInner) {
dirInner = path.resolve(directory, dirInner);
var stat = fs.statSync(dirInner);
console.log(stat)
if (stat.isDirectory()) {
results = results.concat(findFile(dirInner, pattern))
if (stat.isFile() && dirInner.endsWith(pattern)) {
results.push(dirInner);
}
}
console.log(results)
return results;
})
})
//Output
[]
//If I try the following:
getArtifact().then(findFile(directory, file))
// I get same empty array
[]
//If I run "findFile" in its own script after the download extract I get the following:
[
'SOME_FILE_PATH\\document1',
'SOME_FILE_PATH\\document2',
'SOME_FILE_PATH\\document3',
'SOME_FILE_PATH\\document4',
'SOME_FILE_PATH\\document5',
'SOME_FILE_PATH\\document6',
'SOME_FILE_PATH\\document7
]
Any help with how I can incorporate my findFile function into my existing download&extract function is appreciated...
I'd like to extract frames from a gif file in the browser. More specifically, given the url of a gif gifUrl: string, I'd like to download it and obtain it as an array of frames imageList: ImageData[]). I'll be using putImageData on them at various coordinates of a canvas. I'd also like the solution to be lightweight.
On BundlePhobia, omggif is 50ms-long to download via emerging-3G. All alternatives I've seen so far are more around 700ms. However, omggif only offers the basic low level interactions, and common recipes like getting the gif as an array of ImageData are missing.
The best documentation I've found for omggif so far are omggif's types in the DefinitelyTyped project.
There's also movableink's example (awaiting in a PR since January 2019).
I use TypeScript and am thus interested in typed recipes if possible.
Related questions:
How to extract frames from an animated gif using javascript? [closed]
GIF animation on canvas with frame control
Here's how you do it:
import { GifReader } from 'omggif';
export const loadGifFrameList = async (
gifUrl: string,
): Promise<ImageData[]> => {
const response = await fetch(gifUrl);
const blob = await response.blob();
const arrayBuffer = await blob.arrayBuffer();
const intArray = new Uint8Array(arrayBuffer);
const reader = new GifReader(intArray as Buffer);
const info = reader.frameInfo(0);
return new Array(reader.numFrames()).fill(0).map((_, k) => {
const image = new ImageData(info.width, info.height);
reader.decodeAndBlitFrameRGBA(k, image.data as any);
return image;
});
};
If you need transparency, you might want to use canvas, as they can then be interfaced with ctx.drawImage(canvas, x, y):
import { GifReader } from 'omggif';
export const loadGifFrameList = async (
gifUrl: string,
): Promise<HTMLCanvasElement[]> => {
const response = await fetch(gifUrl);
const blob = await response.blob();
const arrayBuffer = await blob.arrayBuffer();
const intArray = new Uint8Array(arrayBuffer);
const reader = new GifReader(intArray as Buffer);
const info = reader.frameInfo(0);
return new Array(reader.numFrames()).fill(0).map((_, k) => {
const image = new ImageData(info.width, info.height);
reader.decodeAndBlitFrameRGBA(k, image.data as any);
let canvas = document.createElement('canvas');
canvas.width = info.width;
canvas.height = info.height;
canvas.getContext('2d')!.putImageData(image, 0, 0);
return canvas;
});
};
I'm using tensorflow js in node and trying to encode my inputs.
const tf = require('#tensorflow/tfjs-node');
const argparse = require('argparse');
const use = require('#tensorflow-models/universal-sentence-encoder');
These are imports, the suggested import statement (ES6) isn't permitted for me in my node environment? Though they seem to work fine here.
const encodeData = (tasks) => {
const sentences = tasks.map(t => t.input);
let model = use.load();
let embeddings = model.embed(sentences);
console.log(embeddings.shape);
return embeddings; // `embeddings` is a 2D tensor consisting of the 512-dimensional embeddings for each sentence.
};
This code produces an error that model.embed is not a function. Why? How do I properly implement an encoder in node.js?
load returns a promise that resolve to the model
use.load().then(model => {
// use the model here
let embeddings = model.embed(sentences);
console.log(embeddings.shape);
})
If you would rather use await, the load method needs to be in an enclosing async function
const encodeData = async (tasks) => {
const sentences = tasks.map(t => t.input);
let model = await use.load();
let embeddings = model.embed(sentences);
console.log(embeddings.shape);
return embeddings; // `embeddings` is a 2D tensor consisting of the 512-dimensional embeddings for each sentence.
};