Using Tensorflows Universal Sentence Encoder in Node.js? - javascript

I'm using tensorflow js in node and trying to encode my inputs.
const tf = require('#tensorflow/tfjs-node');
const argparse = require('argparse');
const use = require('#tensorflow-models/universal-sentence-encoder');
These are imports, the suggested import statement (ES6) isn't permitted for me in my node environment? Though they seem to work fine here.
const encodeData = (tasks) => {
const sentences = tasks.map(t => t.input);
let model = use.load();
let embeddings = model.embed(sentences);
console.log(embeddings.shape);
return embeddings; // `embeddings` is a 2D tensor consisting of the 512-dimensional embeddings for each sentence.
};
This code produces an error that model.embed is not a function. Why? How do I properly implement an encoder in node.js?

load returns a promise that resolve to the model
use.load().then(model => {
// use the model here
let embeddings = model.embed(sentences);
console.log(embeddings.shape);
})
If you would rather use await, the load method needs to be in an enclosing async function
const encodeData = async (tasks) => {
const sentences = tasks.map(t => t.input);
let model = await use.load();
let embeddings = model.embed(sentences);
console.log(embeddings.shape);
return embeddings; // `embeddings` is a 2D tensor consisting of the 512-dimensional embeddings for each sentence.
};

Related

loadLayersModel() or loadGraphModel() for TensorflowJS

So I followed some tutorial on converting a tensorflow model (downloaded from tensorflow hub) to a tfjs model with binary and json files. When I try to use loadLayersModel() it throws a error. When I try to use loadGraphModel(), it loads and runs but the predictions dont seem to work or be meaningful in anyway. How can I tell which method to load models? I need some direction on how to troubleshoot or some recommended workflow particularly in tensorflowJS. I am using this in React Native with expo project. not sure if that matters.
TensorflowHub : https://tfhub.dev/google/aiy/vision/classifier/food_V1/1
my code:
import * as tf from '#tensorflow/tfjs'
import { bundleResourceIO, decodeJpeg } from '#tensorflow/tfjs-react-native'
import * as FileSystem from 'expo-file-system';
const modelJSON = require('./TFJS/model.json')
const modelWeights = require('./TFJS/group1-shard1of1.bin')
export const loadModel = async () => {
const model = await tf.loadLayersModel(
bundleResourceIO(modelJSON, modelWeights)
).catch((e) => {
console.log("[LOADING ERROR] info:", e)
})
return model
}
export const transformImageToTensor = async (uri)=>{
//read the image as base64
const img64 = await FileSystem.readAsStringAsync(uri, {encoding:FileSystem.EncodingType.Base64})
const imgBuffer = tf.util.encodeString(img64, 'base64').buffer
const raw = new Uint8Array(imgBuffer)
let imgTensor = decodeJpeg(raw)
const scalar = tf.scalar(255)
//resize the image
imgTensor = tf.image.resizeNearestNeighbor(imgTensor, [192, 192])
//normalize; if a normalization layer is in the model, this step can be skipped
const tensorScaled = imgTensor.div(scalar)
//final shape of the rensor
const img = tf.reshape(tensorScaled, [1,192,192,3])
return img
}
export const getPredictions = async (image)=>{
await tf.ready()
const model = await loadModel()
const tensor_image = await transformImageToTensor(image)
// const predictions = await makePredictions(1, model, tensor_image)
const prediction = await model.predict(tensor_image)
console.log(prediction)
console.log(prediction.datasync()[0])
return prediction
}
If its a layers model, you you 'loadLayersModel and if its a graph model, you use 'loadGraphModel - they are NOT interchangable and you CANNOT load a model using different method.
So if it loads using loadGraphModel, it is a graph model - as simple as that.
the predictions dont seem to work or be meaningful in anyway
This does not help - what do you expect and what do you actually get?

concatenating two tf.data.Dataset gives "this.lastRead.then is not a function" error in tensorflow.js

I am trying to append a new row in tf.data.Dataset in tensorflow.js, and after searching i figured that the way to do this is by turning the new row which is originally a json object into a dataset object then concatenate it with the pervious one, but i ended up facing this error
"this.lastRead.then is not a function"
I tried to debug it so i tried to concatenate the same dataset with it self and faced the same problem:
csvUrl = 'https://storage.googleapis.com/tfjs-examples/multivariate-linear-regression/data/boston-housing-train.csv';
const a = tf.data.csv(
csvUrl, {
columnConfigs: {
medv: {
isLabel: true
}
}
});
const b = a.concatenate(a);
await b.forEachAsync(e => console.log(e));
and got the same error message, can you help me out?
Currently there is a bug that prevents to concatenate datasets. Until it is deployed in a new version, the dataset iterators can be utilized to do a concatenation. Here is an example:
const csvUrl =
'https://storage.googleapis.com/tfjs-examples/multivariate-linear-regression/data/boston-housing-train.csv';
async function run() {
const csvDataset = tf.data.csv(
csvUrl, {
columnConfigs: {
medv: {
isLabel: true
}
}
});
const numOfFeatures = (await csvDataset.columnNames()).length - 1;
// Prepare the Dataset for training.
const flattenedDataset =
csvDataset
.map(({xs, ys}) =>
{
// Convert xs(features) and ys(labels) from object form (keyed by
// column name) to array form.
return {xs:Object.values(xs), ys:Object.values(ys)};
})
//.batch(10);
const it = await flattenedDataset.iterator()
const it2 = await flattenedDataset.iterator()
const xs = []
const ys = []
// read only the data for the first 5 rows
// all the data need not to be read once
// since it will consume a lot of memory
for (let i = 0; i < 5; i++) {
let e = await it.next()
let f = await it2.next()
xs.push(e.value.xs.concat(f.value.xs))
ys.push(e.value.ys.concat(f.value.ys))
}
const features = tf.tensor(xs)
const labels = tf.tensor(ys)
console.log(features.shape)
console.log(labels.shape)
}
await run();
The only thing to keep in mind is that the above will load all the tensors in memory which cannot be ideal depending on the size of the dataset. A data generator can be used to reduce the memory footprint. here is a very detailed answer to show how to do it
apparently this was a bug in the tensorflow.js library. The bug has been fixed in this pull request: https://github.com/tensorflow/tfjs/pull/5444
Thank you.
Edit
The pull request has been merged and now it is in the version 3.9.0

How to eval a function that use external modules in a web worker

I am trying to implement a simple library to offload given tasks to a pool of web-workers. To do so, I am using the eval operator inside the worker to evaluate stringified functions that come from the main thread. To do so I am leveraging the webpack's plugin worker-plugin to automatically bundle workers inside the application.
First of all, I format the task to normalize arrow functions, class members, and standard functions
function format_task(task: string): string
{
if(task.indexOf('=>') !== -1)
return task;
if(task.indexOf('function') !== -1)
return `(function ${task.slice('function'.length).trim()})`;
return `(function ${task})`;
}
Then, I simply eval the resulting function calling it:
self.onmessage = (event: MessageEvent) =>
{
const {task, data} = event.data;
const formatted_task = format_task(task);
const runnable_task = eval(formatted_task);
const result = await runnable_task(...data);
self.postMessage(result);
}
The main usage would be something like:
const pool = new WorkerPool(new Worker('./worker.ts'), 10);
const task = (a: number, b: number) => a + b;
const worker_task = await pool.create_task(task);
const result = await worker_task.run(42, 12);
So far, so good. The WorkerPool instance is handling a pool of 10 workers, the function gets executed in one of them, and the result gets returned to the main thread.
The problem comes in when I try to use an external dependency inside the task as
import {toUpper} from 'lodash';
const pool = new WorkerPool(new Worker('./worker.ts'), 10);
const task = (a: number, b: number) =>
{
return toUpper('Result is ') + (a + b));
}
const worker_task = await pool.create_task(task);
const result = await worker_task.run(42, 12);
In this case, I get that lodash_1.toUpper is not defined inside the worker. The only solution would be to manually import the same function also inside the worker environment (it works). However, I would like to keep the web-worker implementation as clean as possible for the final user. Something like:
const context: Worker = self as any;
(async () =>
{
const pool_worker = new PoolWorker(context.postMessage.bind(context));
context.onmessage = pool_worker.on_message.bind(pool_worker);
await pool_worker.run();
})().catch(console.error);
I tried importing the needed libraries inside the main worker file but it does not work.
import {toUpper} from 'lodash';
const context: Worker = self as any;
(async () =>
{
const pool_worker = new PoolWorker(context.postMessage.bind(context));
context.onmessage = pool_worker.on_message.bind(pool_worker);
await pool_worker.run();
})().catch(console.error);
It only works if I import it inside the PoolWorker class.
Can you think of a way to "pass" dependencies to the PoolWorker instance letting it importing it through webpack without writing a custom webpack loader/plugin?
Thank you in advance

Is it feasible to use crypto.subtle.digest to generate a digest of the given file?

I've run this example that comes from MDN doc
const text = 'An obscure body in the S-K System, your majesty. The inhabitants refer to it as the planet Earth.';
async function digestMessage(message) {
const msgUint8 = new TextEncoder().encode(message); // encode as (utf-8) Uint8Array
const hashBuffer = await crypto.subtle.digest('SHA-256', msgUint8); // hash the message
const hashArray = Array.from(new Uint8Array(hashBuffer)); // convert buffer to byte array
const hashHex = hashArray.map(b => b.toString(16).padStart(2, '0')).join(''); // convert bytes to hex string
return hashHex;
}
const digestHex = await digestMessage(text);
console.log(digestHex);
I understand the basic usage shown in the example.
However, I don't know how to generates a digest of the given file/blob. I've tried this
const hashBuffer = crypto.subtle.digest('SHA-256', file);
click https://jsfiddle.net/5dn4bjfw/ to see full version.
and got this error.
The provided value is not of type '(ArrayBuffer or ArrayBufferView)'
What should I do?
crypto.subtle.digest() requires an ArrayBuffer or ArrayBufferView.
File can be viewed as ArrayBuffer via file.arrayBuffer().
The function in the example above should be async as in MDN example and use await for all the calls that return Promises.
async function myFunction(){
const finput = document.getElementById('fileinput');
const file = finput.files[0];
const arrayBuffer = await file.arrayBuffer();
const hashBuffer = await crypto.subtle.digest('SHA-256', arrayBuffer); // hash the message
console.log(hashBuffer);
const hashArray = Array.from(new Uint8Array(hashBuffer)); // convert buffer to byte array
const hashHex = hashArray.map(b => b.toString(16).padStart(2, '0')).join('');
console.log(hashHex);
}
If you need an ArrayBuffer, you might able to use the arrayBuffer API
https://developer.mozilla.org/en-US/docs/Web/API/Blob/arrayBuffer
blob.arrayBuffer().then(buffer => {
//your encoding functions here, pass buffer to the function
});

Transfer Learning Tensorflow.js size/shape error

I am trying to apply transfer learning by using a knnClassifier and the mobileNet image recognition model in Tensorflow.js I am, however, receiving the following error:
Size(28672) must match the product of shape 28,3072
I don't know how to tackle this issue, I've tried creating tensor3D, resizing using bilinear and nearest neighbor but to no avail. I was wondering if someone here could check this out.
Note that my idea here is to train images from certain folders and assign them to their class using the add example of the knnClassifier. I have a function that reads the image from a path, and an async function that trains the model and makes a prediction from an image.
................................................................................................
const tf = require('#tensorflow/tfjs');
//MobileNet : pre-trained model for TensorFlow.js
const mobilenet = require('#tensorflow-models/mobilenet');
//The module provides native TensorFlow execution
//in backend JavaScript applications under the Node.js runtime.
const tfnode = require('#tensorflow/tfjs-node');
const knnClassifier = require('./node_modules/#tensorflow-models/knn-classifier/dist/knn-classifier');
var glob = require('glob')
//The fs module provides an API for interacting with the file system.
const fs = require('fs');
const readImage = path => {
//reads the entire contents of a file.
//readFileSync() is synchronous and blocks execution until finished.
const imageBuffer = fs.readFileSync(path);
//Given the encoded bytes of an image,
//it returns a 3D or 4D tensor of the decoded image. Supports BMP, GIF, JPEG and PNG formats.
var tfimage = tfnode.node.decodeImage(imageBuffer);
// const t3d = tf.tensor3d(Array.from(tfimage.dataSync()),[tfimage.shape[0], tfimage.shape[1], 1])
const smalImg = tf.image.resizeNearestNeighbor(tfimage, [32, 32]);
const resized = tf.cast(smalImg, 'float32');
// t3d.reshape([32,32,3])
// var smalImg = tf.image.resizeBilinear(tfimage, [368, 432]);
// const resized = tf.cast(smalImg, 'float32');
return resized;
}
var mainDirectory = "./img_samples/";
const imageClassification = async path => {
const classifier = await knnClassifier.create();
const image = await readImage(path);
// Load the model.
const model = await mobilenet.load();
// Classify the image.
const predictions = await model.classify(image);
// print results on terminal
console.log('Classification Results:', predictions);
var folders = fs.readdirSync(mainDirectory);
var filesPerClass = [];
for(var i=0;i<folders.length;i++){
files = fs.readdirSync(mainDirectory+folders[i]);
var files_complete = [];
for(var j=0;j<files.length;j++){
files_complete.push(mainDirectory+folders[i]+"/"+files[j]);
}
filesPerClass.push(files_complete);
}
for(var i=0;i<filesPerClass.length;i++){
for(var j=0;j<filesPerClass[i].length;j++){
imageSample = readImage(filesPerClass[i][j]);
console.log(imageSample);
activation = await model.infer(imageSample, 'conv_preds'); //main directory
classifier.addExample(activation,i);
}
}
console.log(readImage('./hospitalTest.jpg'))
const predictionsTest = await classifier.predictClass(readImage('./hospitalTest.jpg'));
console.log('classficationTest:',predictionsTest);
}
if (process.argv.length !== 3) throw new Error('Incorrect arguments: node classify.js <IMAGE_FILE>');
imageClassification(process.argv[2]);
Since the knn classifier is trained using an output from a node of mobilenet, the prediction needs to be done likewise
outputMobilenet = await model.infer(readImage('./hospitalTest.jpg'), 'conv_preds')
predicted = await classifier.predictClass(outputMobilenet)

Categories

Resources