Develop Reference

how to HTTP Request code Loop Until Status Code 200 - NodeJS

so i have create a function where i call an api and then fetch from that response but some time the api which i'm using right know it's return success so, i wan't to check for successful response from webscraping.com and retry 5 times if the response is failing, because the actual API request might fail and we need to track that
const wrapper = require('./wrapper');
const {save } = require('./api.service');
function hasHex (url){
var start = new Date();
var googleRemarkeging;
var facbookPixel;
var googleCheck = 'https://www.googletagmanager.com/gtm.js?id=';
var facbookCheck = 'https://connect.facebook.net/en_US/fbevents.js';
/////////////////////////// MAIN FUNCTION //////////////////////////
const options = {
"method": "GET",
"hostname": "api.webscrapingapi.com",
"port": null,
"path": `/v1?api_key=${wrapper.api_key}&url=${encodeURIComponent(wrapper.target_url(url))}&render_js=1&timeout=60000`,
};
const req = wrapper.http.request(options, function(res) {
const chunks = [];
res.on("data", function(chunk) {
chunks.push(chunk);
});
res.on("end", function() {
const body = Buffer.concat(chunks);
const response = body.toString();
const $ = wrapper.cheerio.load(response);
// console.log($.html())
if (res.statusCode == 200) {
$("script").each(function(index,text){
if($(text).text().includes(googleCheck)== true){
googleRemarkeging = "Yes";
return false;
}
})
$("script").each(function(index,text){
if($(text).text().includes(facbookCheck)== true){
facbookPixel = "Yes";
return false;
}
})
const data ={
"status":"success",
"result":{
googleRemarkeging:googleRemarkeging,
facbookPixel:facbookPixel
}
}
var result = JSON.stringify(data.result);
var duration = (new Date() - start)/1000;
var api_endpoint = '/api/hasHex'
save(api_endpoint,url,data,duration,result);
console.log(data);
}
});
});
req.end();
}
module.exports = hasHex;

Firebase Cloud Function error, Function returned undefined, expected Promise or value

I have a cloud function which triggers on certain database write(onCreate), it works as expected but it also throws an error "Function returned undefined, expected Promise or value" though I am returning a promise.
Attaching the code snippet below. there are nested promises in it. is there a better way to handle nested promises, I already checked many posts for nested promises but not able to figure out a proper solution.
Thanks in Advance
exports.calculateAnswer = function(snap, context, dbPath,bucket) {
const answerKey = snap.val();
const incidentId = context.params.incidentId;
const matchId = context.params.match;
var globalIncidentPath = globalIncidentRootPath.replace('${match}', matchId);
globalIncidentPath = globalIncidentPath + incidentId + '/'
var pdPath = pdRootPath.replace('${match}', matchId);
pdPath = pdPath + incidentId
pdPath = pdPath + "/" + bucket
var incidentsPath = incidentsRootPath.replace('${match}', matchId);
var earningsNodePath = earningsNodeRootPath.replace('${match}', matchId);
let app = admin.app();
var globalData = null;
var globalData = null;
const globalPromise = app.database(dbPath).ref(globalIncidentPath).once('value').then(function(snapshot) {
globalData = snapshot.val();
console.log("globalData ",globalIncidentPath, "data ",globalData);
if(globalData) {
console.log("fetching pddata")
return app.database(dbPath).ref(pdPath).once('value')
}
else{
console.log("No global data found");
return true
}
}).then(function(pdSnashot){
const pdData = pdSnashot.val()
if(pdData) {
var promises = []
pdSnashot.forEach(function(childSnap){
console.log('key ',childSnap.key)
console.log('users count ',childSnap.numChildren())
childSnap.forEach(function(usersSnap){
const userId = usersSnap.key
const incidentProcessed = incidentsPath + userId + '/processed/' + incidentId
if (childSnap.key === answerKey) {
const earningUserIdEPath = earningsNodePath + userId
//const earningEPath = earningUserIdEPath + '/e/'
let gocashValue = globalData['v'];
const earningFetchPromise = app.database(dbPath).ref(earningUserIdEPath).once('value').then(function(snapshot1){
let snapDict = snapshot1.val();
var newGoCash = gocashValue
var newPDGoCash = gocashValue
if (snapDict){
let currentGoCash =snapDict['e'];
let currentPDCash = snapDict['pd']
if(currentGoCash) {
newGoCash = currentGoCash + gocashValue;
}
if(currentPDCash) {
newPDGoCash = currentPDCash + gocashValue;
}
}
const obj = Object()
obj["e"] = newGoCash
obj["pd"] = newPDGoCash
const earningPromise = app.database(dbPath).ref(earningUserIdEPath).update(obj)
const tempGlobal = globalData
tempGlobal["skip"] = false;
const processedPromise = app.database(dbPath).ref(incidentProcessed).set(tempGlobal)
return Promise.all([earningPromise,processedPromise])
});
promises.push(earningFetchPromise)
}
else{
const tempGlobal = globalData
tempGlobal["skip"] = true;
const processIncidentPromise = app.database(dbPath).ref(incidentProcessed).set(tempGlobal);
promises.push(processIncidentPromise)
}
})
})
return Promise.all(promises).then(value => {
console.log("Pd promises completed",value);
return true
})
}
else{
console.log("No Pd Data Found");
return true
}
})
.catch(function(error){
console.log('error in promise resolve',error)
})
console.log('global promise',globalPromise)
return Promise.all([globalPromise])
})

I would modify your code as follows. See the comments within the code.
var globalData = null;
const globalPromise = app
.database(dbPath)
.ref(globalIncidentPath)
.once('value')
.then(function(snapshot) {
globalData = snapshot.val();
console.log('globalData ', globalIncidentPath, 'data ', globalData);
if (globalData) {
console.log('fetching pddata');
return app
.database(dbPath)
.ref(pdPath)
.once('value');
} else {
console.log('No global data found');
// return true; Better to throw an error here
throw new Error('No global data found');
}
})
//The following 3 lines don't bring anything
//Moreover they are most probably the reason of your error as you don't return anything in this then()
//.then(function(pdSnashot){
// console.log("");
//})
.catch(function(error) {
console.log('error in promise resolve', error);
return true;
//Note that if you don't need the console.log you may ommit the entire catch since the platform will handle the error itself.
});
console.log('global promise', globalPromise);
//return Promise.all([globalPromise]); // There is no reason to use Promise.all() here since there is only one promise chain returned in globalPromise
return globalPromise;

Multithread node.js instagram parser

Working on node.js instagram parser. At the moment I have 1 thread with proxy working code, but I'm not sure how to make multithread architecture:
'use strict';
var InstagramPrivateAPI = {};
InstagramPrivateAPI = {};
InstagramPrivateAPI.V1 = require(__dirname + '/client/v1');
InstagramPrivateAPI.Helpers = require(__dirname + '/helpers');
var acc = require(__dirname + "/client/v1/account");
var med = require(__dirname + "/client/v1/media")
var Promise = require('../bluebird');
var _ = require('../lodash/');
module.exports = InstagramPrivateAPI;
var Client = require('instagram-private-api').V1;
var device = new Client.Device('maksgmn');
var storage = new Client.CookieFileStorage(__dirname + '/cookies/maksgmn.json');
var session = new Client.Session(device, storage);
function getRandomInt(min, max) {
return Math.floor(Math.random() * (max - min)) + min;
}
var fs = require('fs');
var proxyArray = fs.readFileSync('proxy.txt').toString().split("\n");
var usernamesArray = fs.readFileSync('usernames.txt').toString().split("\n");
var proxy = "http://" + proxyArray[getRandomInt(0,proxyArray.length)]
var username = usernamesArray[getRandomInt(0,usernamesArray.length)]
console.log(proxy)
console.log(username)
Client.Request.setProxy(proxy);
acc.searchForUser(session, username) //поиск id пользователя
.then(function(profile) {
return profile.id
})
.then(function(someId) { //получение промиса lenta
var feed = new Client.Feed.UserMedia(session, someId);
var lenta = Promise.mapSeries(_.range(0, 1), function() {
return feed.get();
}).then(function(lenta) {
return {id: someId, fd : lenta}
})
return lenta
})
.then(function(results) { //обработка промиса и получение ленты пользователя
// result should be Media[][]
var media1 = _.flatten(results.fd);
var urls1 = _.map(media1, function(medium) {
//var arr1 = medium.params.images[0].url;
var arr1 = []
try {
arr1 = medium.params.images[0].url
} catch (err) {
//console.log("lala")
}
return arr1;
//console.log(medium.params.carouselMedia.images[0].url)
})
//console.log(urls1)
return {id : results.id, linksNoCarousel : urls1, med : media1}
})
.then(function(res){
var urls2 = _.map(res.med, function(medium) {
var arr2 = []
try {
arr2 = medium.params.images[0][0].url
//console.log(arr2)
} catch (err) {
}
return arr2
})
for (var i = 0; i < 5; i++) {
if (typeof res.linksNoCarousel[i] == "undefined")
res.linksNoCarousel[i] = urls2[i];
}
var arr3 = []
for (var i = 0; i < 5; i++) {
arr3[i] = res.linksNoCarousel[i]
}
return {id : res.id, links : arr3}
})
.then(function(mediaAndId) {
acc = acc.getById(session, mediaAndId.id)
.then(function(account) {
//console.log(account.params)
let avatar = account.params.profilePicUrl;
let fullName = account.params.fullName;
let bio = account.params.biography;
let media0 = mediaAndId.links[0];
let media1 = mediaAndId.links[1];
let media2 = mediaAndId.links[2];
let media3 = mediaAndId.links[3];
let media4 = mediaAndId.links[4];
console.log(avatar);
console.log(fullName);
console.log(bio);
console.log(media0);
console.log(media1);
console.log(media2);
console.log(media3);
console.log(media4);
})
})
I would like it to work like multithread to be much more faster with proxies. As far as I'm working with node.js 2nd day, asking that question: how to do that?

module.exports is returning undefined

I am currently new to Node JS, and today I was trying to read data from a file data.json.
Here is the JSON file:
{"username":"rahul_v7","password":"9673"} {"username":"7vik","password":"3248"} {"username":"pradypot_2","password":"6824"} {"username":"ad_1","password":"9284"} {"username":"premchand_4","password":"4346"}
And, I was using the below code present in a file GetData.js, to read the data present in the data.json:
'use strict';
const fs = require('fs');
let res = '', resObjs = [];
let fin = fs.createReadStream('F:/RahulVerma/NodeJS/data.json', 'utf-8');
fin.on('data', data => {
if(data.length > 0) res += data;
}).on('end', () => {
if(res.length > 0) {
let resArr = res.trim().split(' ');
for(let i = 0; i < resArr.length; i++) {
resObjs.push(JSON.parse(resArr[i]));
}
module.exports.objects = resObjs;
}
});
As you can see, I am exporting the resObjs array, which is actually an array of objects, to an another file named AppendData.js, which is given below:
'use strict';
const fs = require('fs');
const getObjs = require('./GetData');
console.log(getObjs.objects);
But, when I run AppendData.js in Node.js 9.3.0 (ia32), it gives the following output:

You're trying to use the objects before they've been read. Remember that your code reading the stream runs asynchronously, and nothing in your code attempts to coordinate it with module loading. So AppendData.js isn't seeing the objects export because it doesn't exist yet as of when that code runs.
Instead, return a promise of the objects that AppendData.js can consume; see *** comments:
'use strict';
const fs = require('fs');
// *** Export the promise
module.exports.objectsPromise = new Promise((resolve, reject) => {
let res = '', resObjs = [];
let fin = fs.createReadStream('F:/RahulVerma/NodeJS/data.json', 'utf-8');
fin.on('data', data => {
if(data.length > 0) res += data;
}).on('end', () => {
if(res.length > 0) {
let resArr = res.trim().split(' ');
for(let i = 0; i < resArr.length; i++) {
resObjs.push(JSON.parse(resArr[i]));
}
resolve(resObjs); // *** Resolve the promise
}
}).on('error', error => {
reject(error); // *** Reject the promise
});
});
Note I added a handler for errors.
And then:
'use strict';
const fs = require('fs');
const getObjs = require('./GetData');
getObjs.objectsPromise
.then(console.log)
.catch(error => {
// Do something
});
Again note the error handler.

The problem happens because you're trying to use the objects in AppendData.js before they are loaded on GetData.js due to fs.createReadStream being asynchronous. To fix this just make module.exports be a function that expect a callback in GetData.js like:
'use strict';
const fs = require('fs');
module.exports = function(callback) {
let res = '', resObjs = [];
let fin = fs.createReadStream('F:/RahulVerma/NodeJS/data.json', 'utf-8');
fin.on('data', data => {
if(data.length > 0) res += data;
}).on('end', () => {
if(res.length > 0) {
let resArr = res.trim().split(' ');
for(let i = 0; i < resArr.length; i++) {
resObjs.push(JSON.parse(resArr[i]));
}
callback(resObjs); // call the callback with the array of results
}
});
}
Which you can then use like this in AppendData.js:
'use strict';
const fs = require('fs');
const getObjs = require('./GetData'); // getObjs is now a function
getObjs(function(objects) {
console.log(objects);
});

I would like to analyze the image with tensorflw.js - nodejs

mobilenet.js
var loadFrozenModel = require('#tensorflow/tfjs-converter');
var NamedTensorMap = require('#tensorflow/tfjs-converter');
var tfc = require('#tensorflow/tfjs-core');
var IMAGENET_CLASSES = require('./imagenet_classes');
const GOOGLE_CLOUD_STORAGE_DIR = 'https://storage.googleapis.com/tfjs-models/savedmodel/';
const MODEL_FILE_URL = 'mobilenet_v1_1.0_224/optimized_model.pb';
const WEIGHT_MANIFEST_FILE_URL = 'mobilenet_v1_1.0_224/weights_manifest.json';
const INPUT_NODE_NAME = 'input';
const OUTPUT_NODE_NAME = 'MobilenetV1/Predictions/Reshape_1';
const PREPROCESS_DIVISOR = tfc.scalar(255 / 2);
class MobileNet {
constructor() {}
async load() {
this.model = await loadFrozenModel(
GOOGLE_CLOUD_STORAGE_DIR + MODEL_FILE_URL,
GOOGLE_CLOUD_STORAGE_DIR + WEIGHT_MANIFEST_FILE_URL);
}
dispose() {
if (this.model) {
this.model.dispose();
}
}
predict(input) {
const preprocessedInput = tfc.div(
tfc.sub(input.asType('float32'), PREPROCESS_DIVISOR),
PREPROCESS_DIVISOR);
const reshapedInput =
preprocessedInput.reshape([1, ...preprocessedInput.shape]);
const dict = {};
dict[INPUT_NODE_NAME] = reshapedInput;
return this.model.execute(dict, OUTPUT_NODE_NAME);
}
getTopKClasses(predictions, topK) {
const values = predictions.dataSync();
predictions.dispose();
let predictionList = [];
for (let i = 0; i < values.length; i++) {
predictionList.push({value: values[i], index: i});
}
predictionList = predictionList
.sort((a, b) => {
return b.value - a.value;
})
.slice(0, topK);
return predictionList.map(x => {
return {label: IMAGENET_CLASSES[x.index], value: x.value};
});
}
}
module.exports = MobileNet;
test.js
var tfc = require('#tensorflow/tfjs-core');
var MobileNet = require('./mobilenet');
var fs = require('fs');
var image = require('get-image-data')
var i = 0;
var meta;
image('./cat.jpg', function(err, getImageData){
if(err) throw err;
console.log('start to image data ');
console.log(i++);
console.log("meta : " + getImageData.data.length);
console.log("getImageData :"+getImageData);
const mobileNet = new MobileNet();
console.time('Loading of model');
// await mobileNet.load();
console.timeEnd('Loading of model');
console.log("maybee this is error on the data type");
const pixels = tfc.fromPixels(image);
console.time('First prediction');
let result = mobileNet.predict(pixels);
const topK = mobileNet.getTopKClasses(result, 5);
console.timeEnd('First prediction');
resultElement.innerText = '';
topK.forEach(x => {
resultElement.innerText += `${x.value.toFixed(3)}: ${x.label}\n`;
});
console.time('Subsequent predictions');
result = mobileNet.predict(pixels);
mobileNet.getTopKClasses(result, 5);
console.timeEnd('Subsequent predictions');
mobileNet.dispose();
});
I want to analyze the image using the tensorflow.js.
But it doesn't work.
ReferenceError: ImageData is not defined
at MathBackendCPU.fromPixels (/Users/leeyongmin/Documents/tfjs-converter-master-2/demo/node_modules/#tensorflow/tfjs-core/dist/kernels/backend_cpu.js:75:31)
at Engine.fromPixels (/Users/leeyongmin/Documents/tfjs-converter-master-2/demo/node_modules/#tensorflow/tfjs-core/dist/engine.js:292:29)
at ArrayOps.fromPixels (/Users/leeyongmin/Documents/tfjs-converter-master-2/demo/node_modules/#tensorflow/tfjs-core/dist/ops/array_ops.js:195:41)
at /Users/leeyongmin/Documents/tfjs-converter-master-2/demo/node_modules/#tensorflow/tfjs-core/dist/ops/operation.js:11:61
at Object.Tracking.tidy (/Users/leeyongmin/Documents/tfjs-converter-master-2/demo/node_modules/#tensorflow/tfjs-core/dist/tracking.js:36:22)
at Object.descriptor.value [as fromPixels] (/Users/leeyongmin/Documents/tfjs-converter-master-2/demo/node_modules/#tensorflow/tfjs-core/dist/ops/operation.js:11:26)
at /Users/leeyongmin/Documents/tfjs-converter-master-2/demo/test.js:26:22
at /Users/leeyongmin/Documents/tfjs-converter-master-2/demo/node_modules/get-image-data/index.js:18:7
at load (/Users/leeyongmin/Documents/tfjs-converter-master-2/demo/node_modules/get-image/server.js:18:5)
at FSReqWrap.readFileAfterClose [as oncomplete] (fs.js:511:3)