Working on node.js instagram parser. At the moment I have 1 thread with proxy working code, but I'm not sure how to make multithread architecture:
'use strict';
var InstagramPrivateAPI = {};
InstagramPrivateAPI = {};
InstagramPrivateAPI.V1 = require(__dirname + '/client/v1');
InstagramPrivateAPI.Helpers = require(__dirname + '/helpers');
var acc = require(__dirname + "/client/v1/account");
var med = require(__dirname + "/client/v1/media")
var Promise = require('../bluebird');
var _ = require('../lodash/');
module.exports = InstagramPrivateAPI;
var Client = require('instagram-private-api').V1;
var device = new Client.Device('maksgmn');
var storage = new Client.CookieFileStorage(__dirname + '/cookies/maksgmn.json');
var session = new Client.Session(device, storage);
function getRandomInt(min, max) {
return Math.floor(Math.random() * (max - min)) + min;
}
var fs = require('fs');
var proxyArray = fs.readFileSync('proxy.txt').toString().split("\n");
var usernamesArray = fs.readFileSync('usernames.txt').toString().split("\n");
var proxy = "http://" + proxyArray[getRandomInt(0,proxyArray.length)]
var username = usernamesArray[getRandomInt(0,usernamesArray.length)]
console.log(proxy)
console.log(username)
Client.Request.setProxy(proxy);
acc.searchForUser(session, username) //поиск id пользователя
.then(function(profile) {
return profile.id
})
.then(function(someId) { //получение промиса lenta
var feed = new Client.Feed.UserMedia(session, someId);
var lenta = Promise.mapSeries(_.range(0, 1), function() {
return feed.get();
}).then(function(lenta) {
return {id: someId, fd : lenta}
})
return lenta
})
.then(function(results) { //обработка промиса и получение ленты пользователя
// result should be Media[][]
var media1 = _.flatten(results.fd);
var urls1 = _.map(media1, function(medium) {
//var arr1 = medium.params.images[0].url;
var arr1 = []
try {
arr1 = medium.params.images[0].url
} catch (err) {
//console.log("lala")
}
return arr1;
//console.log(medium.params.carouselMedia.images[0].url)
})
//console.log(urls1)
return {id : results.id, linksNoCarousel : urls1, med : media1}
})
.then(function(res){
var urls2 = _.map(res.med, function(medium) {
var arr2 = []
try {
arr2 = medium.params.images[0][0].url
//console.log(arr2)
} catch (err) {
}
return arr2
})
for (var i = 0; i < 5; i++) {
if (typeof res.linksNoCarousel[i] == "undefined")
res.linksNoCarousel[i] = urls2[i];
}
var arr3 = []
for (var i = 0; i < 5; i++) {
arr3[i] = res.linksNoCarousel[i]
}
return {id : res.id, links : arr3}
})
.then(function(mediaAndId) {
acc = acc.getById(session, mediaAndId.id)
.then(function(account) {
//console.log(account.params)
let avatar = account.params.profilePicUrl;
let fullName = account.params.fullName;
let bio = account.params.biography;
let media0 = mediaAndId.links[0];
let media1 = mediaAndId.links[1];
let media2 = mediaAndId.links[2];
let media3 = mediaAndId.links[3];
let media4 = mediaAndId.links[4];
console.log(avatar);
console.log(fullName);
console.log(bio);
console.log(media0);
console.log(media1);
console.log(media2);
console.log(media3);
console.log(media4);
})
})
I would like it to work like multithread to be much more faster with proxies. As far as I'm working with node.js 2nd day, asking that question: how to do that?
Related
I have two functions that are mostly similar in different files and I want to combine them into one file but as can be seen in the code, some product properties are different from each other and I want to handle this dynamically. What is the correct way to do this?
File-1:
function run(args) {
try {
var sitePreferences = require("sitePreferences");
var pinterestHelpers = require("~/cartridge/scripts/helpers/pinterestHelpers");
var maxRowLength = sitePreferences.get("pinterestProductFeedMaxRow");
var hostName = require("dw/system/Site").current.httpsHostName;
var products = pinterestHelpers.onBeforeStep(args);
var fileNumber = 1;
var lineCount = 1;
pinterestHelpers.removeFiles(args);
var csvObject = pinterestHelpers.createCsvFile(fileNumber, args);
var csv = csvObject.csv;
var fileCreationResponse;
var fileWriter = csvObject.fileWriter;
while (products.hasNext()) {
var currentProductObject = products.next();
var priceObject = pinterestHelpers.getPrice(currentProductObject);
var product = {};
if (pinterestHelpers.shouldBeExported(currentProductObject)) {
if (maxRowLength !== lineCount) {
product.id = pinterestHelpers.getId(currentProductObject);
product.title = pinterestHelpers.getTitle(currentProductObject);
product.description = pinterestHelpers.getDescription(currentProductObject);
product.link = pinterestHelpers.getLink(currentProductObject, hostName);
product.image_link = pinterestHelpers.getImageLink(currentProductObject);
product.price = priceObject && priceObject.list ? priceObject.list : "";
product.availability = pinterestHelpers.getAvailability(currentProductObject);
product.item_group_id = pinterestHelpers.getItemGroupID(currentProductObject);
product.additional_image_link = pinterestHelpers.getAdditionalImageLink(currentProductObject);
product.brand = pinterestHelpers.getBrand(currentProductObject);
product.google_product_category = pinterestHelpers.getGoogleCategoryByProduct(currentProductObject);
product.sale_price = priceObject && priceObject.sale ? priceObject.sale : "";
product.color = pinterestHelpers.getColor(currentProductObject);
product.age_group = pinterestHelpers.getCustomAttribute(currentProductObject, "age_group", "Adult");
product.gender = pinterestHelpers.getCustomAttribute(currentProductObject, "gender", "unisex");
product.condition = pinterestHelpers.getCustomAttribute(currentProductObject, "condition", "new");
product.variant_names = pinterestHelpers.getSize(currentProductObject);
product.variant_values = pinterestHelpers.getColor(currentProductObject);
product.product_type = pinterestHelpers.getProductType(currentProductObject);
product.material = pinterestHelpers.getCustomAttribute(currentProductObject, "material");
product.pattern = pinterestHelpers.getCustomAttribute(currentProductObject, "pattern");
product.shipping = pinterestHelpers.getShipping(currentProductObject, priceObject).group;
product.adult = pinterestHelpers.getCustomAttribute(currentProductObject, "adult", "FALSE");
csv.writeNext([
product.id,
product.title,
product.description,
product.link,
product.image_link,
product.price,
product.availability,
product.item_group_id,
product.variant_names,
product.variant_values,
product.additional_image_link,
product.brand,
product.product_type,
product.sale_price,
product.age_group,
product.color,
product.gender,
product.material,
product.pattern,
product.shipping,
product.adult,
product.condition,
product.google_product_category
]);
lineCount++;
} else {
fileNumber++;
lineCount = 1;
fileCreationResponse = pinterestHelpers.createCsvFile(fileNumber, args);
csv = fileCreationResponse.csv;
}
}
}
products.close();
csv.close();
fileWriter.close();
Logger.info("Pinterest feed exported at {0}", pinterestHelpers.getFeedURL(args, fileCreationResponse.fileName));
return new Status(Status.OK, 'OK', 'Export successful');
} catch (e) {
return new Status(Status.ERROR, 'ERROR', 'Failed to export the feed: ' + e);
}
}
exports.Run = run;
File-2:
function run(args) {
try {
var snapchatHelpers = require("~/cartridge/scripts/helpers/snapchatHelpers");
var maxRowLength = sitePreferences.get("snapchatProductFeedMaxRow");
var hostName = require("dw/system/Site").current.httpsHostName;
var products = snapchatHelpers.onBeforeStep(args);
var fileNumber = 1;
var lineCount = 1;
snapchatHelpers.removeFiles(args);
var csvObject = snapchatHelpers.createCsvFile(fileNumber, args);
var csv = csvObject.csv;
var fileCreationResponse;
var fileWriter = csvObject.fileWriter;
while (products.hasNext()) {
var currentProductObject = products.next();
var priceObject = snapchatHelpers.getPrice(currentProductObject);
var product = {};
if (snapchatHelpers.shouldBeExported(currentProductObject)) {
if (maxRowLength !== lineCount) {
//Required Product Metadata
product.id = snapchatHelpers.getId(currentProductObject);
product.title = snapchatHelpers.getTitle(currentProductObject);
product.description = snapchatHelpers.getDescription(currentProductObject);
product.link = snapchatHelpers.getLink(currentProductObject, hostName);
product.imageLink = snapchatHelpers.getImageLink(currentProductObject);
product.price = priceObject && priceObject.list ? priceObject.list : "";
product.availability = snapchatHelpers.getAvailability(currentProductObject);
product.item_group_id = snapchatHelpers.getItemGroupID(currentProductObject);
product.additional_image_link = snapchatHelpers.getAdditionalImageLink(currentProductObject);
product.brand = snapchatHelpers.getBrand(currentProductObject);
product.google_product_category = snapchatHelpers.getGoogleCategoryByProduct(currentProductObject);
product.sale_price = priceObject && priceObject.sale ? priceObject.sale : "";
product.color = snapchatHelpers.getColor(currentProductObject);
product.age_group = snapchatHelpers.getCustomAttribute(currentProductObject, "age_group", "Adult");
product.gender = snapchatHelpers.getCustomAttribute(currentProductObject, "gender", "Unisex");
product.condition = snapchatHelpers.getCustomAttribute(currentProductObject, "condition", "New");
//Optional Product Metadata
product.product_type = snapchatHelpers.getProductType(currentProductObject);
product.adult = snapchatHelpers.getCustomAttribute(currentProductObject, "adult", "No");
product.size = snapchatHelpers.getSize(currentProductObject);
product.sale_price_effective_date = priceObject && priceObject.startDate ? StringUtils.format("{0}/{1}", priceObject.startDate, priceObject.endDate) : "";
csv.writeNext([
product.id,
product.title,
product.description,
product.link,
product.imageLink,
product.availability,
product.price,
product.brand,
product.age_group,
product.color,
product.condition,
product.gender,
product.item_group_id,
product.google_product_category,
product.product_type,
product.adult,
product.size,
product.additional_image_link,
product.sale_price,
product.sale_price_effective_date
]);
lineCount++;
} else {
fileNumber++;
lineCount = 1;
fileCreationResponse = snapchatHelpers.createCsvFile(fileNumber, args);
csv = fileCreationResponse.csv;
}
}
}
products.close();
csv.close();
fileWriter.close();
Logger.info("Facebook feed exported at {0}", snapchatHelpers.getFeedURL(args, fileCreationResponse.fileName));
return new Status(Status.OK, 'OK', 'Export successful');
} catch (e) {
return new Status(Status.ERROR, 'ERROR', 'Failed to export the feed: ' + e);
}
}
exports.Run = run;
Using inheritance with an Object oriented structure seems logical at first, but since we are working with javascript, I would like to implement it if there is a simpler way by sacrificing object oriented.
Can anyone assist me with loading an array with excel data and returning it as a function? This is my initial code:
var excel = require('exceljs');
var wb = new excel.Workbook();
var path = require('path');
var filePath = path.resolve(__dirname,'data.xlsx');
function signIn(){
var SignIn = [];
wb.xlsx.readFile(filePath).then(function(){
var sh = wb.getWorksheet("Sheet1");
for(var i = 1; i < 3; i++){
SignIn.push(sh.getRow(i).getCell(2).value);
}
});
return SignIn
}
Workbook.readFile is aynchronous, you need to use either a callback or promise type approach. Using promises we can try:
var excel = require('exceljs');
var wb = new excel.Workbook();
var path = require('path');
var filePath = path.resolve(__dirname,'data.xlsx');
function signIn() {
var SignIn = [];
return wb.xlsx.readFile(filePath).then( () => {
var sh = wb.getWorksheet("Sheet1");
for(var i = 1; i < 3; i++){
SignIn.push(sh.getRow(i).getCell(2).value);
}
return SignIn;
});
}
async function testReadData() {
try {
let data = await signIn();
console.log('testReadData: Loaded data: ', data);
} catch (error) {
console.error('testReadData: Error occurred: ', error);
}
}
testReadData();
Or you can use a callback type approach:
function signInWithCallback(callback) {
var SignIn = [];
wb.xlsx.readFile(filePath).then(function(){
var sh = wb.getWorksheet("Sheet1");
for(var i = 1; i < 3; i++){
SignIn.push(sh.getRow(i).getCell(2).value);
}
callback(SignIn);
});
}
signInWithCallback((data) => console.log('Callback: Data: ', data));
How can I handle request fails in this example of axios.all requests. I.e. if all servers are responde with JSON all is okay and I have JSON file at end of a cycle. But if one of this servers not responde with JSON or not responde at all I do have nothing in "/data.json" file, even all other servers are working perfectly. How can I catch a server fail and skip it?
var fs = require("fs");
var axios = require('axios');
var util = require('util');
var round = 0;
var tmp = {};
var streem = fs.createWriteStream(__dirname + '/data.json', {flags : 'w'});
toFile = function(d) { //
streem.write(util.format(d));
};
start();
setInterval(start, 27000);
function start(){
streem = fs.createWriteStream(__dirname + '/data.json', {flags : 'w'});
monitor();
}
function monitor(){
axios.all([
axios.get('server1:api'),
axios.get('server2:api'),
axios.get('server3:api'),
axios.get('server4:api'),
]).then(axios.spread((response1, response2, response3, response4) => {
tmp.servers = {};
tmp.servers.server1 = {};
tmp.servers.server1 = response1.data;
tmp.servers.server2 = {};
tmp.servers.server2 = response2.data;
tmp.servers.server3 = {};
tmp.servers.server3 = response3.data;
tmp.servers.server4 = {};
tmp.servers.server4 = response4.data;
toFile(JSON.stringify(tmp));
round++;
streem.end();
streem.on('finish', () => {
console.error('Round: ' + round);
});
})).catch(error => {
console.log(error);
});
}
The most standard way to approach this would be a recursive function like below.
let promises = [
axios.get('server1:api'),
axios.get('server2:api'),
axios.get('server3:api'),
axios.get('server4:api'),
];
async function monitor() {
const responses = (arguments.length === 1 ? [arguments[0]] : Array.apply(null, arguments))[0];
const nextPromise = promises.shift();
if (nextPromise) {
try {
const response = await getSentenceFragment(offset);
responses.push(response);
}
catch (error) {
responses.push({});
}
return responses.concat(await monitor(responses));
} else {
return responses;
}
}
monitor([]).then(([response1, response2, response3, response4]) => {
tmp.servers = {};
tmp.servers.server1 = {};
tmp.servers.server1 = response1.data;
tmp.servers.server2 = {};
tmp.servers.server2 = response2.data;
tmp.servers.server3 = {};
tmp.servers.server3 = response3.data;
tmp.servers.server4 = {};
tmp.servers.server4 = response4.data;
toFile(JSON.stringify(tmp));
round++;
streem.end();
streem.on('finish', () => {
console.error('Round: ' + round);
});
});
mobilenet.js
var loadFrozenModel = require('#tensorflow/tfjs-converter');
var NamedTensorMap = require('#tensorflow/tfjs-converter');
var tfc = require('#tensorflow/tfjs-core');
var IMAGENET_CLASSES = require('./imagenet_classes');
const GOOGLE_CLOUD_STORAGE_DIR = 'https://storage.googleapis.com/tfjs-models/savedmodel/';
const MODEL_FILE_URL = 'mobilenet_v1_1.0_224/optimized_model.pb';
const WEIGHT_MANIFEST_FILE_URL = 'mobilenet_v1_1.0_224/weights_manifest.json';
const INPUT_NODE_NAME = 'input';
const OUTPUT_NODE_NAME = 'MobilenetV1/Predictions/Reshape_1';
const PREPROCESS_DIVISOR = tfc.scalar(255 / 2);
class MobileNet {
constructor() {}
async load() {
this.model = await loadFrozenModel(
GOOGLE_CLOUD_STORAGE_DIR + MODEL_FILE_URL,
GOOGLE_CLOUD_STORAGE_DIR + WEIGHT_MANIFEST_FILE_URL);
}
dispose() {
if (this.model) {
this.model.dispose();
}
}
predict(input) {
const preprocessedInput = tfc.div(
tfc.sub(input.asType('float32'), PREPROCESS_DIVISOR),
PREPROCESS_DIVISOR);
const reshapedInput =
preprocessedInput.reshape([1, ...preprocessedInput.shape]);
const dict = {};
dict[INPUT_NODE_NAME] = reshapedInput;
return this.model.execute(dict, OUTPUT_NODE_NAME);
}
getTopKClasses(predictions, topK) {
const values = predictions.dataSync();
predictions.dispose();
let predictionList = [];
for (let i = 0; i < values.length; i++) {
predictionList.push({value: values[i], index: i});
}
predictionList = predictionList
.sort((a, b) => {
return b.value - a.value;
})
.slice(0, topK);
return predictionList.map(x => {
return {label: IMAGENET_CLASSES[x.index], value: x.value};
});
}
}
module.exports = MobileNet;
test.js
var tfc = require('#tensorflow/tfjs-core');
var MobileNet = require('./mobilenet');
var fs = require('fs');
var image = require('get-image-data')
var i = 0;
var meta;
image('./cat.jpg', function(err, getImageData){
if(err) throw err;
console.log('start to image data ');
console.log(i++);
console.log("meta : " + getImageData.data.length);
console.log("getImageData :"+getImageData);
const mobileNet = new MobileNet();
console.time('Loading of model');
// await mobileNet.load();
console.timeEnd('Loading of model');
console.log("maybee this is error on the data type");
const pixels = tfc.fromPixels(image);
console.time('First prediction');
let result = mobileNet.predict(pixels);
const topK = mobileNet.getTopKClasses(result, 5);
console.timeEnd('First prediction');
resultElement.innerText = '';
topK.forEach(x => {
resultElement.innerText += `${x.value.toFixed(3)}: ${x.label}\n`;
});
console.time('Subsequent predictions');
result = mobileNet.predict(pixels);
mobileNet.getTopKClasses(result, 5);
console.timeEnd('Subsequent predictions');
mobileNet.dispose();
});
I want to analyze the image using the tensorflow.js.
But it doesn't work.
ReferenceError: ImageData is not defined
at MathBackendCPU.fromPixels (/Users/leeyongmin/Documents/tfjs-converter-master-2/demo/node_modules/#tensorflow/tfjs-core/dist/kernels/backend_cpu.js:75:31)
at Engine.fromPixels (/Users/leeyongmin/Documents/tfjs-converter-master-2/demo/node_modules/#tensorflow/tfjs-core/dist/engine.js:292:29)
at ArrayOps.fromPixels (/Users/leeyongmin/Documents/tfjs-converter-master-2/demo/node_modules/#tensorflow/tfjs-core/dist/ops/array_ops.js:195:41)
at /Users/leeyongmin/Documents/tfjs-converter-master-2/demo/node_modules/#tensorflow/tfjs-core/dist/ops/operation.js:11:61
at Object.Tracking.tidy (/Users/leeyongmin/Documents/tfjs-converter-master-2/demo/node_modules/#tensorflow/tfjs-core/dist/tracking.js:36:22)
at Object.descriptor.value [as fromPixels] (/Users/leeyongmin/Documents/tfjs-converter-master-2/demo/node_modules/#tensorflow/tfjs-core/dist/ops/operation.js:11:26)
at /Users/leeyongmin/Documents/tfjs-converter-master-2/demo/test.js:26:22
at /Users/leeyongmin/Documents/tfjs-converter-master-2/demo/node_modules/get-image-data/index.js:18:7
at load (/Users/leeyongmin/Documents/tfjs-converter-master-2/demo/node_modules/get-image/server.js:18:5)
at FSReqWrap.readFileAfterClose [as oncomplete] (fs.js:511:3)
i want to run at command at my hardware thorugh crome serial api
I use this Opensource code
https://github.com/GoogleChrome/chrome-app-samples/tree/master/servo
this working only for only integer
i want to pass string in the serial port my code is like following
var connectionId = -1;
function setPosition(position) {
var buffer = new ArrayBuffer(1);
var uint8View = new Uint8Array(buffer);
uint8View[0] = '0'.charCodeAt(0) + position;
chrome.serial.write(connectionId, buffer, function() {});
};
function setTxt(data) {
// document.getElementById('txt').innerHTML = data;
var bf = str2ab(data)
chrome.serial.write(connectionId, bf, function() {});
};
function str2ab(str) {
len = str.length;
var buf = new ArrayBuffer(len*2); // 2 bytes for each char
var bufView = new Uint16Array(buf);
for (var i=0, strLen=str.length; i<strLen; i++) {
bufView[i] = str.charCodeAt(i);
}
return buf;
}
function onRead(readInfo) {
var uint8View = new Uint8Array(readInfo.data);
var value1 = String.fromCharCode(uint8View[0])
var value = uint8View[0] - '0'.charCodeAt(0);
var rotation = value * 18.0;
// var dataarr = String.fromCharCode.apply(null, new Uint16Array(readInfo.data));
//alert(rotation);
if(uint8View[0])
document.getElementById('txt').innerHTML = document.getElementById('txt').innerHTML + value1;
document.getElementById('image').style.webkitTransform =
'rotateZ(' + rotation + 'deg)';
// document.getElementById('txt').innerHTML=uint8View[0];
// Keep on reading.
chrome.serial.read(connectionId, 1, onRead);
};
function ab2str(buf) {
return String.fromCharCode.apply(null, new Uint16Array(buf));
}
function onOpen(openInfo) {
connectionId = openInfo.connectionId;
if (connectionId == -1) {
setStatus('Could not open');
return;
}
setStatus('Connected');
setPosition(0);
chrome.serial.read(connectionId, 1, onRead);
};
function setStatus(status) {
document.getElementById('status').innerText = status;
}
function buildPortPicker(ports) {
var eligiblePorts = ports.filter(function(port) {
return !port.match(/[Bb]luetooth/);
});
var portPicker = document.getElementById('port-picker');
eligiblePorts.forEach(function(port) {
var portOption = document.createElement('option');
portOption.value = portOption.innerText = port;
portPicker.appendChild(portOption);
});
portPicker.onchange = function() {
if (connectionId != -1) {
chrome.serial.close(connectionId, openSelectedPort);
return;
}
openSelectedPort();
};
}
function openSelectedPort() {
var portPicker = document.getElementById('port-picker');
var selectedPort = portPicker.options[portPicker.selectedIndex].value;
chrome.serial.open(selectedPort, onOpen);
}
onload = function() {
var tv = document.getElementById('tv');
navigator.webkitGetUserMedia(
{video: true},
function(stream) {
tv.classList.add('working');
document.getElementById('camera-output').src =
webkitURL.createObjectURL(stream);
},
function() {
tv.classList.add('broken');
});
document.getElementById('position-input').onchange = function() {
setPosition(parseInt(this.value, 10));
};
document.getElementById('txt-input').onchange = function() {
setTxt(this.value);
// document.getElementById('txt').innerHTML = this.value;
};
chrome.serial.getPorts(function(ports) {
buildPortPicker(ports)
openSelectedPort();
});
};
string is the passing through serial but this command not run without enter press how to do it any one know
thanks in advance :)
need a \n at the end of string
example:
writeSerial( '#;Bit_Test;*;1;' + '\n' );
take a look at this project http://www.dataino.it/help/document_tutorial.php?id=13