Develop Reference

Can't connect to MQTT client, using client as global variable [Javascript, MQTT.js]

I'm working with MQTT.js and Mosquitto as my broker.
I'm trying to connect to my broker on the POST method and it works, storing the "Client" on the variable.
The problem is that the code doesn't work on the GET where my client isn't connecting and therefore, I can't subscribe or message to my topics.
I don't know what is the reason for that, but I solved it when I do de client connection inside the GET, but in this case, I don't want to do the connection inside the GET.
That's because the POST response is given when I login in the platform, and if I do the client connection on the GET that is for see the variables, always that I recharge the page It will connect again and then it will subscribe to the same topic, and after that when I send one message, if is subscribed three times to the same topic, it will Post 3 times.
PD: When I print the client inside the GET, It prints the same value that is assigned in the POST
const express = require("express");
const router = express.Router();
const request = require("request");
let mqtt = require('mqtt');
require("../requestMethods/get.js")();
require("../requestMethods/post.js")();
let client;
let options;
const postMqttData = async (string, body) => {
const url = `http://localhost:8000/${string}`;
const data = await postData(url, body)
console.log(data)
}
const getToken = async (string) => {
const url = `http://localhost:3000/api/tokenuser/${string}`
const data = await requestData(url)
const token = data;
//console.log(token)
return token;
}
router.post("/apiClientBroker", (req, res,next) => {
let {user} = req.body;
console.log(user)
getToken(user).then(meta => {
const { value } = meta.token[0];
options = { username: user, password: value, clean: true, keepAlive: 60 }
client = mqtt.connect('mqtt://localhost', options);
//console.log(client)
})
})
router.get("/:user/:project",(req, res, next) => {
// clientBrokerResponse(req.client)
// console.log(client)
const { user } = req.params;
const { project } = req.params;
const uri = hostURL + "/variables/" + user + "/" + project;
request.get(uri, (err, resp, body) => {
//body = JSON.parse(body);
if (err || resp.status == 500) {
res.status(500).send({ ERROR: "Error searching" });
} else {
if (body.length) {
const json= JSON.parse(body);
console.log(client)
client.on('connect', function () {
for(let i= 0; i<json.length;i++){
const {deviceN}= json[i];
const {variableN}= json[i];
const {variableT}=json[i];
const topico = `${user}/${project}/${deviceN}/${variableN}`;
if(variableT==='Independiente'){
client.subscribe(topico, function (err) {
console.log(`suscrito a ${topico}`)
if (err) {
console.log("error en la subscripcion")
}
})
}
}
})
client.on('message', function (topic, message) {
// message is Buffer
json1 = JSON.parse(message.toString()); //de esta manera se convierte el mensaje recibido en un json
console.log(json1);
// if (json1.token === value) {
json2 = { 'value': json1.value }
postMqttData(`${topic}/${json1.token}`, json2)
// } else {
// console.log("Se está enviando a otro usuario el dato")
// }
})
res.status(resp.statusCode).send(body);
} else {
res
.status(404)
.send({ message: "Variables not found for this project" });
}
}
});
});

Async Function working in Express but not NestJs

I initially created a little express server to run a report and file write function.
var ssrs = require('mssql-ssrs');
var fs = require('fs');
const express = require('express')
const app = express()
const port = 3001
app.get('/', (req, res) => {
reportCreation();
res.send('File Created');
})
app.get('/api', (req, res) => {
reportCreation();
res.json({'File Created': true});
})
app.listen(port, () => {
console.log(`Report Api listening at http://localhost:${port}`)
})
The function reportCreation() is an async function which gets a report from a SSRS. This works fine
async function reportCreation() {
var serverUrl = 'http://reportServerName/ReportServer/ReportExecution2005.asmx';
ssrs.setServerUrl(serverUrl);
var reportPath = '/ApplicationPortalReports/TestReportNew';
var fileType = 'word';
var parameters = { ApplicationId: 3, TrainingCardId: 267, PortalPersonId: 52 }
var auth = {
username: 'USERNAME',
password: 'PASSWORD',
domain: 'dmz'
};
try {
var report = await ssrs.reportExecution.getReportByUrl(reportPath, fileType, parameters, auth)
} catch (error) {
console.log(error);
}
console.log(report);
try {
fs.writeFile('ReportApiTest.doc', report, (err) => {
if (!err) console.log('Data written');
});
} catch (error) {
console.log(error);
}
I have been working a lot with NestJs recently and wanted to use the same function but within a NestJs service.
#Injectable()
export class AppService {
async getReport(): Promise<string> {
const serverUrl = 'http://reportServerName/ReportServer/ReportExecution2005.asmx';
ssrs.setServerUrl(serverUrl);
const reportPath = '/ApplicationPortalReports/TestReportNew';
const fileType = 'word';
// var parameters = {appId: 3, ReportInstanceId: 1 }
const parameters = {ApplicationId: 3, TrainingCardId: 267, PortalPersonId: 52 };
const auth = {
username: 'USERNAME',
password: 'PASSWORD',
domain: 'dmz'
};
try {
var report = await ssrs.reportExecution.getReportByUrl(reportPath, fileType, parameters, auth)
} catch (error) {
console.log(error);
}
console.log(report);
// excel = xlsx
// word = doc
// pdf = pdf
try {
fs.writeFile('ReportApiTest.doc', report, (err) => {
if (!err) { console.log('Data written');
return 'File Written Succesfully'}
});
} catch (error) {
console.log(error);
return 'File Write Error'
}
}
}
As you can see the files are almost identical, but when I run it through NestJs I get an error which looks like a problem with the line
var report = await ssrs.reportExecution.getReportByUrl(reportPath, fileType, parameters, auth)
not awaiting. Why does this work with Express and not NestJS? Below is the error from NestJs
buffer.js:219
throw new ERR_INVALID_ARG_TYPE(
^
TypeError [ERR_INVALID_ARG_TYPE]: The first argument must be one of type string, Buffer, ArrayBuffer,
Array, or Array-like Object. Received type undefined
at Function.from (buffer.js:219:9)
at new Buffer (buffer.js:179:17)
at Object.createType3Message (C:\Projects\SSRS-report-api\ssrs-report-api\node_modules\httpntlm\ntlm.js:172:19)
at sendType3Message (C:\Projects\SSRS-report-api\ssrs-report-api\node_modules\httpntlm\httpntlm.js:77:23)
at Immediate._onImmediate (C:\Projects\SSRS-report-api\ssrs-report-api\node_modules\httpntlm\httpntlm.js:101:4)
within the mssql-ssrs node package the getReportByURL looks like this
async function getReportByUrl(reportPath, fileType, params, auth) {
try {
var config = {
binary: true, // very important
username: auth.userName,
password: auth.password,
workstation: auth.workstation,
domain: auth.domain,
url: soap.getServerUrl()
+ "?" + (testReportPath(reportPath).replace(/\s/g, '+'))
+ "&rs:Command=Render&rs:Format=" + reportFormat(fileType)
+ formatParamsToUrl(params)
};
} catch (err) { report.errorHandler(err) }
return new Promise((resolve, reject) => {
config.url = encodeURI(config.url);
httpntlm.post(config, function (err, res) {
if (res.statusCode === 500) { reject(res) }
if (err || res.statusCode !== 200) { reject(err) }
else { resolve(res.body) }
})
})
}
Here is the app.controller.ts
#Controller()
export class AppController {
constructor(private readonly appService: AppService) {}
#Get()
getHello(): Promise<string> {
return this.appService.getReport();
}
}

This is not an answer for the question. But after I see your code, I can see an error you will face in future if await ssrs.reportExecution.getReportByUrl(reportPath, fileType, parameters, auth) failed. Actually you see above error because of this.
The way you used the try catch is really bad.
Here's the way I code it.
#Injectable()
export class AppService {
async getReport(): Promise<string> {
const serverUrl = 'http://reportServerName/ReportServer/ReportExecution2005.asmx';
ssrs.setServerUrl(serverUrl);
const reportPath = '/ApplicationPortalReports/TestReportNew';
const fileType = 'word';
// var parameters = {appId: 3, ReportInstanceId: 1 }
const parameters = {ApplicationId: 3, TrainingCardId: 267, PortalPersonId: 52 };
const auth = {
username: 'USERNAME',
password: 'PASSWORD',
domain: 'dmz'
};
const report = await ssrs.reportExecution.getReportByUrl(reportPath, fileType, parameters, auth)
return new Promise(function(resolve, reject) {
fs.writeFile('ReportApiTest.doc', report, , function(err) {
if (err) reject(err);
resolve("File Created");
});
});
}
And in my controller
#POST
async writeFile() {
try {
const res = await this.appService.getReport();
return res;
} catch(err) {
// handle your error
}
}

I had fudged the code in the node_module changing the userName variable to username and had not done the same in the NestJS version. I forgot I had done that so now it is working.

GitHub rate limiting preventing return of user commit history

I have written code to return the "streak" (how many days in a row) a user has committed to GitHub. Unfortunately, it is recursively making GitHub API requests, which ends up quickly running into rate limiting issues (even with an API token). Is there a better way for me to retrieve this information?
Essentially, what I'm looking for is the "green square" data that displays on a users account page:
My sample code that is running into rate limiting:
const express = require('express');
const request = require('request');
const moment = require('moment');
require('dotenv').config();
const app = express();
const port = 5000;
app.get('/streak/:user', async function (req, res) {
const yesterdaysDate = moment().subtract(1, 'day').format('YYYY-MM-DD');
try {
const streakCountTotal = await checkUserCommitForDate(req.params.user, yesterdaysDate);
res.send({ streakCountTotal });
} catch (error) {
console.log(error);
res.sendStatus(500);
}
});
async function checkUserCommitForDate(user, date) {
const authorOptions = {
url: `https://api.github.com/search/commits?q=author:${user}+author-date:${date}`,
headers: {
'User-Agent': 'request',
'Accept': 'application/vnd.github.cloak-preview',
'Authorization': `token ${process.env.GITHUB_SECRET_ACCESS_TOKEN}`
}
};
const committerOptions = {
url: `https://api.github.com/search/commits?q=committer:${user}+committer-date:${date}`,
headers: {
'User-Agent': 'request',
'Accept': 'application/vnd.github.cloak-preview',
'Authorization': `token ${process.env.GITHUB_SECRET_ACCESS_TOKEN}`
}
};
const githubAuthorResponse = await promisify(request)(authorOptions);
const githubCommitterResponse = await promisify(request)(committerOptions);
const githubAuthorCount = Number(JSON.parse(githubAuthorResponse.body).total_count);
const githubCommitterCount = Number(JSON.parse(githubCommitterResponse.body).total_count);
if (isNaN(githubAuthorCount) || isNaN(githubAuthorCount)) {
throw new Error('GitHub contribution count was not a number. Body of response was:', githubAuthorResponse.body);
} else if (githubAuthorCount + githubCommitterCount > 0) {
const previousDaysDate = moment(date).subtract(1, 'day').format('YYYY-MM-DD');
let streakCounter = await checkUserCommitForDate(user, previousDaysDate);
streakCounter++;
console.log('streakCounter', streakCounter);
return streakCounter;
} else {
return 0;
}
}
function promisify(fn) {
return function (...args) {
return new Promise((resolve, reject) => {
fn(...args, (err, result) => {
if (err) {
console.log('error', err);
return reject(err);
}
resolve(result);
});
});
};
};
app.listen(port, function () {
console.log('listening on port', port);
});

You can extract the contribution list from the svg contribution calendar data used in the profile page :
https://github.com/users/bertrandmartel/contributions
Parsing the xml is then very straightforward and user contributions stats can be processed using only one request. The following is a nodeJS example using cheerio to parse xml :
const express = require('express');
const request = require('request');
const cheerio = require('cheerio')
const app = express();
const port = 5000;
app.get('/streak/:user', function(req, res) {
request('https://github.com/users/' + req.params.user + '/contributions', function(error, response, body) {
if (error) {
console.log(error);
res.sendStatus(500);
return;
}
processContrib(res, body);
});
});
function processContrib(res, body) {
var $ = cheerio.load(body);
var data = [];
$('svg').find('rect').each(function(index, element) {
data.push({
count: parseInt($(element).attr('data-count')),
date: new Date($(element).attr('data-date'))
})
});
var yesterday = new Date();
yesterday.setDate(yesterday.getDate() - 1);
data = data.sort(function(a, b) {
return new Date(b.date) - new Date(a.date);
}).filter(function(el) {
return el.date.getTime() <= yesterday.getTime();
});
var streakCount = 0;
for (var i = 0; i < data.length; i++) {
if (data[i].count == 0) {
break;
}
streakCount++
}
res.status(200).json({
streak: streakCount
});
}
app.listen(port, function() {
console.log('listening on port', port);
});
Note that you can also process contributions stats as shown here
The caveat is that it's not using the official API but the events that are considered a contribution on the calendar API are subject to changes. Using the API, you would need to redefine the same conditions as defined here

Upload images to twitter API from node.js

I'm attempting to post an image onto the twitter api, v1.1
I've tried just about all the example out there, and nothing seems to be able to post it.
include Posting images to twitter in Node.js using Oauth
I'm using the oauth library mentioned there, and I also had jsOauth, which I thought I'd give a shot according to https://gist.github.com/lukaszkorecki/1038408
Nothing has worked, and at this point I'm starting to lose hope on whether I can even do this.
function postStatusWithMedia(status, file) {
var err = new Object();
if(fs.existsSync(file) === false) {
err.message = "File not found :(";
parseTwitterError(err);
} else {
var oauth = OAuth(options = {
"consumerKey": consumer_key,
"consumerSecret": consumer_secret,
"accessTokenKey": access_token,
"accessTokenSecret": access_token_secret
});
callbacks = {
onSuccess : function() {
console.log('upload worked!')
},
onFailure : function() {
console.log('upload failed!');
console.dir(arguments);
}
},
uploadData = {
'status' : status,
'media' : Base64.encode(fs.readFileSync(file))
};
oauth.post('https://api.twitter.com/1.1/statuses/update_with_media.json',uploadData, callbacks.onSuccess, callbacks.onFailure);
return false;
}
}
If it can't be done, can you please explain why?
Otherwise, anything that could lead me to the right direction would be great.

var fs = require('fs');
var request = require('request');
var FormData = require('form-data');
var utf8 = require('utf8');
// Encode in UTF-8
status = utf8.encode(status);
var form = new FormData();
form.append('status', status)
form.append('media[]', fs.createReadStream(file));
// Twitter OAuth
form.getLength(function(err, length){
if (err) {
return requestCallback(err);
}
var oauth = {
consumer_key: consumer_key,
consumer_secret: consumer_secret,
token: access_token,
token_secret: access_token_secret
};
var r = request.post({url:"https://api.twitter.com/1.1/statuses/update_with_media.json", oauth:oauth, host: "api.twitter.com", protocol: "https:"}, requestCallback);
r._form = form;
r.setHeader('content-length', length);
});
function requestCallback(err, res, body) {
if(err) {
throw err;
} else {
console.log("Tweet and Image uploaded successfully!");
}
}
I ended up using request and node-form-data to manually construct a multipart/form-data request and send it with the status request, utf8 was for encoding the status into UTF-8, not doing so caused issues with '<3' and other characters.

I have not tested these code.Its from my colleague.sure the code is working.
Perhaps this will help.
//twitter_update_with_media.js
(function() {
var fs, path, request, twitter_update_with_media;
fs = require('fs');
path = require('path');
request = require('request');
twitter_update_with_media = (function() {
function twitter_update_with_media(auth_settings) {
this.auth_settings = auth_settings;
this.api_url = 'https://api.twitter.com/1.1/statuses/update_with_media.json';
}
twitter_update_with_media.prototype.post = function(status, imageUrl, callback) {
var form, r;
r = request.post(this.api_url, {
oauth: this.auth_settings
}, callback);
form = r.form();
form.append('status', status);
return form.append('media[]', request(imageUrl));
};
return twitter_update_with_media;
})();
module.exports = twitter_update_with_media;
}).call(this);
next file
//upload_to_twitter.js
var tuwm = new twitter_update_with_media({
consumer_key: TWITTER_OAUTH_KEY,
consumer_secret: TWITTER_OAUTH_SECRET,
token: access[0],
token_secret: access[1]
});
media_picture.picture = imageURL;
if (media_picture.picture) {
console.log('with media upload');
request.head(media_picture.picture,
function (error, response, body) {
if (!error && response.statusCode == 200) {
var image_size = response.headers['content-length'];
if (image_size > 2000000) { // 2mb max upload limit
console.log('greater than 2mb');
sendMessageWithoutImage(err, req, res, next, twit, wallpost, access);
} else {
console.log('less than 2mb');
console.log('twitter text', content);
tuwm.post(content, media_picture.picture, function(err, response) {
if (err) {
console.log('error', err);
return next(err);
}
error_parse = JSON.parse(response.body);
console.log('with media response', response.body);
if (error_parse.errors) {
console.log('have errors', error_parse);
res.json({
status: 500,
info: error_parse.errors[0].code + ' ' + error_parse.errors[0].message
});
} else {
res.json({
status: 200,
info: "OK",
id: response.id
});
}
});
}
}
});

Using Node.js to connect to a REST API

Is it sensible to use Node.js to write a stand alone app that will connect two REST API's?
One end will be a POS - Point of sale - system
The other will be a hosted eCommerce platform
There will be a minimal interface for configuration of the service. nothing more.

Yes, Node.js is perfectly suited to making calls to external APIs. Just like everything in Node, however, the functions for making these calls are based around events, which means doing things like buffering response data as opposed to receiving a single completed response.
For example:
// get walking directions from central park to the empire state building
var http = require("http");
url = "http://maps.googleapis.com/maps/api/directions/json?origin=Central Park&destination=Empire State Building&sensor=false&mode=walking";
// get is a simple wrapper for request()
// which sets the http method to GET
var request = http.get(url, function (response) {
// data is streamed in chunks from the server
// so we have to handle the "data" event
var buffer = "",
data,
route;
response.on("data", function (chunk) {
buffer += chunk;
});
response.on("end", function (err) {
// finished transferring data
// dump the raw data
console.log(buffer);
console.log("\n");
data = JSON.parse(buffer);
route = data.routes[0];
// extract the distance and time
console.log("Walking Distance: " + route.legs[0].distance.text);
console.log("Time: " + route.legs[0].duration.text);
});
});
It may make sense to find a simple wrapper library (or write your own) if you are going to be making a lot of these calls.

Sure. The node.js API contains methods to make HTTP requests:
http.request
http.get
I assume the app you're writing is a web app. You might want to use a framework like Express to remove some of the grunt work (see also this question on node.js web frameworks).

/*Below logics covered in below sample GET API
-DB connection created in class
-common function to execute the query
-logging through bunyan library*/
const { APIResponse} = require('./../commonFun/utils');
const createlog = require('./../lib/createlog');
var obj = new DB();
//Test API
routes.get('/testapi', (req, res) => {
res.status(201).json({ message: 'API microservices test' });
});
dbObj = new DB();
routes.get('/getStore', (req, res) => {
try {
//create DB instance
const store_id = req.body.storeID;
const promiseReturnwithResult = selectQueryData('tablename', whereField, dbObj.conn);
(promiseReturnwithResult).then((result) => {
APIResponse(200, 'Data fetched successfully', result).then((result) => {
res.send(result);
});
}).catch((err) => { console.log(err); throw err; })
} catch (err) {
console.log('Exception caught in getuser API', err);
const e = new Error();
if (err.errors && err.errors.length > 0) {
e.Error = 'Exception caught in getuser API';
e.message = err.errors[0].message;
e.code = 500;
res.status(404).send(APIResponse(e.code, e.message, e.Error));
createlog.writeErrorInLog(err);
}
}
});
//create connection
"use strict"
const mysql = require("mysql");
class DB {
constructor() {
this.conn = mysql.createConnection({
host: 'localhost',
user: 'root',
password: 'pass',
database: 'db_name'
});
}
connect() {
this.conn.connect(function (err) {
if (err) {
console.error("error connecting: " + err.stack);
return;
}
console.log("connected to DBB");
});
}
//End class
}
module.exports = DB
//queryTransaction.js File
selectQueryData= (table,where,db_conn)=>{
return new Promise(function(resolve,reject){
try{
db_conn.query(`SELECT * FROM ${table} WHERE id = ${where}`,function(err,result){
if(err){
reject(err);
}else{
resolve(result);
}
});
}catch(err){
console.log(err);
}
});
}
module.exports= {selectQueryData};
//utils.js file
APIResponse = async (status, msg, data = '',error=null) => {
try {
if (status) {
return { statusCode: status, message: msg, PayLoad: data,error:error }
}
} catch (err) {
console.log('Exception caught in getuser API', err);
}
}
module.exports={
logsSetting: {
name: "USER-API",
streams: [
{
level: 'error',
path: '' // log ERROR and above to a file
}
],
},APIResponse
}
//createlogs.js File
var bunyan = require('bunyan');
const dateFormat = require('dateformat');
const {logsSetting} = require('./../commonFun/utils');
module.exports.writeErrorInLog = (customError) => {
let logConfig = {...logsSetting};
console.log('reached in writeErrorInLog',customError)
const currentDate = dateFormat(new Date(), 'yyyy-mm-dd');
const path = logConfig.streams[0].path = `${__dirname}/../log/${currentDate}error.log`;
const log = bunyan.createLogger(logConfig);
log.error(customError);
}

A more easy and useful tool is just using an API like Unirest; URest is a package in NPM that is just too easy to use jus like
app.get('/any-route', function(req, res){
unirest.get("https://rest.url.to.consume/param1/paramN")
.header("Any-Key", "XXXXXXXXXXXXXXXXXX")
.header("Accept", "text/plain")
.end(function (result) {
res.render('name-of-the-page-according-to-your-engine', {
layout: 'some-layout-if-you-want',
markup: result.body.any-property,
});
});