Develop Reference

I'm getting data from the Spotify API, but after a while running it marks an error and starts logging the same information over and over

I'm trying to understand Spotify API. This small code console logs the information of the current beat of the song that the user is listening to.
It works fine but after a few seconds running, it logs an error and after that error it starts logging the information of the same beat over and over again. Any ideas? Here is my code and thank you in advance!
var spotifyApi, spotifyToken;
var current, currentId, timeProgress, beatStart, beatsTotal;
var dataArray = [];
var beatsArray = [];
var spotifyToken = 'token';
spotifyApi = new SpotifyWebApi();
spotifyApi.setAccessToken( spotifyToken );
init();
function init(){
onReceiveCurrent();
}
function onReceiveCurrent ( err, data ) {
if (err) {
console.error(err);
return;
}
spotifyApi.getMyCurrentPlaybackState({}, onReceiveCurrent );
currentId = data.item.id;
timeProgress = data.progress_ms;
spotifyApi.getAudioAnalysisForTrack(currentId, onReceiveAnalysisCallback);
}
function onReceiveAnalysisCallback ( err, data ) {
if (err) {
console.error(err);
return;
}
for (var i=0; i<data.beats.length; i++) {
beatsArray[i] = data.beats[i].start;
}
var found = beatsArray.find(function (element) {
return element >= timeProgress/1000;
});
var a = beatsArray.indexOf(found);
beatStart = data.beats[a];
console.log(beatStart);
}
Here's the error:

You've accidentally created an infinite loop in the onReceiveCurrent method, which calls getMyCurrentPlaybackState which has a callback of onReceiveCurrent. It's calling itself over and over until it hits Spotify's limit and returns a 429.
The callback should be a separate function like so:
function callSpotify () {
spotifyApi.getMyCurrentPlaybackState({}, onReceiveStateCallback );
}
function onReceiveStateCallback(err, data) {
if (err) {
console.error(err);
return;
}
currentId = data.item.id;
timeProgress = data.progress_ms;
spotifyApi.getAudioAnalysisForTrack(currentId, onReceiveAnalysisCallback);
}

Cannot await for sqlite3.Database.get() function completion in Node.js

I'm struggling with some basic async/await problem in node.js using node-sqlite3.
My objective is to select some value from SQLite DB, check it for some condition and take some actions in case the condition is met. Here's the code:
const sqlite3 = require('sqlite3').verbose();
main();
async function main() {
let ordersDb = await createDbConnection('./ProcessedOrders.db');
var orderProcessed = await orderAlreadyProcessed(ordersDb, "555");
console.log("orderProcessed = " + orderProcessed);
if (!orderProcessed) {
console.log("So condition is met!");
}
}
async function orderAlreadyProcessed(ordersDb, orderNumberStr) {
console.log('starting orderAlreadyProcessed function'); //DEBUG
var result;
var query = 'select count(SoldOrderNumber) as "recsCount" from ProcessedSoldOrders where SoldOrderNumber = ?;';
await ordersDb.get(query
,[orderNumberStr]
,(err, row) => {
console.log('Row with count = ' + row); //DEBUG
console.log('row.recsCount = ' + row.recsCount); //DEBUG
result = typeof row !== 'undefined' && row.recsCount > 0;
});
console.log('Returning ' + result); //DEBUG
return result;
}
async function createDbConnection(dbFileName) {
let db = new sqlite3.Database(dbFileName, (err) => {
if (err) {
console.log(err.message);
}
});
return db;
}
But what I get is code executing further, not awaiting for Database.get() method at all! As a result, here's what I see printing in console:
starting orderAlreadyProcessed function
Returning undefined
orderProcessed = undefined
So IF condition met!
Row with count = [object Object]
row.recsCount = 1
As we can see, we return from orderAlreadyProcessed too early with return value = 'undefined'. So condition is met, actions taken, and only then Database.get() returns. But if it was properly awaited, condition would not be met.
How can I make it await for result value?

Since you want to use async/await, and the node-sqlite3 (sqlite3) library does not support the Promise API, you need to use the node-sqlite (sqlite) library, which is a wrapper over sqlite3 and adds support for the Promise API. Then, your code will look something like this:
const sqlite3 = require('sqlite3');
const { open } = require('sqlite');
async function main() {
try {
sqlite3.verbose();
const ordersDb = await createDbConnection('./ProcessedOrders.db');
const orderProcessed = await orderAlreadyProcessed(ordersDb, "555");
console.log("orderProcessed = " + orderProcessed);
if (!orderProcessed) {
console.log("So condition is met!");
}
} catch (error) {
console.error(error);
}
}
async function orderAlreadyProcessed(ordersDb, orderNumberStr) {
try {
console.log('Starting orderAlreadyProcessed function');
const query = 'SELECT COUNT(SoldOrderNumber) as `recsCount` from ProcessedSoldOrders where SoldOrderNumber = ?;'
const row = await ordersDb.get(query, [orderNumberStr]);
console.log('Row with count =', row);
console.log('row.recsCount =', row.recsCount);
const result = typeof row !== 'undefined' && row.recsCount > 0;
console.log('Returning ' + result);
return result;
} catch (error) {
console.error(error);
throw error;
}
}
function createDbConnection(filename) {
return open({
filename,
driver: sqlite3.Database
});
}
main();
I specifically did not remove your console.log and other parts of the code so as not to confuse the original logic of your program.

If we don't to use another library
then we can return a new Promise function & use await, as below:
Note: Below has example for INSERT/run, instead of SELECT/get, but promise/await works same
const sqlite3 = require("sqlite3").verbose();
let db;
db = new sqlite3.Database('./Chinook.db');
function insert() {
return new Promise((resolve, reject) => { // return new Promise here <---
const userId = uuid4();
let sql = `INSERT INTO Users(id) VALUES (?)`; // INSERT <----
let params = [userId];
return db.run(sql, params, function (err, res) { // .run <----
if (err) {
console.error("DB Error: Insert failed: ", err.message);
return reject(err.message);
}
return resolve("done");
});
});
}
let result = await insert(); // now await works fine <------
res.send({ result });

Error: Cannot create a string longer than 0x3fffffe7 characters while using createReadStream

I am parsing extremely large CSV files (~37gbs). I am using fs.createReadStream and csv-parser. I break them into 5000 rows and then insert those into a mongo db. This error happens even when the mongo portion is commented out, however.
Here's the function to parse the file:
function parseCsv(fileName: string, db: Db): Promise<any[]> {
let parsedData: any[] = [];
let counter = 0;
return new Promise((resolve, reject) => {
const stream = fs.createReadStream(fileName)
.pipe(csvParser())
.on('data', async (row) => {
const data = parseData(row);
parsedData.push(data);
if (parsedData.length > 5000) {
stream.pause();
// insert to mongo
counter++;
console.log('counter - ', counter, parsedData[0].personfirstname, parsedData[23].personfirstname);
parsedData = [];
// try {
// await db.collection('people').insertMany(parsedData, { ordered: false });
// parsedData = [];
// }
// catch (e) {
// console.log('error happened', e, parsedData.length);
// process.exit();
// }
stream.resume();
}
})
.on('error', (error) => {
console.error('There was an error reading the csv file', error);
})
.on('end', () => {
console.log('CSV file successfully processed');
resolve()
});
});
}
And here's the function to parse the data. It's kind of messy with all the values in one cell separated by pipes so I just split on them:
function parseData(data: any) {
let values = '';
for (var key in data) {
if (data.hasOwnProperty(key)) {
values += data[key];
}
}
const splitValues = values.split('|');
let parsedData: any = {};
// Remove deep reference
parsedData = JSON.parse(JSON.stringify(template));
let keyCounter = 0;
for (let key in parsedData) {
if (parsedData.hasOwnProperty(key)) {
try {
parsedData[key] = splitValues[keyCounter].trim();
}
catch (e) {
console.log('error probably trimming', key, splitValues[keyCounter], splitValues, data);
throw '';
}
keyCounter++;
}
}
const now = new Date();
parsedData.createdAt = now;
parsedData.updatedAt = now;
return parsedData;
}
It'll parse fine (until ~2 million rows) and then hang. Finally after leaving it hanging all night, I checked in the morning and see the following error:
buffer.js:580
if (encoding === 'utf-8') return buf.utf8Slice(start, end);
^
Error: Cannot create a string longer than 0x3fffffe7 characters
at stringSlice (buffer.js:580:44)
at Buffer.toString (buffer.js:643:10)
at CsvParser.parseValue (C:\js_scripts\csv-worker\node_modules\csv-parser\index.js:175:19)
at CsvParser.parseCell (C:\js_scripts\csv-worker\node_modules\csv-parser\index.js:86:17)
at CsvParser.parseLine (C:\js_scripts\csv-worker\node_modules\csv-parser\index.js:142:24)
at CsvParser._flush (C:\js_scripts\csv-worker\node_modules\csv-parser\index.js:196:10)
at CsvParser.prefinish (_stream_transform.js:140:10)
at CsvParser.emit (events.js:200:13)
at prefinish (_stream_writable.js:633:14)
at finishMaybe (_stream_writable.js:641:5) {
code: 'ERR_STRING_TOO_LONG'
}
Shouldn't createReadStream ensure this doesn't happen? There are 415 columns in this each row. Is it possible that a single row is too big? It always stops at the same place so this seems likely. Since the files are so big I don't have a way to oepn them. If so, how can I detect this and just skip this line or handle it a different way?

reading google spreadsheet data using node js not working..?

I am using postman to call api. I am trying to read google spread sheet row using node js. Response is printing on console but its not returning to postman.
index.js file
app.post('/myapi/getClientkey',async (req, res) => {
var response = null;
console.log("Inside myapi");
try {
response = await spreadsheet.getRecord();
} catch(err){
}
res.send(response);
});
spreadsheet.js
var config = require('./config.json');
var GoogleSpreadsheet = require('google-spreadsheet');
var creds = {
client_email: config.client_email,
private_key: config.private_key
}
var doc = new GoogleSpreadsheet(config.GOOGLE_SHEET_ID)
var sheet = null;
exports.getRecord = async function () {
console.log('Inside - getRecord');
var name = null;
var jsonObj = {};
try {
doc.useServiceAccountAuth(creds, async function (err) {
// Get all of the rows from the spreadsheet.
await doc.getRows(1, async function (err, rows) {
if(rows != null){
var oneRow = rows[0];
name = oneRow.name;
console.log("name :"+name);
jsonObj.client_name = name.toString();
}
console.log(jsonObj);
});
});
}catch(err){
console.log("err :"+err);
}
return jsonObj;
};
How to wait till response is returned from getRecord Function

Move res.send(response); inside try block and read about how to return response from an async call.
Also return jsonObj should be inside try block

I used promise call now it working.
exports.getRecord = async function () {
console.log('Inside - getRecord');
var name = null;
return new Promise( async function(resolve,reject){
try {
var jsonObj = {};
doc.useServiceAccountAuth(creds, async function (err) {
// Get all of the rows from the spreadsheet.
await doc.getRows(1, async function (err, rows) {
if(rows != null){
var oneRow = rows[0];
name = oneRow.name;
console.log("name :"+name);
jsonObj.client_name = name.toString();
}
console.log(jsonObj);
resolve(jsonObj);
});
});
}catch(err){
console.log("err :"+err);
reject();
}
}); // end of promise
};

Async function calls and accumulation handed elegantly or idiomatically

I seem to regularly get stuck on handling async i/o issues and always seem to come up with clunky solutions. See this snippet for my current challenge.
Purpose: AWS Lambda function which reads the contents of a DynamoDB table and writes it to a file stored in S3. This Lambda function gets called whenever there is an update to the DynamoDB table.
Problem: see the commented out section of code in the center of the function onScan? That is to handle the case where it takes multiple calls to dynDoc.scan() to deliver the entire contents of the DynamoDB table. The limit is 100 rows per call. However, ideally the S3 file would be written once with the entire contents of the table have been delivered, not on every call of dynDoc.scan() as the code is currently constructed. This is a challenge with the asynchronous I/O to DynamoDB and the onScan callback. In addition, I clear the variable fileContents each time onScan is executed, because, if you invoke this lambda function twice with less than 5 minutes between, it will stay in memory and the global fileContents will accumulate two copies of the table.
One idea I have is to initialize a counter before the initial call to dynDoc.scan() and then increment the counter for each subsequent call to dynDoc.scan(). I would decrement the counter after the commented out section of code and then test for it to be zero before writing out the S3 file and clearing fileContents.
Is there a more elegant solution? Something more idiomatic Javascript?
Note that Lambda currently supports node.js version 8.10.
Thank you for looking at this!
'use strict';
var AWS = require("aws-sdk");
AWS.config.update({ region: "us-east-1" });
var s3 = new AWS.S3();
var s3Params = {
Body: "testing",
Bucket: "my-bucket",
Key: "my-file.csv"
};
var dyn = new AWS.DynamoDB();
var dynDoc = new AWS.DynamoDB.DocumentClient;
var dynParamsDoc = { TableName: "MyTable" };
var itemCount = 0;
var fileContents = "";
exports.handler = (event, context, callback) => {
function onScan(err,data) {
if (err) {
console.error("Unable to scan Dynamodb.\nError JSON:",
JSON.stringify(err, null, 2));
} else {
fileContents = ""; // added, because it was not getting cleared
data.Items.forEach((entry) => {
fileContents += entry.ClientName + "," + entry.ClientAbbrev + "\n";
});
// eventually, we should really loop on partial DynamoDB table transfers:
// if (typeof data.LastEvaluatedKey != "undefined") {
// console.log("Scanning for more...");
// dynParamsDoc.ExclusiveStartKey = data.LastEvaluatedKey;
// dynDoc.scan(dynParamsDoc, onScan);
// }
// Save S3 file
s3Params.Body = fileContents;
s3.putObject(s3Params, function(err,data) {
if (err) console.log(err,err.stack);
else console.log(data);
});
};
};
// Now retrieve the entire table from DynamoDB and write it to a file
dynDoc.scan(dynParamsDoc, onScan);
callback(null, "Successfully processed table.");
};

In addition, I clear the variable fileContents each time onScan is executed
That I think is the problem. You should not need to clear it - because you should not have used a global (module-scoped, static) variable. You should declare an initialise var fileContents = ""; inside the exports.handler function, not in onScan. With that fixed, I would expect your commented-out approach to work:
var AWS = require("aws-sdk");
AWS.config.update({ region: "us-east-1" });
var s3 = new AWS.S3;
var dyn = new AWS.DynamoDB;
var dynDoc = new AWS.DynamoDB.DocumentClient;
exports.handler = (event, context, callback) => {
var s3Params = {
Body: "testing",
Bucket: "my-bucket",
Key: "my-file.csv"
};
var dynParamsDoc = { TableName: "MyTable" };
var fileContents = "";
function onScan(err,data) {
if (err) {
callback("Unable to scan Dynamodb.\nError JSON:",
JSON.stringify(err, null, 2));
} else {
data.Items.forEach((entry) => {
fileContents += entry.ClientName + "," + entry.ClientAbbrev + "\n";
});
if (typeof data.LastEvaluatedKey != "undefined") {
console.log("Scanning for more...");
dynParamsDoc.ExclusiveStartKey = data.LastEvaluatedKey;
dynDoc.scan(dynParamsDoc, onScan);
} else {
// Save S3 file
s3Params.Body = fileContents;
s3.putObject(s3Params, function(err,data) {
if (err) {
callback(err);
} else {
console.log(data);
callback(null, "Successfully processed table.");
}
});
}
}
}
// Now retrieve the entire table from DynamoDB and write it to a file
dynDoc.scan(dynParamsDoc, onScan);
};
Is there a more elegant solution? Something more idiomatic Javascript?
Yes, a modern approach would use promises with async/await:
var AWS = require("aws-sdk");
AWS.config.update({ region: "us-east-1" });
var s3 = new AWS.S3;
var dyn = new AWS.DynamoDB;
var dynDoc = new AWS.DynamoDB.DocumentClient;
exports.handler = async (event, context) => {
var dynParamsDoc = { TableName: "MyTable" };
var fileContents = "";
do {
var data = await dynDoc.scan(dynParamsDoc).promise();
for (var entry of data.Items) {
fileContents += entry.ClientName + "," + entry.ClientAbbrev + "\n";
}
dynParamsDoc.ExclusiveStartKey = data.LastEvaluatedKey;
} while (typeof data.LastEvaluatedKey != "undefined");
var s3Params = {
Body: "testing",
Bucket: "my-bucket",
Key: "my-file.csv",
Body: fileContents,
};
var res = await s3.putObject(s3Params).promise();
console.log(res);
return "Successfully processed table.";
};

Based only on your code (i.e. I can't assert the general architecture here), you can pass partial content to the recursive call:
// one more arg!
function onScan(err, data, memory = []) {
if (err) {
console.error(...);
return callback(err); // see Bergi's comment on your post
}
// add current data to our "global" data
memory.push.apply(memory, data.Items);
// in case there's more...
if (typeof data.LastEvaluatedKey !== "undefined") {
dynParamsDoc.ExclusiveStartKey = data.LastEvaluatedKey;
// ...pass the "global" data to next scan, and stop here
return dynDoc.scan(dynParamsDoc, (err, res) => {
onScan(err, res, memory);
});
}
// if we got here, we have no more data to fetch, so we address S3 now
s3Params.Body = memory.map((row) => {
return `${row.ClientName},${row.ClientAbbrev}`;
}).join("\n") + "\n"; // that last \n to exactly reproduce your behavior
s3.putObject(s3Params, function(err, data) {
if (err) console.log(err, err.stack);
else console.log(data);
callback(err, "Successfully processed table."); // see Bergi's comment on your post
});
}
dynDoc.scan(dynParamsDoc, onScan);