I have implemented the following code to parse a csv file, convert to a JSON array and send the JSON result to apex controller, which invokes the batch class to process the DML operation for opportunityLineItem object. The code is working fine up to a maximum of 4000 rows (files have 22 columns with values). When there are 5000 records, the process throws an error and stops (it does not call the apex server). Why does it stop if there are 4000 records? Is there any limit for parsing the csv records in LWC?
Code:
if (!this.csvFile) {
console.log("file not found");
return;
}
this.showprogressbar = true;
let reader = new FileReader();
let ctx = this; // to control 'this' property in an event handler
reader.readAsText(this.csvFile, "Shift_JIS");
reader.onload = function (evt) {
console.log('reader:'+evt.target.result);
let payload = ctx.CSV2JSON(evt.target.result, ctx.CSVToArray);
let json = null;
let error = null;
console.log("payload:" + payload);
setJSON({
payload: payload,
crud: crud,
csvFile:ctx.csvFile
})
.then((result) => {
json = result;
var err = json.includes("Error");
console.log('err====='+err);
if(err)
{
console.log('json==###=='+json);
ctx.error = json;
console.log("error:"+ctx.error);
///s/ alert('error');
ctx.showloader=false;
ctx.hasError=true;
}
else{
ctx.jobinfo(json);
console.log("apex call setJSON() ===> success: " + json);
//ctx.error = undefined;
}
})
.catch((error) => {
error =ctx.error;
console.log("error:" + error.errorCode + ', message ' + error.message);
///s/ alert('error');
ctx.showloader=false;
ctx.hasError=true;
if (error && error.message) {
json = "{'no':1,'status':'Error', 'msg':'" + error.message + "'}";
} else {
json = "{'no':1,'status':'Error','msg':'Unknown error'}";
}
});
};
reader.onerror = function (evt) {
/*ctx.mycolumns = [
{ label: "no", fieldName: "no", type: "number", initialWidth: 50 },
{
label: "status",
fieldName: "status",
type: "text",
initialWidth: 100
},
{ label: "message", fieldName: "msg", type: "text" }
];
ctx.mydata = [{ no: "1", status: "Error", msg: "error reading file" }];
*/
//$A.util.toggleClass(spinner, "slds-hide"); // hide spinner
};
// ctx.showloader=false;
console.log("mydata:===" + ctx.mydata);
alert('onerror');
}
CSV2JSON(csv, csv2array) {
let array = csv2array(csv);
//console.log("csv:::"+csv);
//console.log("csv2array:"+csv2array);
let objArray = [];
//console.log("objArray:"+objArray);
var headervar = oppheader;//'Name,BillingCity,Type,Industry';
console.log('headervar:::'+headervar);
let headerArray = headervar.split(',');
for (let i = 1; i < array.length; i++) {
objArray[i - 1] = {};
/*for (let k = 0; k < array[0].length && k < array[i].length; k++) {
let key = array[0][k];
if(key === 'DW予定日')
elseif(key === 'DW予定日')
elseif(key === 'DW予定日')
console.log("key:"+key);
this.hearder=key;
objArray[i - 1][key] = array[i][k];
}*/
for (let k = 0; k < headerArray.length; k++) {
let key = headerArray[k];
console.log("key====:"+key);
this.hearder=key;
objArray[i - 1][key] = array[i][k];
}
}
objArray.pop();
//console.log("objArray:==="+objArray.length);
this.rowCount = objArray.length;
//console.log("rowCount+++++++" + this.rowCount);
let json = JSON.stringify(objArray);
//console.log("json:==="+json.length);
let str = json.replace("/},/g", "},\r\n");
//console.log("str:======="+str);
return str;
}
CSVToArray(strData, strDelimiter) {
console.log('CSVToArray');
// Check to see if the delimiter is defined. If not,
// then default to comma.
//console.log('strData:'+strData);
//console.log("strDelimiter::" + strDelimiter);
strDelimiter = strDelimiter || ",";
//console.log("strDelimiter:" + strDelimiter);
// Create a regular expression to parse the CSV values.
var objPattern = new RegExp(
// Delimiters.
"(\\" +
strDelimiter +
"|\\r?\\n|\\r|^)" +
// Quoted fields.
'(?:"([^"]*(?:""[^"]*)*)"|' +
// Standard fields.
'([^"\\' +
strDelimiter +
"\\r\\n]*))",
"gi"
);
// Create an array to hold our data. Give the array
// a default empty first row.
// console.log("objPattern:" + objPattern);
var arrData = [[]];
// Create an array to hold our individual pattern
// matching groups.
// console.log("arrData:" + arrData);
var arrMatches = null;
// Keep looping over the regular expression matches
// until we can no longer find a match.
while ((arrMatches = objPattern.exec(strData))) {
// Get the delimiter that was found.
var strMatchedDelimiter = arrMatches[1];
// Check to see if the given delimiter has a length
// (is not the start of string) and if it matches
// field delimiter. If id does not, then we know
// that this delimiter is a row delimiter.
if (strMatchedDelimiter.length && strMatchedDelimiter != strDelimiter) {
// Since we have reached a new row of data,
// add an empty row to our data array.
arrData.push([]);
}
// Now that we have our delimiter out of the way,
// let's check to see which kind of value we
// captured (quoted or unquoted).
if (arrMatches[2]) {
// We found a quoted value. When we capture
// this value, unescape any double quotes.
var strMatchedValue = arrMatches[2].replace(new RegExp('""', "g"), '"');
} else {
// We found a non-quoted value.
var strMatchedValue = arrMatches[3];
}
// Now that we have our value string, let's add
// it to the data array.
arrData[arrData.length - 1].push(strMatchedValue);
}
// Return the parsed data.
return arrData;
}
Related
I'm using nodejs-mysql module to do query in node.js recently, and in my working case I could only use the parameter-binding syntax like:
SELECT * FROM table WHERE name = ?
Now I want to build dynamic sql with these ? OR ?? parameters. Assume that I have 2 conditions(name and age) which either of them could be null (if user doesn't provide it),
So I want to build MySQL in 3 cases:
only name=Bob: SELECT * FROM table WHERE name = 'Bob'
only age=40: SELECT * FROM table WHERE age > 40
both: SELECT * FROM table WHERE name = 'Bob' AND age > 40
I know it's easy if you build the query on your own, but how can I achieve it when using placeholders which can only bind field or values ?
In document of nodejs-mysql, placeholder ? only stands for values and ?? stands for fields:
https://github.com/felixge/node-mysql/#escaping-query-values
https://github.com/felixge/node-mysql/#escaping-query-identifiers
My first thinking of solution is to insert query piece by using these placeholders, but it comes to failure because both ? and ?? will escape my query piece, and my query will be executed incorrectly.
My code so far is as below, which I'm defenitly sure it's not correct because query piece has been escaped:
// achieve paramters from url request
var condition = {};
if(params.name)condition["name"] = ["LIKE", "%" + params.name + "%"];
if(params.age)condition["age"] = parseInt(params.age, 10);
//build query
var sqlPiece = buildQuery(condition);
//try to replace ? with query
var sql = 'SELECT * FROM table WHERE ?';
connection.query(sql, sqlPiece, function(err, results) {
// do things
});
// my own query build function to proceed conditions
function buildQuery(condition) {
var conditionArray = [];
for(var field in condition){
var con = condition[field];
if(con !== undefined){
field = arguments[1] ? arguments[1] + "." + field : field;
var subCondition;
if(con instanceof Array) {
subCondition = field + " " + con[0] + " " + wrapString(con[1]);
}else{
subCondition = field + " = " + wrapString(con);
}
conditionArray.push(subCondition);
}
}
return conditionArray.length > 0 ? conditionArray.join(" AND ") : "1";
}
//wrap string value
function wrapString(value){
return typeof value === "string" ? "'" + value + "'" : value;
}
So is there any way I can fix this problem?
Update
Thanks to Jordan's Offer, it's working, but :
I know building query by string concat is very good, but in my case I can't use that, because I'm using some middleware or handle mysql and controller, so what I can do is to define interface, which is a sql string with placeholders. So, the interface string is predefined before, and I can't modify it during my controller function.
You're off to a really good start, but you may have been overthinking it a bit. The trick is to build a query with placeholders (?) as a string and simultaneously build an array of values.
So, if you have params = { name: 'foo', age: 40 }, you want to build the following objects:
where = 'name LIKE ? AND age = ?';
values = [ '%foo%', 40 ];
If you only have { name: 'foo' }, you'll build these instead:
where = 'name LIKE ?';
values = [ '%foo%' ];
Either way, you can use those objects directly in the query method, i.e.:
var sql = 'SELECT * FROM table WHERE ' + where;
connection.query(sql, values, function...);
How do we build those objects, then? In fact, the code is really similar to your buildQuery function, but less complex.
function buildConditions(params) {
var conditions = [];
var values = [];
var conditionsStr;
if (typeof params.name !== 'undefined') {
conditions.push("name LIKE ?");
values.push("%" + params.name + "%");
}
if (typeof params.age !== 'undefined') {
conditions.push("age = ?");
values.push(parseInt(params.age));
}
return {
where: conditions.length ?
conditions.join(' AND ') : '1',
values: values
};
}
var conditions = buildConditions(params);
var sql = 'SELECT * FROM table WHERE ' + conditions.where;
connection.query(sql, conditions.values, function(err, results) {
// do things
});
For Inserting into MYSQL like DB:
function generateInsertQuery(data, tableName) {
let part1 = `INSERT INTO ${tableName} (`;
let part2 = ")",
part3 = "VALUES (",
part4 = ")";
let tableKeys = "",
tableValues = "";
for (let key in data) {
tableKeys += `${key},`;
tableValues += `'${data[key]}',`
}
tableKeys = tableKeys.slice(0, -1);
tableValues = tableValues.slice(0, -1);
let query = `${part1}${tableKeys}${part2} ${part3}${tableValues}${part4}`;
return query;
}
generateInsertQuery({name: "Sam", tel: 09090909, email: "address#domain.com"}, "Person")
Output:
INSERT INTO Person (name,tel,email) VALUES ('Sam','9090909','address#domain.com');
Code Snippet for Update query:
function generateUpdateQuery(data, tableName, clauseKey, clauseValue) {
let part1 = `UPDATE ${tableName} SET`;
let part2 = `WHERE ${clauseKey} = ${clauseValue};`; //Add any number of filter clause statements here
let updateString = "";
for (let key in data) {
updateString += `${key} = '${data[key]}',`;
}
updateString = updateString.slice(0, -1);
let query = `${part1} ${updateString} ${part2}`;
return query;
}
generateUpdateQuery({
name: "Tanjiro",
tel: 77777777,
email: "tanjiro#demonslayer.com"
}, "Person", "ID", 111);
Output:
UPDATE Person SET name = 'Tanjiro',tel = '77777777',email = 'tanjiro#demonslayer.com' WHERE ID = 111;
I modify your code #Jordan-Running
describe("Test generateFilterQuery", () => {
it("Query filter with params", () => {
let params = []
params.push(Query.generateParams("title", "%_%", "Coding"))
params.push(Query.generateParams("published", "=", true))
console.log(Query.generateFilterQuery(params))
});
});
const qInclude = require('./QueryInclude');
exports.generateParams = (name, eq, value) => {
return {
name: name,
eq: eq, // %_%, %_, _%, =, >, <, !=,
value: value
}
}
exports.generateFilterQuery = (params) => {
let conditions, values = []
let conditionsStr;
if (params.length == 0) {
return false
}
[conditions, values] = qInclude.queryCondition(params)
let build = {
where: conditions.length ?
conditions.join(' AND ') : '1',
values: values
};
let query = 'SELECT * FROM table WHERE ' + build.where;
return [query, build.values]
}
exports.queryCondition = (params) => {
var conditions = [];
var values = [];
params.forEach(item => {
switch (item.eq) {
case '=': {
conditions.push(item.name + " = ?");
values.push(item.value);
break;
}
case '!=': {
conditions.push(item.name + " != ?");
values.push(item.value);
break;
}
case '<': {
conditions.push(item.name + " < ?");
values.push(item.value);
break;
}
case '>': {
conditions.push(item.name + " > ?");
values.push(item.value);
break;
}
case '%_%': {
conditions.push(item.name + " LIKE ?");
values.push("%" + item.value + "%");
break;
}
case '%_': {
conditions.push(item.name + " LIKE ?");
values.push("%" + item.value);
break;
}
case '_%': {
conditions.push(item.name + " LIKE ?");
values.push(item.value + "%");
break;
}
}
});
return [conditions, values]
}
This question already has answers here:
How do I return the response from an asynchronous call?
(41 answers)
Closed 5 years ago.
The server needs to respond to a http get with all the data from separate file. Here I am using a csv parser.
function sendFileData(req, res) {
var result = []
// Convert an csv file to Json entries
for (var i = 0; i < 4; i++) {
var dataArray = []
csvFilePath = ...
time = '"'
value = '""'
fs.createReadStream(csvFilePath)
.pipe(csv({headers: ['date', 'time', 'value']}))
.on('data', function (data) {
date = ...
time = ...
value = ...
dataArray.push('{' + date + time + value + '}')
})
.on('end', function () {
var sensorData = '[' + dataArray + ']'
result.push(sensorData)
})
}
res.send(result)
}
Since the for loop takes some time to finish, the result is always [], so I consider to add setTimeout() and a callback function, but I feel the setTimeout is a bad approach.
function sendFileData(req, res, callback) {
var result = []
// Convert an csv file to Json entries
for (var i = 0; i < 4; i++) {
var dataArray = []
csvFilePath = ...
time = '"'
value = '""'
fs.createReadStream(csvFilePath)
.pipe(csv({headers: ['date', 'time', 'value']}))
.on('data', function (data) {
date = ...
time = ...
value = ...
dataArray.push('{' + date + time + value + '}')
})
.on('end', function () {
var sensorData = '[' + dataArray + ']'
result.push(sensorData)
})
}
setTimeout(function () {
callback(res, result)
}, 1000)
}
function sendData (res, result) {
res.send(result)
}
// calling function
sendFileData(req, res, sendData)
Is there a better way to send all the data after the csv-parser finished reading ?
Send the response only when you have all the data. See the code below, which waits until the length of the result array is 4 (the number of iterations of the loop):
function sendFileData(req, res) {
var result = []
// Convert an csv file to Json entries
for (var i = 0; i < 4; i++) {
var dataArray = []
csvFilePath = ...
time = '"'
value = '""'
fs.createReadStream(csvFilePath)
.pipe(csv({headers: ['date', 'time', 'value']}))
.on('data', function (data) {
date = ...
time = ...
value = ...
dataArray.push('{' + date + time + value + '}')
})
.on('end', function () {
var sensorData = '[' + dataArray + ']'
result.push(sensorData)
remaining -= 1
if (result.length === 4) { // all done
res.send(result)
}
})
}
}
when I print the whole array it's print.but if I try to print element by element it's print as undefined.this is my function. I print the arrays at end of the function.client functions are used to connect ajax API.i tried to get integer id that matching to a specific string from database via ajax functions and push them into the two arrays.
function fetch() {
var arrayForClass = [];//this is a array get undefined at the end
var arrayForMessage = [];//this is a array get undefined at the end
exceptionPattern ="";
receivedData.length = 0;
var queryInfo;
var queryForSearchCount = {
tableName: "LOGANALYZER",
searchParams: {
query: "_eventTimeStamp: [" + from + " TO " + to + "]",
}
};
client.searchCount(queryForSearchCount, function (d) {
if (d["status"] === "success" && d["message"] > 0) {
var totalRecordCount = d["message"];
queryInfo = {
tableName: "LOGANALYZER",
searchParams: {
query: "_eventTimeStamp: [" + from + " TO " + to + "]",
start: 0, //starting index of the matching record set
count: totalRecordCount //page size for pagination
}
};
client.search(queryInfo, function (d) {
var obj = JSON.parse(d["message"]);
if (d["status"] === "success") {
for (var i = 0; i < obj.length; i++) {
if(obj[i].values._level === "ERROR" || obj[i].values._level === "WARN"){
receivedData.push([{
date: new Date(parseInt(obj[i].values._eventTimeStamp)).toUTCString(),
level: obj[i].values._level,
class: obj[i].values._class,
content: obj[i].values._content,
trace: (obj[i].values._trace ? obj[i].values._trace : ""),
timestamp: parseInt(obj[i].values._eventTimeStamp)
}]);
}else{
continue;
}
}
console.log(receivedData);
for (forLoopI = 0; forLoopI < receivedData.length; forLoopI++){
var className = receivedData[forLoopI][0].class;
var strclassname = className.toString();
var messageContent = receivedData[forLoopI][0].content;
queryInfo = {
tableName: "EXCEPTION_CLASS_FOR_ERROR_PATTERNS",
searchParams: {
query: "class_name: "+ strclassname + "",
start: 0, //starting index of the matching record set
count: 1 //page size for pagination
}
};
client.search(queryInfo,function(d){
var obj = JSON.parse(d["message"]);
if (d["status"] === "success") {
var num = obj[0].values.id;
var strnum = num.toString();
arrayForClass.push(strnum);
}else{
$(canvasDiv).html(gadgetUtil.getCustemText("No content to display","error while creating the error pattern" +
" please try again"));
}
},function(error){
console.log(error);
error.message = "Internal server error while data indexing.";
onError(error);
});
queryInfo = {
tableName: "ERROR_MESSAGE_CONTENTS",
searchParams: {
query: "message: \""+ messageContent + "\"",
start: 0, //starting index of the matching record set
count: 1 //page size for pagination
}
};
client.search(queryInfo,function(d){
var obj = JSON.parse(d["message"]);
console.log(obj);
if (d["status"] === "success") {
var num2 = obj[0].values.id;
var strnum2 = num2.toString();
arrayForMessage.push(strnum2);
}else{
$(canvasDiv).html(gadgetUtil.getCustemText("No content to display","error while creating the error pattern" +
" please try again"));
}
},function(error){
console.log(error);
error.message = "Internal server error while data indexing.";
onError(error);
});
}
}
}, function (error) {
console.log(error);
error.message = "Internal server error while data indexing.";
onError(error);
});
}else{
$(canvasDiv).html(gadgetUtil.getCustemText("No content to display","there are no error patterns which include this error" +
" please try another one"));
}
}, function (error) {
console.log(error);
error.message = "Internal server error while data indexing.";
onError(error);
});
console.log("------------------");
for (var j = 0; j < 8; j++) {
console.log(arrayForClass[j]);//prints undefine
}
console.log("------------------");
console.log(arrayForClass[0]); //prints undefine
console.log(arrayForClass);//prints corectly
console.log(arrayForMessage);//printd corectly
}
Your API call is asynchronous, which mean it's continue working to the next line even your call is not finished.
You get undefined because your console.log reference to the not exists yet variable. arrayForClass is empty at that moment, so arrayForClass[0] is not exists.
Next line you get correct result because you console.log to an existing variable, even it's empty at the moment, but your debugger tool is trying to be smart by update it for you in the console when your data came in.
if you really want to see the actual value at that point, you need to somehow make it immutable, for example :
console.log(JSON.parse(JSON.stringify(arrayForClass)));
This is only explain why you get data in the console like that.If you need to use those variable, It's has to be done inside those callback function regarding each calls.
I'm trying to create a parser to parse my coordinate data into json. The data is in a text file in a simple x,y format. I'm trying to get the the text before [i], is that possible with .split()?
Code:
function visualize()
{
if(currDoc == null)
{
var location = window.prompt("Please enter the name of the dataset file, and make sure it is in the data directory. Current supported formats txt.");
location = "data/" + location;
jQuery.get(location, function(data) {
data = data.replace(/\s/g, '');
var length = data.length;
var commaCount = 0;
for(var i=0;i<length;i++)
{
if(data[i] == ",")
{
commaCount += 1;
if(commaCount == 2)
{
//get text before [i]
}
}
}
}, "text").fail(function(){ alert("File not found. Did you enter the file name correctly?") });
}
else
{
alert("A dataset is already visualized");
}
}
If your data is delimited by commas like this x1,y1,x2,y2,...,xn,yn you can use the split function to split the string into tokens. Then you can iterate through them to collect whatever you need from the input.
For example if you need x and y pairs you would do something like this:
function visualize()
{
if(currDoc == null)
{
var location = window.prompt("Please enter the name of the dataset file, and make sure it is in the data directory. Current supported formats txt.");
location = "data/" + location;
jQuery.get(location, function(data) {
data = data.replace(/\s/g, '');
// split the string 'x1,y1,...,xn,yn' into tokens ['x1', 'y1', ... 'xn', 'yn']
var tokens = data.split(',');
// iterate over all tokens using a step of 2 (i += 2)
// Note: if you have an odd number of tokens the last one will be ignored
// (this is by design because you are expecting x,y pairs)
for(var i = 1; i < tokens.length; i += 2)
{
// print the (x,y) pair to the console
console.log("New pair (" + tokens[i-1] + "," + tokens[i] + ")");
}
}, "text").fail(function(){ alert("File not found. Did you enter the file name correctly?") });
}
else
{
alert("A dataset is already visualized");
}
}
I would add a pointer and a second array witch is filled by characters when you hit a comma join the array and you have the previews text,
Does it make any sense.
function visualize()
{
if(currDoc == null)
{
var location = window.prompt("Please enter the name of the dataset file, and make sure it is in the data directory. Current supported formats txt.");
location = "data/" + location;
jQuery.get(location, function(data) {
data = data.replace(/\s/g, '');
var length = data.length;
var commaCount = 0;
var charArray = [];
for(var i=0;i<length;i++)
{
if(data[i] == ",")
{
console.log('text' , charArray.join(''));
charArray = [];
commaCount += 1;
if(commaCount == 2)
{
//get text before [i]
}
}else {
charArry.puch(data[i]);
}
}
}, "text").fail(function(){ alert("File not found. Did you enter the file name correctly?") });
}
else
{
alert("A dataset is already visualized");
}
}
I'm trying to make some update request in PostgreSQL using javascript, but I'm having some trouble with the update method :
What I used to have was this :
var query = client.query("UPDATE COMPANY SET company_name=$1, phone=$2 WHERE company_id = $3", [data.company_name, data.phone, company_id_requested]);
This would update the table company no matter the value of data.company_name or data.phone. The thing is that these values can be undefined, but I don't want to update the tables value to NULL if these values are undefined.
I've tried a new code to solve this problem, but I really don't find it clean, especially since I have to do it for each update method...
var queryText = "UPDATE COMPANY SET ";
var params = [];
if (data.company_name !== undefined) {
if (params.length != 0)
queryText += ", ";
params.push(data.company_name);
queryText += "company_name=$" + params.length;
}
if (data.phone !== undefined) {
if (params.length != 0)
queryText += ", ";
params.push(data.phone);
queryText += "phone=$" + params.length;
}
params.push(company_id_requested);
queryText += " WHERE company_id = $" + params.length;
console.log(queryText);
var query = client.query(queryText, params);
Here I pretty much construct the query. Isn't there any other way to do it?
The following function will generate such an update for you, from a table name and an object with properties to be set:
function createUpdate(table, data) {
if (!table || typeof table !== 'string') {
throw new TypeError("Parameter 'table' must be a non-empty string.");
}
if (!data || typeof data !== 'object') {
throw new TypeError("Parameter 'data' must be an object.");
}
var keys = Object.keys(data)
.filter(function (k) {
return data[k] !== undefined;
});
var names = keys.map(function (k, index) {
return k + ' = $' + (index + 1);
}).join(', ');
var values = keys.map(function (k) {
return data[k];
});
return {
query: 'UPDATE ' + table + ' SET ' + names,
values: values
};
}
Testing:
var obj = {
one: 1,
two: 'two',
last: undefined
};
var update = createUpdate("Company", obj);
This creates object:
{ query: 'UPDATE Company SET one = $1, two = $2', values: [ 1, 'two' ] }
So you can call:
client.query(update.query, update.values);
Try to add the condition into the where clause:
var query = client.query("UPDATE COMPANY SET company_name=$1, phone=$2 WHERE company_id = $3 AND $1 IS NOT NULL AND $2 IS NOT NULL", [data.company_name, data.phone, company_id_requested]);