What needs to happen is that from my React web app there's this form and data inputted should be sent to Google Spreadsheet.
Is there away to avoid using Timestamp when inserting new data in the spreadsheet?
How do I update a SPECIFIC row?
For example, i have Joe and I need to update his number to 456
| NAME | NUMBER |
|-------|---------|
|Joe |123 |
|Brian |420 |
|Raine |143 |
How do I do this coming from the web app?
I've tried this but still no luck
https://mashe.hawksey.info/2018/02/google-apps-script-patterns-conditionally-updating-rows-of-google-sheet-data-by-reading-and-writing-data-once/
This is from the web app with some code in the componentDidMount()
This code is used to the form data and to prevent the page from redirecting to the App script link. Basically it stays on the page and even reloads the form.
componentDidMount() {
console.log("Contact form submission handler loaded successfully.");
// bind to the submit event of our form
var forms = document.querySelectorAll("form.gform");
for (var i = 0; i < forms.length; i++) {
forms[i].addEventListener("submit", handleFormSubmit, false);
}
//document.addEventListener("DOMContentLoaded", loaded, false);
function validateHuman(honeypot) {
if (honeypot) { //if hidden form filled up
console.log("Robot Detected!");
return true;
} else {
console.log("Welcome Human!");
}
}
// get all data in form and return object
function getFormData(form) {
var elements = form.elements;
var honeypot;
var fields = Object.keys(elements).filter(function (k) {
if (elements[k].name === "honeypot") {
honeypot = elements[k].value;
return false;
}
return true;
}).map(function (k) {
if (elements[k].name !== undefined) {
return elements[k].name;
// special case for Edge's html collection
} else if (elements[k].length > 0) {
return elements[k].item(0).name;
}
}).filter(function (item, pos, self) {
return self.indexOf(item) === pos && item;
});
var formData = {};
fields.forEach(function (name) {
var element = elements[name];
// singular form elements just have one value
formData[name] = element.value;
// when our element has multiple items, get their values
if (element.length) {
var data = [];
for (var i = 0; i < element.length; i++) {
var item = element.item(i);
if (item.checked || item.selected) {
data.push(item.value);
}
}
formData[name] = data.join(', ');
}
});
// add form-specific values into the data
formData.formDataNameOrder = JSON.stringify(fields);
formData.formGoogleSheetName = form.dataset.sheet || "responses"; // default sheet name
//formData.formGoogleSend = form.dataset.email || ""; // no email by default
console.log(formData);
return { data: formData, honeypot };
}
function handleFormSubmit(event) { // handles form submit without any jquery
event.preventDefault(); // we are submitting via xhr below
var form = event.target;
var formData = getFormData(form);
var data = formData.data;
// If a honeypot field is filled, assume it was done so by a spam bot.
if (formData.honeypot) {
return false;
}
disableAllButtons(form);
var url = form.action;
var xhr = new XMLHttpRequest();
xhr.open('POST', url);
// xhr.withCredentials = true;
xhr.setRequestHeader("Content-Type", "application/x-www-form-urlencoded");
xhr.onreadystatechange = function () {
console.log(xhr.status, xhr.statusText);
console.log(xhr.responseText);
form.reset();
return;
};
// url encode form data for sending as post data
var encoded = Object.keys(data).map(function (k) {
return encodeURIComponent(k) + "=" + encodeURIComponent(data[k]);
}).join('&');
xhr.send(encoded);
}
function disableAllButtons(form) {
var buttons = form.querySelectorAll("button");
for (var i = 0; i > buttons.length; i++) {
buttons[i].disabled = true;
}
}
}
This is the snippet form in the web app
<form className="gform" method="POST"
action="GOOGLE APP SCRIPT">
<div className="row Late-body-container">
<div className="col-6">
<select name="Message" className="form-control reason-text" id="selectLateReason" onClick={this.checkOption} onChange={this.loaded}>
<option value="--">--</option>
<option value="Sick Leave">Sick Leave</option>
<option value="Emergency Leave">Emergency Leave</option>
<option value="Family Affairs">Family Affairs</option>
<option value="To travel">To travel</option>
<option value="Other reasons">Other reasons</option>
</select>
</div>
<button type="submit" className="btn btn-send" id="btnSubmit" value="Submit" >Send</button>
</div>
</form>
and then finally this is the Google app script code
/******************************************************************************
* This tutorial is based on the work of Martin Hawksey twitter.com/mhawksey *
* But has been simplified and cleaned up to make it more beginner friendly *
* All credit still goes to Martin and any issues/complaints/questions to me. *
******************************************************************************/
// if you want to store your email server-side (hidden), uncomment the next line
// var TO_ADDRESS = "example#email.net";
// spit out all the keys/values from the form in HTML for email
// uses an array of keys if provided or the object to determine field order
function formatMailBody(obj, order) {
var result = "";
if (!order) {
order = Object.keys(obj);
}
// loop over all keys in the ordered form data
for (var idx in order) {
var key = order[idx];
result += "<h4 style='text-transform: capitalize; margin-bottom: 0'>" + key + "</h4><div>" + sanitizeInput(obj[key]) + "</div>";
// for every key, concatenate an `<h4 />`/`<div />` pairing of the key name and its value,
// and append it to the `result` string created at the start.
}
return result; // once the looping is done, `result` will be one long string to put in the email body
}
// sanitize content from the user - trust no one
// ref: https://developers.google.com/apps-script/reference/html/html-output#appendUntrusted(String)
function sanitizeInput(rawInput) {
var placeholder = HtmlService.createHtmlOutput(" ");
placeholder.appendUntrusted(rawInput);
return placeholder.getContent();
}
function doPost(e) {
try {
Logger.log(e); // the Google Script version of console.log see: Class Logger
record_data(e);
// shorter name for form data
var mailData = e.parameters;
// names and order of form elements (if set)
var orderParameter = e.parameters.formDataNameOrder;
var dataOrder;
if (orderParameter) {
dataOrder = JSON.parse(orderParameter);
}
// determine recepient of the email
// if you have your email uncommented above, it uses that `TO_ADDRESS`
// otherwise, it defaults to the email provided by the form's data attribute
var sendEmailTo = (typeof TO_ADDRESS !== "undefined") ? TO_ADDRESS : mailData.formGoogleSendEmail;
// send email if to address is set
if (sendEmailTo) {
MailApp.sendEmail({
to: String(sendEmailTo),
subject: "Contact form submitted",
// replyTo: String(mailData.email), // This is optional and reliant on your form actually collecting a field named `email`
htmlBody: formatMailBody(mailData, dataOrder)
});
}
return ContentService // return json success results
.createTextOutput(
JSON.stringify({"result":"success",
"data": JSON.stringify(e.parameters) }))
.setMimeType(ContentService.MimeType.JSON);
} catch(error) { // if error return this
Logger.log(error);
return ContentService
.createTextOutput(JSON.stringify({"result":"error", "error": error}))
.setMimeType(ContentService.MimeType.JSON);
}
}
/**
* record_data inserts the data received from the html form submission
* e is the data received from the POST
*/
function record_data(e) {
var lock = LockService.getDocumentLock();
lock.waitLock(30000); // hold off up to 30 sec to avoid concurrent writing
try {
Logger.log(JSON.stringify(e)); // log the POST data in case we need to debug it
// select the 'responses' sheet by default
var doc = SpreadsheetApp.getActiveSpreadsheet();
var sheetName = e.parameters.formGoogleSheetName || "responses";
var sheet = doc.getSheetByName(sheetName);
var oldHeader = sheet.getRange(1, 1, 1, sheet.getLastColumn()).getValues()[0];
var newHeader = oldHeader.slice();
var fieldsFromForm = getDataColumns(e.parameters);
var row = [new Date()]; // first element in the row should always be a timestamp
// loop through the header columns
for (var i = 1; i < oldHeader.length; i++) { // start at 1 to avoid Timestamp column
var field = oldHeader[i];
var output = getFieldFromData(field, e.parameters);
row.push(output);
// mark as stored by removing from form fields
var formIndex = fieldsFromForm.indexOf(field);
if (formIndex > -1) {
fieldsFromForm.splice(formIndex, 1);
}
}
// set any new fields in our form
for (var i = 0; i < fieldsFromForm.length; i++) {
var field = fieldsFromForm[i];
var output = getFieldFromData(field, e.parameters);
row.push(output);
newHeader.push(field);
}
// more efficient to set values as [][] array than individually
var nextRow = sheet.getLastRow() + 1; // get next row
sheet.getRange(nextRow, 1, 1, row.length).setValues([row]);
// update header row with any new data
if (newHeader.length > oldHeader.length) {
sheet.getRange(1, 1, 1, newHeader.length).setValues([newHeader]);
}
}
catch(error) {
Logger.log(error);
}
finally {
lock.releaseLock();
return;
}
}
function getDataColumns(data) {
return Object.keys(data).filter(function(column) {
return !(column === 'formDataNameOrder' || column === 'formGoogleSheetName' || column === 'formGoogleSendEmail' || column === 'honeypot');
});
}
function getFieldFromData(field, data) {
var values = data[field] || '';
var output = values.join ? values.join(', ') : values;
return output;
}
As mentioned earlier
I have Joe and I need to update his number to 456
| NAME | NUMBER |
|-------|---------|
|Joe |123 |
|Brian |420 |
|Raine |143 |
so the Expected result should be that
| NAME | NUMBER |
|-------|---------|
|Joe |456 |
|Brian |420 |
|Raine |143 |
Additional note:
The code works but when I insert new data
I can't get rid of the Timestamp. How do I get rid of it?
I'm dividing this answer in two parts:
1. Get rid of the Timestamp
As you see in your Google Appscript code:
var row = [new Date()]; // first element in the row should always be a timestamp
You just have to remove the new Date() function at leave it as an empty array declaration:
var row = []; //There is no first element yet.
Right after this there is a for loop whose index is starting from 1 to avoid the Timestamp, but since we have removed it there is no need to start at 1, so we have to change it.
for (var i = 0; i < oldHeader.length; i++) { // Now we start at 0 instead of 1
var field = oldHeader[i];
var output = getFieldFromData(field, e.parameters);
row.push(output);
....
2. Updating a cell depending on the condition
Since you already know the Name's condition (Joe) to update the Number, we just have to compare the values of each cell in A with the String 'Joe':
function updateCell() {
var doc = SpreadsheetApp.getActiveSpreadsheet(); //this line is not necessary if you just paste this code in yours
var names = doc.getRange("A2:A").getValues(); //We store all the names from A2 to the last one
var newValue = '456';
for (var n = 2; n < names.length + 2; n++){ //We start at 2 since A1 and B1 are the headers of the table
if (names[n - 2] == 'Joe'){ //But the array's first position is 0, so we deduct 2 form n.
doc.getRange("B" + (n)).setValue(newValue);
}
}
}
Result:
Related
I am trying to send the dummy values using Gscript code to my google spreadsheet.
Code:
function doGet(e) {
Logger.log( JSON.stringify(e) ); // view parameters
var result = 'Ok'; // assume success
if (e.parameter == 'undefined')
{
result = 'No Parameters';
}
else {
var sheet_id = 'google sheet ID /d<ID>/edit'; //
Spreadsheet ID
var sheet = SpreadsheetApp.openById(sheet_id).getActiveSheet();
var newRow = sheet.getLastRow() + 1;
var rowData = [];
rowData[0] = new Date(); //
Timestamp in column A
for (var param in e.parameter) {
Logger.log('In for loop, param=' + param);
var value = stripQuotes(e.parameter[param]);
Logger.log(param + ':' + e.parameter[param]);
switch (param) {
case 'tempC': //Parameter
rowData[1] = value; //Value in column B
result = 'Written on column B';
break;
case 'tempF': //Parameter
rowData[2] = value; //Value in column B
result = 'Written on column C';
break;
case 'humiD': //Parameter
rowData[3] = value; //Value in column C
result += ' ,Written on column D';
break;
default:
result = "unsupported parameter";
}
}
Logger.log(JSON.stringify(rowData));
// Write new row below
var newRange = sheet.getRange(newRow, 1, 1, rowData.length);
newRange.setValues([rowData]);
}
// Return result of operation
return ContentService.createTextOutput(result);
}
/**
* Remove leading and trailing single or double quotes
*/
function stripQuotes( value ) {
return value.replace(/^["']|['"]$/g, "");
}
//-----------------------------------------------
// End of file
//-----------------------------------------------
I am new with this work and whenever I am using script id URL to post the dummy data to google spreadsheet
like below:
https://script.google.com/macros/s/<*GScript ID>/exec?tempC=1&tempF=2&humiD=3
then the error of unsupported parameters shown at my execution screen and even Logger. logs show "undefined" message.
If someone has worked with this kind of issue before please share your advice will be a great much help.
I couldn't find anything on SO that matched my question. I'm using Sheetjs plugin to convert an excel sheet into json, and displaying it using jquery in the browser. I'm able to do the conversion and display, but I have a use-case where I need to validate each of the json rows with data returned from a jquery ajax 'GET' call.
I'm able to perform that validation as well. Once each excel json row is validated against the values from the ajax response, based on a set of rules, the excel json row is marked either a success row or an error row. For success rows, I perform no action. For error row, I need to add an additional key/value pair in the json element, denoting the error type, and the error description. Further, this error row, when displayed in the browser needs to have a css style with a color:red for red text, to indicate an error.
I haven't seen anything in Sheetjs documentation that might allow me to do this, but I'm pretty sure it can be done. In the code below, I have to modify the helper function called BindTable() in order to add the css style to set the text color to red IF it is an error row. I also have to somehow add a for each of the error rows in order to display the error type and error description.
In the below code, I need to be able to display the invalidRequests JSON object with the css style applied to display the text in red color. Or, if there is a way to directly manipulate the exceljson JSON object to somehow append the key/value pairs of MSG1/message to each of the error rows, that would be even better. I realize that due to the nature of this question, I can't create a jsfiddle, but any ideas/suggestion/comments would be extremely helpful, even if it doesn't provide the complete solution.
Expected format:
author1 JOHN DOE USA N.AMERICA
ERROR: THIS AUTHOR NAME ALREADY EXISTS IN THE SYSTEM!
This is the code that I currently have:
//Excel Reader
function ExcelToTable(event) {
event.preventDefault();
var regex = /^([a-zA-Z0-9\s_\\.\-:])+(.xlsx|.xls)$/;
/*Checks whether the file is a valid excel file*/
if (regex.test($("#excelfile").val().toLowerCase())) {
var xlsxflag = false; /*Flag for checking whether excel is .xls
format or .xlsx format*/
if ($("#excelfile").val().toLowerCase().indexOf(".xlsx") > 0) {
xlsxflag = true;
}
/*Checks whether the browser supports HTML5*/
if (typeof (FileReader) != "undefined") {
var reader = new FileReader();
reader.onload = function (e) {
var data = e.target.result;
//pre-process data
var binary = "";
var bytes = new Uint8Array(data);
var length = bytes.byteLength;
for(var i=0;i<length;i++){
binary += String.fromCharCode(bytes[i]);
}
// /pre-process data
/*Converts the excel data in to object*/
if (xlsxflag) {
// var workbook = XLSX.read(data, { type: 'binary' });
var workbook = XLSX.read(binary, {type: 'binary'});
}
else {
var workbook = XLS.read(binary, { type: 'binary' });
}
/*Gets all the sheetnames of excel in to a variable*/
var sheet_name_list = workbook.SheetNames;
// console.log('Sheet name list : ' + sheet_name_list);
var cnt = 0; /*This is used for restricting the script to
consider only first sheet of excel*/
// sheet_name_list.forEach(function (y) { /*Iterate through
all sheets*/
/*Convert the cell value to Json*/
if (xlsxflag) {
exceljson =
XLSX.utils.sheet_to_json(workbook.Sheets['CUSTOM_EXCEL_TAB'],{defval:
"NULL"});
var emptyAuthorCells =[];
var invalidCountryCells = [];
Object.keys(exceljson).forEach(function(value, key) {
if(exceljson[key].AUTHOR == 'ADD'){
}
else if(exceljson[key].AUTHOR == 'NULL'){
emptyAuthorCells.push({'MARKET':
exceljson[key].MARKET, 'REGION':exceljson[key].REGION,
'PARTNER':exceljson[key].PARTNER, 'AUTHOR': exceljson[key].AUTHOR });
}
//check effective end date
if((exceljson[key].DATE_ENDING != '') ||
(exceljson[key].DATE_ENDING <= getTodayDate())){
invalidCountryCells.push({
'MARKET': exceljson[key].MARKET,
'REGION':exceljson[key].REGION, 'PARTNER':exceljson[key].PARTNER, 'AUTHOR':
exceljson[key].AUTHOR
});
}
});
var emptyActionCellsMessage = "There were " +
emptyAuthorCells.length + " rows with Author=Null <br />";
var completedActionCellsMessage = " Success! There
were " + emptyAuthorCells.length + " rows with authro=Null <br />";
var invalidDateMsg = "There are missing or incorrect
date values.";
var validCompareDataMessage = "Success! All data has been successfully validated!";
var invalidCompareDataMessage = "Validation Failed!
Data does not match Rules.";
}
else {
var exceljson =
XLS.utils.sheet_to_row_object_array(workbook.Sheets[y]);
}
var conflictRows = [];
var returnedRows = [];
var errorReturnedRows = [];
if(emptyAuthorCells.length == 0){
var uniqueAuthor = $.unique(exceljson.map(function
(d){
return d.MARKET;
}));
var doAllValidations = function(){
var ajaxList = [];
var ajxIndex = 1;
$.each(uniqueAuthor, function (index, value){
var jqResponse =
$.ajax({
type: "get",
url: "authorlist.cfm?method=getlist&name=" +
value,
dataType: "json"
});
ajaxList.push(jqResponse);
jqResponse.then(
function( apiResponse ){
$.each (apiResponse, function (cc) {
if(apiResponse[cc].hasOwnProperty('SUCCESS')){
errorReturnedRows.push({
'success':
apiResponse[cc].SUCCESS,
'message':
apiResponse[cc].MESSAGE,
'country_code' : value
});
}
else{
returnedRows.push(apiResponse[cc]);
}
// }
// }
});
}
);
});
return ajaxList;
};
// /LOOP OVER country_code
}
var invalidRequests = [];
var validRequests = [];
$(function() {
var ajaxCalls = doAllValidations();
//begin apply
$.when.apply($, ajaxCalls).done(function(){
//console.log(ajaxList);
$('#hidReturnedRows').val();
$('#hidReturnedRows').val(JSON.stringify(returnedRows));
if (exceljson.length > 0 && cnt == 0) {
if((emptyAuthorCells.length != 0) ||
(errorReturnedRows.length!=0) ) {
//data is invalid
console.log("data is invalid");
$('#displayErrors tr
td.previewSuccessClass').html("");
$('#displayErrors tr
td.previewErrorsClass').html(emptyActionCellsMessage);
$('#export-file').addClass('hidebtn');
}
else{
//outer loop
var found = false;
var book_found = false;
var response_validation_errors = [];
var message = "The author's zone is
incorrect";
var message2 = "This book already
exists";
$.each(exceljson, function(x, ej){
// console.log("inside outer
loop");
found = false;
$.each(returnedRows, function(y,
rr){
//compare inner row with outer
row to make sure they're the same
if(rr.AUTHOR_ID == ej.ID &&
rr.AUTHOR_NAME == ej.NAME)
{
if((rr.AUTHOR ==
ej.NATIVE_AUTHOR) && (rr.BOOK_QUALITY == ej.AUTHOR_ZONE)){
// console.log("found!");
found = true;
}
}
});
if(found){
invalidRequests.push({
"AUTHOR": ej.NAME,
"AUTHOR_ZONE":
ej.AUTHOR_ZONE,
"COUNTRY": ej.COUNTRY
});
}
else{
validRequests.push(ej);
}
});
// /outer loop
}
BindTable(exceljson, '#exceltable');
cnt++;
}
})();
//end apply
});
};
if (xlsxflag) {/*If excel file is .xlsx extension than creates a
Array Buffer from excel*/
reader.readAsArrayBuffer($("#excelfile")[0].files[0]);
}
else {
reader.readAsBinaryString($("#excelfile")[0].files[0]);
}
}
else {
alert("Sorry! Your browser does not support HTML5!");
}
}
else {
alert("Please upload a valid Excel file!");
}
}
//Helper funcs
function BindTable(jsondata, tableid, invalidreqs) {/*Function used to convert the JSON
array to Html Table*/
var columns = BindTableHeader(jsondata, tableid); /*Gets all the column
headings of Excel*/
//ADDED .map() & .find() INSTEAD OF NESTED LOOPS
jsondata.map(a => {
// SEARCH FOR AN ELEMENT IN invalidreqs THAT MATCH THE
// CRITERIA TESTED FOR IN THE FUNCTION
if (invalidreqs.find(b => {
return a.AUTHOR == b.AUTHOR && a.BOOKNAME == b.BOOKNAME && a.COUNTRY ==
b.COUNTRY;
})) {
a.MSG = "THIS ROW ALREADY EXISTS";
}
});
console.log (jsondata);
//THE BELOW CODE NEEDS TO BE CHANGED
var row$ = $('<tr/>');
for (var colIndex = 0; colIndex < columns.length; colIndex++) {
var cellValue = jsondata[i][columns[colIndex]];
row$.append($('<td/>').html(cellValue));
}
//console.log("before table append");
$(tableid).append(row$);
if( has_error ){
row$.addClass( 'response-errors' );//add class to make text red
var error_row = $('<tr/>');
var error_cell = $('<td/>');
error_cell.attr('colspan', column.length); //set cols to span lenght of row
error_cell.html("SET ERROR MESSAGE TO DISPLAY BASED ON invalidreq object");
error_row.append( error_cell );
$( tableid ).append( error_row );
}
}
// /Outer loop
}
function BindTableHeader(jsondata, tableid) {/*Function used to get all
column names from JSON and bind the html table header*/
var columnSet = [];
var headerTr$ = $('<tr/>');
for (var i = 0; i < jsondata.length; i++) {
var rowHash = jsondata[i];
for (var key in rowHash) {
if (rowHash.hasOwnProperty(key)) {
if ($.inArray(key, columnSet) == -1) {/*Adding each unique
column names to a variable array*/
columnSet.push(key);
// console.log(key);
headerTr$.append($('<th/>').html(key));
}
}
}
}
$(tableid).append(headerTr$);
return columnSet;
}
Ok so what you want to do is:
1) Assign the row index to the invalidRequests object, on line 191 like this:
invalidRequests.push({
"AUTHOR": ej.NAME,
"AUTHOR_ZONE": ej.AUTHOR_ZONE,
"COUNTRY": ej.COUNTRY,
"index": x,
"MSG1": "Put the error message here"
});
Now it is very easy to determine which row has an error.
Since the invalidRequests is a private object of the ExcelTable function, you will need to
2) pass it on to the BindTable function like this:
BindTable(exceljson, '#exceltable', invalidRequests);
3) modify the BindTable function to check for invalidRequests and handle them:
function BindTable(jsondata, tableid, invalidreqs) {
var columns = BindTableHeader(jsondata, tableid);
for (var i = 0; i < jsondata.length; i++) {
//look for rows with error
var has_error = false
var invalidreq
for(var u=0;u<invalidreqs.length;u++){
if(i==invalidreqs[u].index){
//found invalid request belonging to current row, set flag
has_error = true
invalidreq = invalidreqs[u] // and store the current invalidrequest infos on temp obj
//break - not really needed
}
}
var row$ = $('<tr/>');
for (var colIndex = 0; colIndex < columns.length; colIndex++) {
var cellValue = jsondata[i][columns[colIndex]];
row$.append($('<td/>').html(cellValue));
}
$(tableid).append(row$);
if(has_error){
row$.addClass('error') // add css class which will make the text red or whatever
var error_row = $('<tr/>') // create error row
var error_cell = $('<td/>')
error_cell.attr('colspan',columns.length) // set column to span over all columns of table
error_cell.html(invalidreq.MSG1)
error_row.append(error_cell)
$(tableid).append(error_row);
}
}
}
Please note it is not clear, nor specified in your code, in which column the error should appear. Try to implement that yourself by pushing that info into the invalidRequests object and reading it out on BindTable.
I'm writing little snippets to learn more about using Javascript with API's, and have stumbled onto another problem I can't figure out on my own. I have a global variable (object?) "coins", read in from the API, and its' data field "symbol". I can use "symbol" to reference the data held there, in part of my code, without any errors. Later in the code, I use it again, and I get an error about it being undefined, despite the fact that the values returned from using it, are both defined, and, what I expected. While we are at it, maybe someone can tell me why I assign values to global variables (declared outside of all of the functions), but the variables when called, are "undefined". To see it in action, visit www.mattox.space/XCR and open up dev tools.
/*
FLOW:
get ALL coins, store NAME and SYMBOL into an object.
loop over the names object comparing to $SYMBOL text from form, return the NAME when found.
hit the API again, with the $NAME added to the URL.
create a table row.
insert data from second API hit, into table row
SOMEWHERE in there, do the USD conversion from BTC.
*/
//var name = getName();
var bitcoinValue = 0;
var coins = new Array;
var form = ""; // Value pulled from the form
var symbol = ""; // "id" on the table
var id = ""; // value pulled from the table at coins[i].id matched to coins[i].symbol
var formSym = "";
var formUSD = 0;
var formBTC = 0;
var form24h = 0;
function run() {
getFormData();
allTheCoins("https://api.coinmarketcap.com/v1/ticker/");
testGlobal();
}
function testGlobal() {
console.log("These are hopefully the values of the global variables");
console.log(formSym + " testGlobal");
console.log(formUSD + " testGlobal");
console.log(formBTC + " testGlobal");
console.log(form24h + " testGlobal");
}
function getFormData(){ //This function works GREAT!
form = document.getElementById("symbol").value //THIS WORKS
form = form.toUpperCase(); //THIS WORKS
}
function allTheCoins(URL) {
var tickerRequest = new XMLHttpRequest();
tickerRequest.open('GET', URL);
tickerRequest.send();
tickerRequest.onload = function() {
if (tickerRequest.status >= 200 && tickerRequest.status < 400) {
var input = JSON.parse(tickerRequest.responseText);
for(var i in input)
coins.push(input[i]);
testFunction(coins);
}
else {
console.log("We connected to the server, but it returned an error.");
}
console.log(formSym + " allTheCoins!"); // NOPE NOPE NOPE
console.log(formUSD) + " allTheCoins!"; // NOPE NOPE NOPE
console.log(formBTC + " allTheCoins!"); // NOPE NOPE NOPE
console.log(form24h + " allTheCoins!"); // NOPE NOPE NOPE
}
}
function testFunction(coins) {
for (var i = 0; i < coins.length; i++) {
if (coins[i].symbol == form) { // But right here, I get an error.
formSym = coins[i].name;
formUSD = coins[i].price_usd;
formBTC = coins[i].price_btc;
form24h = coins[i].percent_change_24h;
console.log(formSym + " testFunction");
console.log(formUSD + " testFunction");
console.log(formBTC + " testFunction");
console.log(form24h + " testFunction");
//DO EVERYTHING RIGHT HERE! On second thought, no, this needs fixed.
}
else if (i > coins.length) {
formSym = "Error";
formUSD = 0;
formBTC = 0;
form24h = 0;
}
}
}
/*
if (24h >= 0) {
colorRED
}
else {
colorGreen
}
*/
here is a possible way of doing it that you can get inspired by. its based on a httpRequest promise that set the headers and method.
let allTheCoins = obj => {
return new Promise((resolve, reject) => {
let xhr = new XMLHttpRequest();
xhr.open(obj.method || obj.method, obj.url);
if (obj.headers) {
Object.keys(obj.headers).forEach(key => {
xhr.setRequestHeader(key, obj.headers[key]);
});
}
xhr.onload = () => {
if (xhr.status >= 200 && xhr.status < 300) {
resolve(xhr.response);
} else {
reject(xhr.statusText);
}
};
xhr.onerror = () => reject(xhr.statusText);
xhr.send(obj.body);
});
};
allTheCoins({
url: "https://api.coinmarketcap.com/v1/ticker/",
method: "GET",
headers: {"Accept-Encoding": "gzip"}
})
.then(data => {
ParseCoins(data);
})
.catch(error => {
console.log("We connected to the server, but it returned an error.");
});
function ParseCoins(data) {
const coins = JSON.parse(data);
const form = getFormVal();/*retrieve form val*/
const id = getTableId(); /*retrieve table id*/
const bitcoinValue = getBitcoinVal();/*retrieve bitcoin Value*/
const final_result = [];
for (let i = 0, len = coins[0].length; i < len; i++) {
const coin = coins[0][i];
for (let ii in coin) {
if (coin.hasOwnProperty(ii)) {
if (coin[ii].symbol == form) {
let element = {
formSym: coin[ii].name,
formUSD: coin[ii].price_usd,
formBTC: coin[ii].price_btc,
form24h: coin[ii].percent_change_24h
};
final_result.push(element);
}
}
}
}
coontinueElseWhere(final_result);
}
I have a script that uses AJAX/PHP/SQL to query data and pushes it into an array with a bunch of IF's statements. The changeData function is called every 6 seconds. The first query I return I have 6 arrays. The second time i send a request, my push array(IsVacant1) is double and went to 12. after a while, I have over 500 arrays going into my .each statement.
How do I 'clear' this every time I make a request so that I am not adding arrays? Any help is most appreciated.
function changeData() {
isPaused = true;
var mydata0 = null;
$.post('php/ProductionChange.php', {
'WC': cc
}, function(data) { // This is Where I use an AJAX call into a php file.
mydata0 = data; // This takes the array from the call and puts it into a variable
var pa = JSON.parse(mydata0); // This parses the data into arrays and elements
var temp = {};
var bayData = '';
if (pa != null) {
for (var i = 0; i <= pa.length - 1; i++) {
var job = pa[i][0];
var shipdate = pa[i][1];
var status = pa[i][2];
var name = pa[i][3];
var EnclLoc = pa[i][13];
var Enclsize = pa[i][14];
var backpan = pa[i][15];
var percentCom = pa[i][16];
var IsVisible = pa[i][17];
var png = pa[i][18];
var WorkC = pa[i][20];
baydata = 'bayData' + i + '';
temp = {
job, shipdate, name, EnclLoc, Enclsize, backpan, percentCom, IsVisible, png, WorkC, status
};
isVacant1.push({
baydata: temp
});
}
} else {
ii = 1;
//alert("There are no more job numbers in this bay location. Thank you. ");
}
$.each(isVacant1, function(key, value) {
var job = value.baydata.job;
var ship = value.baydata.shipdate;
var name = value.baydata.name;
var encl = value.baydata.EnclLoc;
var EnclSize = value.baydata.EnclLoc;
var percentCom = value.baydata.percentCom;
var backpan = value.baydata.backpan;
var PngLogo = value.baydata.png;
var IsVisible = value.baydata.IsVisible;
var WorkC = value.baydata.WorkC;
var status = value.baydata.status;
var p = WorkC;
WorkC = (WorkC < 10) ? ("0" + WorkC) : WorkC;
//// remember if the encl location matches the workcell cell then do stuff based on that....... hint encl image not hiding becase of duplicate 17s
if (((encl == p) || (backpan == p)) && job != 123) {
$('#WC' + p).show();
document.getElementById("bayData" + p).innerHTML = name + ' ' + ship; // Work Cell Name and Ship Date
document.getElementById("bayData" + p + "a").innerHTML = job; // Work cell Job Number
document.getElementById("percentCom" + p).innerHTML = percentCom + '%'; // Work Cell Percent Complete
} else {
$('#WC' + p).hide();
From your question it looks like you want to clear the isVacant1 array.
In your ajax callback just put isVacant1 = []; as the first line. Like this
function(data) { // This is Where I use an AJAX call into a php file.
isVacant1 = [];
mydata0 = data; // This takes the array from the call and puts it into a variable
var pa = JSON.parse(mydata0); // This parses the data into arrays and elements
var temp = {};
var bayData = '';
..................
From your code it's not clear how you are declaring/initializing isVacant1 so i have suggested isVacant1 = [] otherwise you can also use isVacant1.length = 0.
You can also take a look here How do I empty an array in JavaScript?
I am attempting to create a web scraper (in node.js) that will pull down information from a site, and write it to a file. I have it built to correctly work for one page, but when I try to use the function in a for loop, to iterate through multiple games, I get bad data in all of the games.
I understand that this is related to Javascript's asynchronous nature, and I have read about callback functions, but I'm not sure I understand how to apply it to my code. Any help would be GREATLY appreciated:
for(x = 4648; x < 4650; x++){ //iterate over a few gameIDs, used in URL for request
scrapeGame(x);
}
function scrapeGame(gameId){
//request from URL, scrape HTML to arrays as necessary
//write final array to file
}
Essentially, what I am looking to do, is within the for loop, tell it to WAIT to finish the scrapeGame(x) function before incrementing x and running it for the next game -- otherwise, the arrays start to overwrite each other and the data becomes a huge mess.
EDIT: I've now included the full code which I am attempting to run! I'm getting errors when looking in the files after they are written. For example, the first file is 8kb, second is ~16, 3rd is ~32, etc. It seems things aren't getting cleared before running the next game.
Idea of the program is to pull Jeopardy questions/answers from the archive site in order to eventually build a quiz app for myself.
//Iterate over arbitrary number of games, scrape each
for(x = 4648; x < 4650; x++){
scrapeGame(x, function(scrapeResult) {
if(scrapeResult){
console.log('Scrape Successful');
} else {
console.log('Scrape ERROR');
}
});
}
function scrapeGame(gameId, callback){
var request = require('request');
cheerio = require('cheerio');
fs = require('fs');
categories = [];
categorylist = [];
ids = [];
clues = [];
values = ['0','$200','$400','$600','$800','$1000','$400','$800','$1200','$1600','$2000'];
valuelist = [];
answers = [];
array = [];
file = [];
status = false;
var showGameURL = 'http://www.j-archive.com/showgame.php?game_id=' + gameId;
var showAnswerURL = 'http://www.j-archive.com/showgameresponses.php?game_id=' + gameId;
request(showGameURL, function(err, resp, body){
if(!err && resp.statusCode === 200){
var $ = cheerio.load(body);
//add a row to categories to avoid starting at 0
categories.push('Category List');
//pull all categories to use for later
$('td.category_name').each(function(){
var category = $(this).text();
categories.push(category);
});
//pull all clue IDs (coordinates), store to 1d array
//pull any id that has "stuck" in the string, to prevent duplicates
$("[id*='stuck']").each(function(){
var id = $(this).attr('id');
id = id.toString();
id = id.substring(0, id.length - 6);
ids.push(id);
//if single J, pick category 1-6
if (id.indexOf("_J_") !== -1){
var catid = id.charAt(7);
categorylist.push(categories[catid]);
var valId = id.charAt(9);
valuelist.push(values[valId]);
}
//if double J, pick category 7-12
else if (id.indexOf("_DJ_") !== -1){
var catid = parseInt(id.charAt(8)) + 6;
categorylist.push(categories[catid]);
var valId = parseInt(id.charAt(10)) + 5;
valuelist.push(values[valId]);
}
//if final J, pick category 13
else {
categorylist.push(categories[13]);
}
});
//pull all clue texts, store to 1d array
$('td.clue_text').each(function(){
var clue = $(this).text();
clues.push(clue);
});
//push pulled values to big array
array.push(ids);
array.push(categorylist);
array.push(valuelist);
array.push(clues);
//new request to different URL to pull responses
request(showAnswerURL, function(err, resp, body){
if(!err && resp.statusCode === 200){
var $ = cheerio.load(body);
$('.correct_response').each(function(){
var answer = $(this).text();
answers.push(answer);
});
//push answers to big array
array.push(answers);
//combine arrays into 1-d array to prep for writing to file
for(var i = 0; i < array[0].length; i++){
var print = array[0][i] + "|" + array[1][i] + "|" + array[2][i] + "|" + array[3][i] + "|" + array[4][i];
var stringPrint = print.toString();
file.push(stringPrint);
}
//update string, add newlines, etc.
var stringFile = JSON.stringify(file);
stringFile = stringFile.split('\\').join('');
stringFile = stringFile.split('","').join('\n');
//write to file, eventually will append to end of one big file
fs.writeFile('J_GAME_' + gameId +'.txt', stringFile, function(err) {
if(err) {
console.log(err);
} else {
console.log("Game #" + gameId + " has been scraped.");
status = true;
}
});
}
});
}
});
//clear arrays used
valuelist = [];
answers = [];
categories = [];
categorylist = [];
ids = [];
clues = [];
array = [];
file = [];
//feed callback status
callback(status);
}
// Iterate over a few gameIDs, used in URL for request.
for (x = 4648; x < 4650; x++) {
// Pass in the callback as an anonymous function.
// So below I am passing in the id and the function I want to execute.
// AND, defining the results I am expecting as passed in arguments.
scrapeGame(x, function(scrapeResult, err) {
// This will *NOT* execute *UNTIL* you call it in the function below.
// That means that the for loop's execution is halted.
// This function receives the status that is passed in,
// in this case, a boolean true/false and an error if any.
if (scrapeResult) {
// Scrape was true, nothing to do.
// The for loop will now move on to the next iteration.
console.log('Scrape Successful');
} else {
// Scrape was false, output error to console.log and
// break loop to handle error.
console.log('Scrape ERROR :: ' + err);
// Notice we are calling break while in the
// scope of the callback function
// Remove the break if you want to just move onto
// the next game ID and not stop the loop
break;
}
});
}
// This function now accepts two arguments.
function scrapeGame(gameId, callback) {
// ************************************************
// ** Do Your Work Here **
// Request from URL, scrape HTML to arrays as necessary.
// Write final array to file.
// After file creation, execute the callback and pass bool
// status (true/false).
// ************************************************
var request = require('request'),
cheerio = require('cheerio'),
fs = require('fs'),
categories = [],
categorylist = [],
ids = [],
clues = [],
values = [
'0',
'$200',
'$400',
'$600',
'$800',
'$1000',
'$400',
'$800',
'$1200',
'$1600',
'$2000'
],
valuelist = [],
answers = [],
array = [],
file = [],
showGameURL = 'http://www.j-archive.com/showgame.php?game_id=' + gameId,
showAnswerURL = 'http://www.j-archive.com/showgameresponses.php?game_id=' + gameId;
request(showGameURL, function(err, resp, body) {
if (!err && resp.statusCode === 200) {
var $ = cheerio.load(body);
//add a row to categories to avoid starting at 0
categories.push('Category List');
//pull all categories to use for later
$('td.category_name').each(function() {
var category = $(this).text();
categories.push(category);
});
//pull all clue IDs (coordinates), store to 1d array
//pull any id that has "stuck" in the string, to prevent duplicates
$("[id*='stuck']").each(function() {
var id = $(this).attr('id');
id = id.toString();
id = id.substring(0, id.length - 6);
ids.push(id);
//if single J, pick category 1-6
if (id.indexOf("_J_") !== -1) {
var catid = id.charAt(7);
categorylist.push(categories[catid]);
var valId = id.charAt(9);
valuelist.push(values[valId]);
}
//if double J, pick category 7-12
else if (id.indexOf("_DJ_") !== -1) {
var catid = parseInt(id.charAt(8)) + 6;
categorylist.push(categories[catid]);
var valId = parseInt(id.charAt(10)) + 5;
valuelist.push(values[valId]);
}
//if final J, pick category 13
else {
categorylist.push(categories[13]);
}
});
//pull all clue texts, store to 1d array
$('td.clue_text').each(function() {
var clue = $(this).text();
clues.push(clue);
});
//push pulled values to big array
array.push(ids);
array.push(categorylist);
array.push(valuelist);
array.push(clues);
//new request to different URL to pull responses
request(showAnswerURL, function(err, resp, body) {
if (!err && resp.statusCode === 200) {
var $ = cheerio.load(body);
$('.correct_response').each(function() {
var answer = $(this).text();
answers.push(answer);
});
//push answers to big array
array.push(answers);
//combine arrays into 1-d array to prep for writing to file
for (var i = 0; i < array[0].length; i++) {
var print = array[0][i] + "|" + array[1][i] + "|" + array[2][i] + "|" + array[3][i] + "|" + array[4][i];
var stringPrint = print.toString();
file.push(stringPrint);
}
//update string, add newlines, etc.
var stringFile = JSON.stringify(file);
stringFile = stringFile.split('\\').join('');
stringFile = stringFile.split('","').join('\n');
//write to file, eventually will append to end of one big file
fs.writeFile('J_GAME_' + gameId + '.txt', stringFile, function(err) {
//clear arrays used
valuelist = [];
answers = [];
categories = [];
categorylist = [];
ids = [];
clues = [];
array = [];
file = [];
if (err) {
// ******************************************
// Callback false with error.
callback(false, err);
// ******************************************
} else {
console.log("Game #" + gameId + " has been scraped.");
// ******************************************
// Callback true with no error.
callback(true);
// ******************************************
}
});
}
});
}
});
}
My assumption is that you want them to be scraped one after one, not in parallel. So, for loop won't help. The following approach should do the trick:
var x = 4648;
var myFunc = scrapeGame(x, function cb(){
if(x >= 4650){
return;
}
x++;
return myFunc(x, cb);
});
function scrapeGame(gameId){
//request from URL, scrape HTML to arrays as necessary
//write final array to file
}
For nested async function, where you want them be executed in serial manner, you should just forget about for loop.
An example of correct request handling with http client:
function scrapeGame(gameId, cb){
//your code and set options
http.request(options, function(response){
var result = "";
response.on('data', function (chunk) {
result += chunk;
});
response.on('end',function(){
//write data here;
//do the callback
cb();
});
});
}
I solved the ROOT cause of the issue that I was seeing, though I do believe without the callback assistance from red above, I would have been just as lost.
Turns out the data was processing correctly, but the file write was scrambling. Turns out that there is a different method to call instead of writeFile or appendFile:
fs.appendFileSync();
Calling the Synchronous version processed the writes to the file IN THE ORDER they got appended to the file, instead of just going for it. This, in addition to the callback help above, solved the issue.
Thanks to everyone for the assistance!