Sending multiple HTTP GET requests to api with a loop - javascript

I'm looking for a way to send many requests to an api using a different api url each time.
An example url for my project is:
http://api.bandsintown.com/artists/Hippo%20Campus/events.json?lapi_version=2.0&app_id=music_matcher
I'm using an HTTP request to pull the JSON info into my script and works perfectly...the first time. However, I want to be able to call it 50-100 ish times (max) in a loop with different artist names in the url (I'm using the BandsInTown API). For some reason, when I try to use a loop to call the http request multiple times, only one output appears and it is unpredictable which element in the order it will be (it's usually the output associated with the first or second element in the array). This is what my code looks like:
// HTTP GET call to BandsInTown API
function httpGetAsync(theUrl, callback) { //theURL or a path to file
var httpRequest = new XMLHttpRequest();
httpRequest.onreadystatechange = function() {
if (httpRequest.readyState == 4 && httpRequest.status == 200) {
var data = JSON.parse(httpRequest.responseText);
if (callback) {
callback(data);
}
}
else {
alert("error loading JSON doc");
}
};
httpRequest.open('GET', theUrl, true);
httpRequest.send(null);
}
//extracts data from api for each artist
function parseEvent(artist) {
var url = "http://api.bandsintown.com/artists/" + artist + "/events.json?lapi_version=2.0&app_id=music_matcher";
httpGetAsync(url, function(data) {
var numEvents = Object.keys(data).length;
//var events = [];
for (var j = 0; j < numEvents; j++) {
document.write(data[j].venue.name + "-> ");
document.write("LAT:" + data[j].venue.latitude + " " + "LNG:" + data[j].venue.longitude);
document.write("ARTIST: " + data[j].artists[0].name);
document.write("DATE: " + data[j].datetime);
document.write(" " + j + " ");
}
});
}
var artists = ["Drake", "Mac Demarco", "Hippo Campus", "STRFKR"];
for (var i = 0; i < artists.length; i++) {
parseEvent(artists[i]);
document.write(" ---NEXT ARTIST--- ");
}
So I can't tell exactly what's going on but things are acting weird with my current code. I don't have a whole lot of javascript and web development experience yet so any help is appreciated! I was preferably looking for a way to implement this with pure javascript. I have had trouble figureing out how to handle Node.js and/or JQuery in Eclipse Neon (the IDE I am using)

You have implemented closure pretty well so clearly this isn't a problem of success callback of one function overwriting response of all others.But now when you look at document.write() it all gets clear, this function first wipes your whole content clean then it writes whatever you told it to .That's why you hardly see anyone use it
`document.write('a');`
`document.write('b');`
`document.write('c');` // a and b are gone you would see only 'c'
So after loop gets over you would only see the output of the last call.Though it's mostly random as to which call would finish last it mostly biased towards some particular value due to the the way servers are tuned.
So better approach is to use some <div> or something and pour your results into it like this one
<div id="op"></div>
and
function parseEvent(artist) {
var url = "http://api.bandsintown.com/artists/" + artist + "/events.json?lapi_version=2.0&app_id=music_matcher";
httpGetAsync(url, function(data) {
var numEvents = Object.keys(data).length;
var op = document.getElementById('op');
op.innerHTML = op.innerHTML + " <br><br> <h2>---NEXT ARTIST---<h2> <br>";
//var events = [];
for (var j = 0; j < numEvents; j++) {
op.innerHTML = op.innerHTML + "<br>" + data[j].venue.name + "-> ";
op.innerHTML = op.innerHTML + "<br>" + "LAT:" + data[j].venue.latitude + " " + "LNG:" + data[j].venue.longitude ;
op.innerHTML = op.innerHTML + "<br>" +"ARTIST: " + data[j].artists[0].name;
op.innerHTML = op.innerHTML + "<br>" +"DATE: " + data[j].datetime;
op.innerHTML = op.innerHTML + "<br>" + " " + j + " <br>";
}
});
}
var artists = ["Drake", "Hippo Campus", "STRFKR","Mac Demarco"];
for (var i = 0; i < artists.length; i++) {
parseEvent(artists[i]);
}

Related

How to sort the loaded XML data

In my application, I have an XML file that connects to different API and gathers information. Then in a div, it shows the fetched information.
I do it successfully, but the issue is, the XML will not load if element one by one. It loads data in parallel, so the divs it creates are not in the right order like how they are in the xml file.
Here is a demo that get some data from local, SoundCloud, Spotify and iTunes.
And this is the XML file it is loading from:
$.ajax({
url: "https://dl.dropboxusercontent.com/u/33538012/text.xml",
dataType: "xml",
success: parse,
error: function() {
alert("Error");
}
});
function parse(document) {
var inc = 1;
$(document).find("track").each(function() {
var rowNum = inc;
var trackType = $(this).find('type').text();
var trackTitle = $(this).find('title').text();
var soundcloud_url = $(this).find('soundcloud_url').text();
var spotify_track_uri = $(this).find('spotify_track_uri').text();
var itunes_id = $(this).find('itunes_id').text();
if (trackType == "audio") {
inc = inc + 1;
$(".playlist").append("<div id='" + rowNum + "' >" + rowNum + " " + trackTitle + "</div>");
} else if (trackType == "soundcloud") {
inc = inc + 1;
SC.initialize({
client_id: "b8f06bbb8e4e9e201f9e6e46001c3acb"
});
SC.resolve(soundcloud_url).then(function(sound) {
trackTitle = sound.title;
$(".playlist").append("<div id='" + rowNum + "' >" + rowNum + " " + trackTitle + "</div>");
});
} else if (trackType == "spotify") {
inc = inc + 1;
var spotifyLink = "https://api.spotify.com/v1/tracks/" + spotify_track_uri;
$.getJSON(spotifyLink, function(data) {
trackTitle = data.name;
$(".playlist").append("<div id='" + rowNum + "' >" + rowNum + " " + trackTitle + "</div>");
});
} else if (trackType == "itunes") {
inc = inc + 1;
var iTuensLink = "https://itunes.apple.com/lookup?id=" + itunes_id + "&callback=?";
$.getJSON(iTuensLink, function(data) {
trackTitle = data.results[0].trackName;
$(".playlist").append("<div id='" + rowNum + "' >" + rowNum + " " + trackTitle + "</div>");
})
}
});
}
<script src="https://connect.soundcloud.com/sdk/sdk-3.1.2.js"></script>
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.1.1/jquery.min.js"></script>
<div class="playlist"></div>
If you look at the result of the code, the the number next to each row shows the order in the original XML file, but they are not loaded exactly one after each other.
Any way to sort the loaded files just like how they are written in the XML file?
You can use deffered object which are an extension of promise and can resolve it self.So they can act as a trigger when used with resolve
Crate an array that would hold our deffered objects note that order is important therefore numeric array is used
Just create a deffered
var dfd = jQuery.Deferred();
Define what should be done on
dfd.then(function(){
$(".playlist").append("<div id='" + rowNum + "' >" + rowNum + " " + trackTitle + "</div>");
})
Finally push it on our array
x.push(dfd)
After all api calls are over just call resolve on each
$.each(x, function(){
this.resolve(); //then callback gets called one by one on each deffered
});
https://jsfiddle.net/sanddune/rr7tord6/
You might want to have some mechanism to detect when the slowest call finishes and then call resolve
EDIT:
The latency of individual api varies greatly across geographies for example for me soundcloud was the most "laggy" so it ended up last but for someone else itunes may be the last to finish this is due to many variables like distance from server,ISP,throttling etc. so clearly you can never rely on "last man standing " approach so i could only suggest that use some sort of counter like this to track how many request are outstanding
First get total no. of nodes (inn xml) that have an api involved say you have just soundcloud and shopify so now total = 2 ; count = 0
so when the api request yields increment it by one like
$.getJSON(spotifyLink, function(data) {
trackTitle = data.name;
alert('shopify loaded');
count++;check();
});
or
SC.resolve(soundcloud_url).then(function(sound) {
trackTitle = sound.title;
alert('soundcloud loaded');
count++;check();
});
note that you need to increment on even failed api request (which may result from bad parameter etc.)
call a check function that will check if count == total on every success/fail part of api request
function check(){
if(count==total)
alert('all api request finished');
//resolve all deffereds
}

Unable to use jquery ajax to get sound file due to cross origin access control

I seem to be unable to check if a audio file exists before it actions anything due to No Access-Control-Allow-Origin'.
Is it possible to have this and if so, how ?
pText[n] = any word, for example: and, about.
But googles API do not hold names, so I need to check if a name is added to the text, and if so, use a different source url.
// audio check
var audioCheck = $.get('https://ssl.gstatic.com/dictionary/static/sounds/de/0/' + pText[n] +'.mp3');
console.log(audioCheck);
I have also tried $.ajax but without success.
This is the full script so you can see what I am doing
function populate(pText) {
for(var n=0; n < pText.length; n++) {
// audio check
var audioCheck = $.get('https://ssl.gstatic.com/dictionary/static/sounds/de/0/' + pText[n] +'.mp3');
console.log(audioCheck);
// if(audioCheck) { link is live } else { link is 404 }
//console.log(pText[n]);
$('[name=p1_1]').append('<span id="s' + n + '"><audio id="a' + n + '" src="https://ssl.gstatic.com/dictionary/static/sounds/de/0/' + pText[n] +'.mp3" type="audio/mpeg"></audio>' + pText[n] + '</span> ');
}
}
And help would be appreciated :)
I have managed to resolve this using the simple method of onerror
Here is the full code :)
function populate(pText) {
var tex = '';
for(var n=0; n < pText.length; n++) {
//console.log(pText[n]);
tex += '<span id="s' + n + '"><audio id="a' + n + '" src="https://ssl.gstatic.com/dictionary/static/sounds/de/0/' + pText[n] +'.mp3" onerror="mediaerro(' + n + ',"' + pText[n] +'")" type=audio/mpeg"></audio>' + pText[n] + ' </span>';
}
$('[name=p1Text]').html(tex);
}
$(document).on("click", "span", function() {
$(this).find('audio')[0].play();
});
mediaerro = function(id, word) {
console.log(word);
switch(word) {
case "Rocko":
$('#a' + id).attr('src', 'http://domain.com/books/book1/words/rocko.mp3');
break;
case "Caroline":
$('#a' + id).attr('src', 'http://domain.com/books/book1/words/caroline.mp3');
break;
case "Benji":
$('#a' + id).attr('src', 'http://domain.com/books/book1/words/benji.mp3');
break;
}
};
Using the dynamic audio id and managed to create a function based on an error, which replaces the broken link with a new one where the new recording is. Works a treat!

HTML read javascript and put result into a div

I'm new to HTML/javascript and I want to make something that displays Last.FM current playing songs, into a div on html, which displays it in text, I have a code that sends the current song through a chat on www.irccloud.com, and I was wondering If you could change it so that It could get received and put into a DIV on a page, the code is below:
and var r is the completed code, so how would I do something in the div that picks up the source as the link above and then grabs var r from it? If so, how would I do it??
I have tried the following code here
Sorry if I do not make sense.
(function () {
var e = "DeviousRunner";
window.lfmRecentTrack = function (t) {
var n = (new Array).concat(t.recenttracks.track)[0];
var album, spurl;
if (n.album["#text"]) {
album = " (from " + n.album["#text"] + ")";
} else {
album = "";
}
try {
var spotify = new XMLHttpRequest();
spotify.open("GET", "https://ws.spotify.com/search/1/track.json?q=" + encodeURIComponent(n.artist["#text"] + " - " + n.name), false);
spotify.send();
var spotresp = JSON.parse(spotify.responseText);
if (spotresp["tracks"].length > 0) {
//var urisplit = spotresp["tracks"][0]["href"].split(":");
//spurl = " https://open.spotify.com/" + urisplit[1] + "/" + urisplit[2];
spurl = spotresp["tracks"][0]["href"];
} else {
console.log("spotify: couldn't get url");
spurl = "";
}
} catch(e) {
console.log("spotify: " + e.message);
spurl = "";
}
var r = "is listening to " + n.name + " by " + n.artist["#text"] + " " + album + " (" + spurl + ")";
}
var n = document.createElement("script");
n.setAttribute("type", "text/javascript");
n.setAttribute("src", "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user=" + e + "&api_key=dd5fb083b94a7196cf696b9d7d11bc63&limit=1&format=json&callback=window.lfmRecentTrack");
document.body.appendChild(n)
})();
I updated your FIDDLE,
by moving this:
var element = document.getElementById("rss");
element.innerHTML = r;
inside the function...
hope this is useful for you

Using casperjs and phantomjs to scrape multiple pages

I'm trying to scrape a number of pages that have a standard format. I've been able to use Phantomjs to successfully scrape a single page, but when I try to iterate over multiple ones, the asynchronous processing makes things hang up. What's the proper way to tell Casper/Phantom to wait?
var page = require('webpage').create();
var fs = require('fs');
page.onConsoleMessage = function(msg) {
phantom.outputEncoding = "utf-8";
console.log(msg);
};
// this overwrites the previous output file
f = fs.open("lat_long.txt", "w");
f.write("--");
f.close();
// this is the unique identifier for the locations. For now, I just have three datapoints
var EPAID = ["KYD980501076","ME8170022018", "MEN000103584"];
/// this code will be used to loop through the different locations. For now, set to look at only one.
for (q= 0; q < 1; q++) {
var processing = false;
//we construct the target url
var url = "http://iaspub.epa.gov/enviro/efsystemquery.cerclis?fac_search=site_epa_id&fac_value=" + EPAID[0] + "&fac_search_type=Beginning+With&postal_code=&location_address=&add_search_type=Beginning+With&city_name=&county_name=&state_code=&program_search=1&report=2&page_no=1&output_sql_switch=TRUE&database_type=CERCLIS" ;
page.open(url);
page.onLoadFinished = function(status) {
if ( status === "success" ) {
page.includeJs("http://ajax.googleapis.com/ajax/libs/jquery/1.6.1/jquery.min.js", function() {
var str = page.evaluate(function() {
$value = [];
$Object = $(".result tr");
for (i =0 ; i < 10; i++) {
$value.push($Object.find('td').html(),$Object.find('td').next().next().html() );
$Object = $Object.next();
}
$string = "{ EPAID: "+ $value[0] + ", " +
"Name: "+ $value[1] + ", " +
"City: "+ $value[4] + ", " +
"State: "+ $value[6] + ", " +
"ZipCode: "+ $value[8] + ", " +
"Latitude: "+ $value[14] + ", " +
"Longitude: "+ $value[16] + " }" ;
return $string;
});
f = fs.open("lat_long.txt", "a");
f.write(str);
f.close();
processing = true;
console.log("writing to file");
phantom.exit();
});
}
// right here it should delay until the previous page is completed
// while (!processing) {
// setTimeout(function(){ console.log("waiting....");},1000);
// }
};
}
console.log("finished all pages");
If you switched to using casperJS, it is as simple as changing your page.open() into page.thenOpen(). (This CasperJS - How to open up all links in an array of links question looks very similar to yours?)
If you wanted to stick with PhantomJS you need to start the next page load in the onSuccess callback of the previous load. This is tedious, and needs care to avoid large memory usage. (I did it once or twice, but now simply use CasperJS.)
An alternative approach is to create the page object inside the loop. However that is not quite answering your question, as then they will run in parallel. But you could use setTimeout to stagger each once to avoid a burst of activity if you have hundreds of URLs!
Here is the code that ultimately works (using the timeout approach since I wasn't able to get the success callback to work better).
With casperjs installed, I named this file "process.js" and was able to run it from the command line as "casperjs process.js"
var page = require('webpage').create();
var fs = require('fs');
page.onConsoleMessage = function(msg) {
phantom.outputEncoding = "utf-8";
console.log(msg);
};
// this overwrites the previous output f
// this is the unique identifier for the locations.
var EPAID = ["NED981713837",... , "FLD049985302", "NJD986643153"];
f = fs.open("lat_long.txt", "w");
f.write("-<>-");
f.close();
var count = 0;
var target = 1400;
var written = [];
function yourFunction(){
if (count < target) {
process(count);
count++;
setTimeout(yourFunction, 5000);
} else {
console.log("exiting");
phantom.exit();
return;
}
}
function process(counter){
var processing = false;
console.log("Beginning record #" + counter);
//we construct the target url
var url = "http://iaspub.epa.gov/enviro/efsystemquery.cerclis?fac_search=site_epa_id&fac_value=" + EPAID[counter] + "&fac_search_type=Beginning+With&postal_code=&location_address=&add_search_type=Beginning+With&city_name=&county_name=&state_code=&program_search=1&report=2&page_no=1&output_sql_switch=TRUE&database_type=CERCLIS" ;
page.open(url);
page.onLoadFinished = function(status) {
if ( status === "success" ) {
page.includeJs("http://ajax.googleapis.com/ajax/libs/jquery/1.6.1/jquery.min.js", function() {
var str = page.evaluate(function() {
$value = [];
$Object = $(".result tr");
for (i =0 ; i < 10; i++) {
$value.push($Object.find('td').html(),$Object.find('td').next().next().html() );
$Object = $Object.next();
}
$string = "{ \"EPAID\": \""+ $value[0] + "\", " +
"\"Name\": \""+ $value[1] + "\", " +
"\"City\": \""+ $value[4] + "\", " +
"\"State\": \""+ $value[6] + "\", " +
"\"ZipCode\": \""+ $value[8] + "\", " +
"\"Latitude\": "+ $value[14] + ", " +
"\"Longitude\": "+ $value[16] + " }," ;
return $string;
});
if (written[counter] === undefined) {
f = fs.open("lat_long.txt", "a");
f.write(str);
f.close();
written[counter] = true;
console.log("Writing to file #"+ counter);
}
});
}
};
}
console.log("Start...");
yourFunction();

CasperJS Load next page in loop

I've been working on a script which collates the scores for a list of user from a website. One problem is though, I'm trying to load the next page in the while loop, but the function is not being loaded...
casper.then(function () {
var fs = require('fs');
json = require('usernames.json');
var length = json.username.length;
leaderboard = {};
for (var ii = 0; ii < length; ii++) {
var currentName = json.username[ii];
this.thenOpen("http://www.url.com?ul=" + currentName + "&sortdir=desc&sort=lastfound", function (id) {
return function () {
this.capture("Screenshots/" + json.username[id] + ".png");
if (!casper.exists(x("//*[contains(text(), 'That username does not exist in the system')]"))) {
if (casper.exists(x('//*[#id="ctl00_ContentBody_ResultsPanel"]/table[2]'))) {
this.thenEvaluate(tgsagc.tagNextLink);
tgsagc.cacheCount = 0;
tgsagc.
continue = true;
this.echo("------------ " + json.username[id] + " ------------");
while (tgsagc.
continue) {
this.then(function () {
this.evaluate(tgsagc.tagNextLink);
var findDates, pageNumber;
pageNumber = this.evaluate(tgsagc.pageNumber);
findDates = this.evaluate(tgsagc.getFindDates);
this.echo("Found " + findDates.length + " on page " + pageNumber);
tgsagc.checkFinds(findDates);
this.echo(tgsagc.cacheCount + " Caches for " + json.username[id]);
this.echo("Continue? " + tgsagc["continue"]);
this.click("#tgsagc-link-next");
});
}
leaderboard[json.username[id]] = tgsagc.cacheCount;
console.log("Final Count: " + leaderboard[json.username[id]]);
console.log(JSON.stringify(leaderboard));
} else {
this.echo("------------ " + json.username[id] + " ------------");
this.echo("0 Caches Found");
leaderboard[json.username[id]] = 0;
console.log(JSON.stringify(leaderboard));
}
} else {
this.echo("------------ " + json.username[id] + " ------------");
this.echo("No User found with that Username");
leaderboard[json.username[id]] = null;
console.log(JSON.stringify(leaderboard));
}
});
while (tgsagc.continue) {
this.then(function(){
this.evaluate(tgsagc.tagNextLink);
var findDates, pageNumber;
pageNumber = this.evaluate(tgsagc.pageNumber);
findDates = this.evaluate(tgsagc.getFindDates);
this.echo("Found " + findDates.length + " on page " + pageNumber);
tgsagc.checkFinds(findDates);
this.echo(tgsagc.cacheCount + " Caches for " + json.username[id]);
this.echo("Continue? " + tgsagc["continue"]);
return this.click("#tgsagc-link-next");
});
}
Ok, looking at this code I can suggest a couple of changes you should make:
I don't think you should be calling return from within your function within then(). This maybe terminating the function prematurely. Looking at the casperjs documentation, the examples don't return anything either.
Within your while loop, what sets "tgsagc.continue" to false?
Don't use "continue" as a variable name. It is a reserved word in Javascript used for terminating an iteration of a loop. In your case this shouldn't be a problem, but its bad practice anyhow.
Don't continually re-define the method within your call to the then() function. Refactor your code so that it is defined once elsewhere.
We ended up having to scope the function, so it loads the next page in the loop.
This is mainly because CasperJS is not designed to calculate scores, and it tries to asynchronously do the calculation, missing the required functions

Categories

Resources