I am trying to Post a large JSON into my Database, the JSON has at least 400.000 Objects.
If I cut the file, and try to Post 20.000 Objects everything works just fine, so the problem should be JSON's size.
I've split the JSON into 20 chunks, and my idea is to upload one at a time but i'm struggling to make it work.
This is what I'm using:
var rows = {};
Papa.parse(content, {
header: false,
delimiter: '|',
worker: true,
encoding: "utf16le",
dynamicTyping: true,
skipEmptyLines: true,
complete: function(results) {
rows = results.data;
let obj = []
for(var i=0; i < rows.length; i++){
obj.push(rows[i])
}
let result = []
for(let i in obj) {
let temp = {}
if(i > 0) {
temp["id"] = obj[i][0]
temp["name"] = obj[i][1]
temp["tel"] = obj[i][2]
temp["email"] = obj[i][3]
temp["status"] = obj[i][5]
result.push(temp)
}
}
var array1 = result.map((e) => {
return {
id: e.id,
name: e.name,
email: e.email
}
})
let chunked = []
let size = 20000;
Array.from({length: Math.ceil(array1.length / size)}, (val, i) => {
chunked.push(array1.slice(i * size, i * size + size))
})
console.log(chunked); // at this point I have my array divided into chunks of 20000
axios({
url: 'url',
method: 'post',
data: chunked
})
.then(function (response) {
// your action after success
console.log(response);
})
.catch(function (error) {
// your action on error successif (error.response) {
console.log(error);
});
You can send it one by one as follow. Assuming your backend is able to accept the data format.
for(let i=0;i<chunked.length;i++){
axios({
url: 'url',
method: 'post',
data: chunked[i]
})
.then(function (response) {
// your action after success
console.log(response);
})
.catch(function (error) {
// your action on error successif (error.response) {
console.log(error);
});
}
Or a modern solution could be to use Promise.all([])
let promiseArray = [];
for(let i=0;i<chunked.length;i++){
promiseArray.push(axios({
url: 'url',
method: 'post',
data: chunked[i]
}))
}
Promise.all(promiseArray)
.then((responses) => {
console.log(responses); //Will return resolved/rejected promises in an array
})
.catch(error => {
console.error(error.response)
});;
You can read more about Promise.all([]) https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/all
Related
I want to post files to groups. by storing the group id I want to send files to groupsFinal, but axios.post() will only post to the first array of groupsFinal, I tried for loop but it fails axios post.
let groups = [{id:"a001", groupName:"Group101"},{id:"a002", groupName:"Group202"}]
let groupsFinal = [];
for (let a = 0; a < groups.length; a++) {
let d = JSON.parse(JSON.stringify(groups[a]));
d.img = null;
groupsFinal.push(d);
}
let p = {
fileId: "file_001",
groups: groupsFinal, // array[]
};
return new Promise((resolve, reject) => {
axios
.post("/shareFileToGroup", p, {
headers: {
"Content-Type": "application/json"
},
})
.then(function (response) {
resolve("success");
})
.catch(function (error) {
reject("error");
})
.finally(function () {
// always executed
});
});
Here is my code, I pass array of ids and for each ids I hit Api request.
for (let i = 0; i < 10; i++) {
cy.request({
method: 'GET',
url: `https://[API endpoint ]/api/v1/[requst]/${ids[i]}`,
headers: {
"authorization": token
}
})
.then((res) => {
return usernames.push([{ value: res.body.instagrams[0].username }])
})
.then((res) => {
return cy.wait(2000)
})
}
Whenever I execute the code, for 2-3 requests it works fine and then throws error like
Uncaught Error: invalid payload
I can see my URI is proper and sending correct request.
you should modify your code to wrap all the pending promises into an array. A modified snippet is as below:
async function processRequest(){
let result;
let promises = [];
for (let i = 0; i < 10; i++) {
promises.push(cy.request({
method: 'GET',
url: `https://[API endpoint ]/api/v1/[requst]/${ids[i]}`,
headers: {
"authorization": token
}
}))
}
result = await Promise.all(promises);
for(let i = 0; i < 10; i++){
usernames.push([{ value: result[i].username }])
}
}
Here cy.request doesn't work properly with Promise.
Either while pushing it to promise array, remove cy.request() or purely handle with javascript.
async findUserName(ids, token) {
let promise = []
let usernames = []
ids.forEach(id => {
promise.push(
fetch(`https://[host]/api/v1/[endpoint]/${id}`, {
headers: {
'Content-Type': 'application/json',
"authorization": token
}
})
.then((res) => {
return res.json();
})
)
})
let result = await Promise.all(promise);
result.forEach((data) => {
usernames.push([{ value: data.instagrams[0].username }])
})
return usernames
}
I am getting the below result after my API call.
My node version is 12.x
{"type":"Buffer","data":[123,34,101,114,114,111,114,115,34,58,91,34,74,87,84,32,105,115,32,101,120,112,105,114,101,100,32,111,114,32,100,111,101,115,32,110,111,116,32,104,97,118,101,32,112,114,111,112,101,114,32,39,101,120,112,39,32,99,108,97,105,109,34,93,125,11]}
Please see the code snippet below:
let postOptions = {
host: 'vault.server',
path: '/v1/auth/gcp/login',
method: HTTPS.POST_REQUEST,
headers: {
'Content-Type': 'application/json; charset=utf-8',
'X-Vault-Namespace': 'mynamespace'
},
json: true,
rpDefaults: {
strictSSL: false
}
};
let requestPayLoad = {
"role": this._vaultRole,
"jwt": signedJWT
};
console.log(JSON.stringify(requestPayLoad));
console.log(JSON.stringify(postOptions));
try {
let result = await HTTPS.makeRequest(postOptions, JSON.stringify(requestPayLoad), HTTPS.POST_REQUEST);
console.log('Response***************',JSON.stringify(result));
return result.auth.client_token;
}
Please see the below code snippet for the http make request method.
return new Promise((resolve, reject) => {
let rq = https.request(options, (res) => {
let response;
let chunks = [];
res.on('data', (chunk) => {
chunks.push(chunk);
});
res.on('end', () => {
response = Buffer.concat(chunks);
return resolve(response);
});
});
rq.on('error', (e) => {
return reject({'statusCode': 500, 'success': false, 'error': e.toString()});
});
if (type === 'POST') {
rq.write(data);
}
rq.end();
});
Please help me to resolve this
You are receiving the data as a Buffer. Use the toString() method to convert this buffer into a string inside the try block.
try {
let result = await HTTPS.makeRequest(postOptions, JSON.stringify(requestPayLoad), HTTPS.POST_REQUEST);
console.log('Response***************', result.toString());
return result.auth.client_token;
}
If you want to access the data from the response returned from you API call
do:
let data = result.data;
and I you want to get client_token as showing here:
return result.auth.client_token;
it's not possible because the response does not have auth attribute on it:
{"type":"Buffer","data":[123,34,101,114,114,111,114,115,34,58,91,34,74,87,84,32,105,115,32,101,120,112,105,114,101,100,32,111,114,32,100,111,101,115,32,110,111,116,32,104,97,118,101,32,112,114,111,112,101,114,32,39,101,120,112,39,32,99,108,97,105,109,34,93,125,11]}
I have an ajax call that responds with a list of movies, and then I have an other function with an other ajax call that returns the genre names, since the first call only has the id of the genre, then I relate the genreId with its name and I assign it to the JSON on the first ajax call, kind of like assigning it to a variable. The problem that I have is that since ajax is asynchronous, it always ends up undefined. I don't want to make it synchronous since it will end up in a bad user experience.
First ajax call
$.ajax({
url: urlString,
success: (searchResults) => {
const results = searchResults.results
var movieRows = [];
results.forEach(movie => {
movie.genres = this.getMovieGenres(movie.media_type, movie.genre_ids);
const movieRow = <MovieRow key={movie.id} movie={movie}/>;
movieRows.push(movieRow)
});
this.setState({rows: movieRows})
},
error: (xhr, status, err) => {
console.log("Failed to fetch data...")
}
})
Function that I call with the second ajax call
getMovieGenres (mediaType ,movieGenreIds) {
urlString = `https://api.themoviedb.org/3/genre/movie/list?api_key=${config.movieDBbApiKey}&language=en-US`ApiKey}&language=en-US`
var genres = []
$.ajax({
url: urlString,
async: true,
crossDomain: true,
method: "GET",
success: searchResults => {
for (let i = 0; i < movieGenreIds.length; i++) {
for (let j = 0; j < searchResults.genres.length; i++){
console.log(searchResults.genres[j].id)
if (movieGenreIds[i] === searchResults.genres[j].id) {
genres.push(searchResults.genres[j].name)
}
}
}
},
error: (xhr, status, err) => {
console.log("Failed to fetch data...")
}
})
return genres
}
There are variouse solutions to your problem.
The simplest is to make the second ajax call in the callback of the first ajax call. In the second call you'll have both results from first ajax call and the second. Then make transforms and set state in the second call.
Other options (the reccomended one) is to work with Axios or Fetch that rely on Promises.
Promises can be evaded with Promise.all:
var p1 = Promise.resolve(3);
var p2 = 1337;
var p3 = new Promise((resolve, reject) => {
setTimeout(resolve, 100, "foo");
});
Promise.all([p1, p2, p3]).then(values => {
console.log(values); // [3, 1337, "foo"]
});
you can try this:
var promise = new Promise((resolve,reject)=> {
return $.ajax({
url: urlString,
success: (searchResults) => {
return resolve(searchResults.results)
},
error: (xhr, status, err) => {
return reject("Failed to fetch data...")
}
})
})
function that return a promise
getMovieGenres (mediaType ,movieGenreIds) {
urlString = `https://api.themoviedb.org/3/genre/movie/list?api_key=${config.movieDBbApiKey}&language=en-US`ApiKey}&language=en-US`
var genres = []
return new Promise((resolve,reject)=> {
$.ajax({
url: urlString,
async: true,
crossDomain: true,
method: "GET",
success: searchResults => {
for (let i = 0; i < movieGenreIds.length; i++) {
for (let j = 0; j < searchResults.genres.length; i++){
console.log(searchResults.genres[j].id)
if (movieGenreIds[i] === searchResults.genres[j].id) {
genres.push(searchResults.genres[j].name)
}
}
}
return resolve(genres)
},
error: (xhr, status, err) => {
return reject("Failed to fetch data...")
}
})
})
}
finally:
promise().then(results => {
var movieRows = [];
var tasks = []
results.forEach(movie => {
tasks.push(getMovieGenres(movie.media_type, movie.genre_ids))
});
Promise.all(tasks).then(output => {
output.forEach(movie => {
const movieRow = <MovieRow key={movie.id} movie={movie}/>;
movieRows.push(movieRow)
})
this.setState({rows: movieRows})
})
})
Hope to help you!
I have a list of 15+ ajax requests that need to be called in a specific order. I need each ajax call to wait until the previous function finishes before making the next call. This issue arises because my ajax call, has a direct callback that is also an ajax call.
createCheckIn() {
this.selectedList = [...] // long list of objects
count = 0
for ( i=0; i < this.selectedList.length; i++ ) {
$.ajax({
method: "POST",
url: url,
data: {
check_in: {
client_id: this.selectClient.id,
program_id: this.program_id
}
},
success: function(res) {
that.createWeighIn(count, res.id)
count = count + 1
},
error: function(err) {
console.log(err)
}
})
}
},
createWeighIn(index, check_in_id) {
let data = {}
let that = this
data.weigh_in = this.selectedList[index]
$.ajax({
method: "POST",
url: url,
data: data,
success: function(res) {
console.log(res)
},
error: function(err) {
console.log(err)
}
})
}
the correct data is generated but I believe the ordering is off because eventually there is a call to createCheckIn() that begins before the previous entry has completed.
Is there a way to chain these functions such that createCheckIn() and createWeighIn() are called (and complete) before selectedList iterates.
your for loop in createCheckIn() will not stop to wait on your ajax return. you can do something like:
function createCheckIn(oldI, oldCount){
var count = 0;
var currentI = 0;
if(oldCount != null){
count = oldCount;
}
if(oldI != null){
currentI = oldI;
}
if(currentI < this.selectedList.length){
$.ajax({
method: "POST",
url: url,
data: {
check_in: {
client_id: this.selectClient.id,
program_id: this.program_id
}
},
success: function(res) {
that.createWeighIn(count, res.id)
createCheckIn(currentI + 1, count + 1);
},
error: function(err) {
console.log(err)
}
}); //ajax
} // if
}
seems likely that you can eliminate one of those counters too, the i or the count
Seems like this is missing some potentially really important details about what you need to do leading up to this (ie. this.selectedItems generation) and what happens after (what if one call checkin fails, what if a checkin succeeds but its corresponding weighIn fails, etc..). That said...
It seems you are not actually using the counter for anything other than to reference data you already have, so why not just pass that in directly like:
createWeighIn(weighInData, check_in_id) {
let data = {};
let that = this;
data.weigh_in = weighInData;
// ... your other code
}
I would make createCheckIn only handle doing the ajax request and making a single "reservation" in your system. Then i would make a new method called checkIn that uses the two previous method to process all of selected items:
checkIn() {
let self = this;
let promises = [];
let this.selectedList = [...];
for (let = 0; i < this.selectedList.length; i++) {
// always create the deferred outside the call
let def = $.Deferred();
promises.push(def.promise());
this.createCheckIn().done(function (res) {
self.createWeighIn(self.selectedList[i], res.id))
.done(function () {
// resolve
def.resolve.apply(def, Array.prototype.slice.call(arguments);
})
.fail(function () {
def.reject.apply(def, Array.prototype.slice.call(arguments);
});
}).fail(function () {
// if checkin fails, always reject because we know weighIn wont be called
def.reject.apply(def, Array.prototype.slice.call(arguments);
});
};
// this will resolve/fail when all promises (from createWeighIn) resolve/fail
return $.when.apply(null, promises);
}
so putting it all together:
{
createCheckIn() {
let request = $.ajax({
method: "POST",
url: url,
data: {
check_in: {
client_id: this.selectClient.id,
program_id: this.program_id
}
}
})
.fail(function(err) {
console.log(err)
});
};
return request;
},
createWeighIn(data, check_in_id) {
let params = {};
params.weigh_in = data;
let request = $.ajax({
method: "POST",
url: url,
data: params,
success: function(res) {
console.log(res)
},
error: function(err) {
console.log(err)
}
});
return request;
},
checkIn() {
let self = this;
let promises = [];
let this.selectedList = [...];
for (let = 0; i < this.selectedList.length; i++) {
// always create the deferred outside the call
let def = $.Deferred();
promises.push(def.promise());
this.createCheckIn().done(function (res) {
self.createWeighIn(self.selectedList[i], res.id))
.done(function () {
// resolve
def.resolve.apply(def, Array.prototype.slice.call(arguments);
})
.fail(function () {
def.reject.apply(def, Array.prototype.slice.call(arguments);
});
}).fail(function () {
// if checkin fails, always reject because we know weighIn wont be called
def.reject.apply(def, Array.prototype.slice.call(arguments);
});
};
// this will resolve/fail when all promises (from createWeighIn) resolve/fail
return $.when.apply(null, promises);
}
}
I ended up introducing promises, and some recursion and removing the loop altogether. I basically begin the process by calling createCheckIn() with an index of 0:
this.createCheckIn(0)
createCheckIn(index) {
this.selectedList = [...] // long list of objects
count = 0
let prom = new Promise(function(resolve, reject) {
$.ajax({
method: "POST",
url: url,
data: {
check_in: {
client_id: that.selectClient.id,
program_id: that.program_id
}
},
success: function(res) {
resolve(that.createWeighIn(index, res.id))
},
error: function(err) {
reject(console.log(err))
}
})
})
},
createWeighIn(index, check_in_id) {
let data = {}
let that = this
data.weigh_in = this.selectedList[index]
let prom = new Promise(function(resolve, reject) {
$.ajax({
method: "POST",
url: url,
data: data,
success: function(res) {
console.log(res)
if ( index == (that.selectedList.length - 1) ) {
that.complete = true
resolve(console.log("complete"))
} else {
index++
resolve(that.createCheckIn(index))
}
},
error: function(err) {
console.log(err)
}
})
})
}