Develop Reference

How to write file using fs.createWriteStream

am trying to build a web scraper that downloads all the pdfs in a website. i've written all the logic necessary to do this but for some reason it downloads an empty pdf file which is not suppose to be so, the problem seems to be coming from the downloadFile function when i try to pipe the data which for some reason seems not to be working because i get an empty pdf file after the function is ran. i'll would appreciate it if someone can help me out with this problem, thanks.
here's a sample of my code:
app.js
const fs = require("fs");
const path = require("path");
const cheerio = require("cheerio");
const axiosInstance = require("./getAxios");
const axios = axiosInstance();
const Surl = "https://www.health.gov.ng/";
// linkList sample: "https://www.health.gov.ng/index.php?option=com_content&view=article&id=143&Itemid=512";
let = connectionFailCount = 0;
let linkList = [];
let dlinkList = [];
const getWebsiteLinks = async (Surl) => {
try {
console.log(`Crawling all links from: ${Surl}`);
const response = await axios.get(Surl);
const $ = cheerio.load(response.data);
const ranges = $("a").each(function (idx, el) {
if ($(el).attr("href")) {
return $(el).attr("href");
}
});
for (let index = 0; index < ranges.length; index++) {
let raw_links = $("a")[index].attribs.href;
if (raw_links.startsWith("/")) {
linkList.push(Surl + raw_links);
}
}
if (linkList.length > 0) {
console.log(`Finished crawling links: Found ${linkList.length} links`);
console.log(
"--------------------------------------------------------\n\n"
);
}
return;
} catch (error) {
if (connectionFailCount === 0) {
connectionFailCount += 1;
getWebsiteLinks(Surl);
console.log(`Connection error. \n
Reconnecting to server....`);
} else if (connectionFailCount === 5) {
console.error(`Can not connect to server. Try again later.`);
}
}
};
const downloadLinks = async (linkList) => {
try {
console.log("Crawling links to find pdf links. this may take a while...");
for (const link of linkList) {
const response = await axios.get(link);
// Skip where there's delayed server response
if (response.code === "ECONNRESET") continue;
const $ = cheerio.load(response.data);
$("a").each(function (idx, el) {
if ($(el)?.attr("href")?.endsWith(".pdf")) {
let addr = $(el).attr("href");
let dlink = Surl + addr;
dlinkList.push({
pathName: addr,
url: dlink,
});
}
});
}
console.log(dlinkList);
if (dlinkList.length > 0) {
console.log(`Crawling Finish: Found ${dlinkList.length} pdf links`);
console.log(
"--------------------------------------------------------\n\n"
);
}
} catch (error) {
if (connectionFailCount === 0) {
connectionFailCount += 1;
console.log(`Connection error. \n
Reconnecting to server: ${connectionFailCount} count`);
downloadLinks(linkList);
}
if (connectionFailCount === 3) {
console.error(`Can not connect to server. Try again later.`);
return;
}
// console.error("downloadLinksError: ", error);
}
};
const downloadFiles = async (dlinkList) => {
console.log("Creating directory to save PDF files");
const appRoot = path.dirname(path.resolve(__dirname));
// Had to change and restructure code due to error
const folderName = `PDF/${Surl.split("/").pop()}`;
const subFolderName = Surl.split("/").pop();
try {
if (!fs.existsSync(path.join(appRoot, folderName))) {
fs.mkdirSync(path.join(appRoot, "PDF"));
fs.mkdirSync(path.join(`${appRoot}/PDF`, subFolderName));
}
dlinkList.forEach(async (link) => {
let name = link.pathName;
let url = link.url;
let file = fs
.createWriteStream(
`${appRoot}/${folderName}/${name.split("/").pop()}`,
"utf-8"
)
.on("error", (err) => {
console.error("createWriteStreamError: ", err);
});
try {
console.log("Downloading PDF file...");
const { data } = await axios({
url,
method: "GET",
responseType: "stream",
});
if (data) {
console.log("PDF file Downloaded");
data.pipe(file);
}
} catch (error) {
console.error(error);
}
});
return;
} catch (error) {
console.error("downloadFilesError: ", error);
}
};
(async () => {
await getWebsiteLinks(Surl);
await downloadLinks(linkList);
await downloadFiles(dlinkList);
})();
getAxios.js
const axios = require("axios");
const https = require("https");
module.exports = function () {
const domain = "https://www.health.gov.ng/";
let instance;
if (!instance) {
//create axios instance
instance = axios.create({
baseURL: domain,
timeout: 60000, // Increase time out incase of network delay or delayed server response
maxContentLength: 500 * 1000 * 1000, // Increase maximum response ata length
httpsAgent: new https.Agent({ keepAlive: true }),
headers: { "Content-Type": "application/xml" },
});
}
return instance;
};

Unhandled error Error: Data cannot be encoded in JSON error at firebase serverless functions

I'm trying to deploy an api for my application. Using these codes raises Unhandled error "Error: Data cannot be encoded in JSON.
const functions = require("firebase-functions");
const axios = require("axios");
exports.getDatas = functions.https.onCall(async (d)=>{
functions.logger.log(d["name"]);
cname = d["name"];
ts1=d["ts1"];
ts2=d["ts2"];
const data = await axios.get(
"https://api.coingecko.com/api/v3/coins/" +
cname +
"/market_chart/range?vs_currency=usd&from=" +
ts1 +
"&to=" +
ts2,
);
functions.logger.log(data);
return {data: data};
});
The error log is
Unhandled error Error: Data cannot be encoded in JSON: function httpAdapter(config) {
return new Promise(function dispatchHttpRequest(resolvePromise, rejectPromise) {
var onCanceled;
function done() {
if (config.cancelToken) {
config.cancelToken.unsubscribe(onCanceled);
}
if (config.signal) {
config.signal.removeEventListener('abort', onCanceled);
}
}
var resolve = function resolve(value) {
done();
resolvePromise(value);
};
var rejected = false;
var reject = function reject(value) {
done();
rejected = true;
rejectPromise(value);
};
var data = config.data;
var headers = config.headers;
var headerNames = {};
Object.keys(headers).forEach(function storeLowerName(name) {
headerNames[name.toLowerCase()] = name;
});
// Set User-Agent (required by some servers)
// See https://github.com/axios/axios/issues/69
if ('user-agent' in headerNames) {
// User-Agent is specified; handle case where no UA header is desired
if (!headers[headerNames['user-agent']]) {
delete headers[headerNames['user-agent']];
}
// Otherwise, use specified value
} else {
// Only set header if it hasn't been set in config
headers['User-Agent'] = 'axios/' + VERSION;
}
if (data && !utils.isStream(data)) {
if (Buffer.isBuffer(data)) {
// Nothing to do...
} else if (utils.isArrayBuffer(data)) {
data = Buffer.from(new Uint8Array(data));
} else if (utils.isString(data)) {
data = Buffer.from(data, 'utf-8');
} else {
return reject(createError(
'Data after transformation must be a string, an ArrayBuffer, a Buffer, or a Stream',
config
));
}
if (config.maxBodyLength > -1 && data.length > config.maxBodyLength) {
return reject(createError('Request body larger than maxBodyLength limit', config));
}
// Add Content-Length header if data exists
if (!headerNames['content-length']) {
headers['Content-Length'] = data.length;
}
}
// HTTP basic authentication
var auth = undefined;
if (config.auth) {
var username = config.auth.username || '';
var password = config.auth.password || '';
auth = username + ':' + password;
}
// Parse url
var fullPath = buildFullPath(config.baseURL, config.url);
var parsed = url.parse(fullPath);
var protocol = parsed.protocol || 'http:';
if (!auth && parsed.auth) {
var urlAuth = parsed.auth.split(':');
var urlUsername = urlAuth[0] || '';
var urlPassword = urlAuth[1] || '';
auth = urlUsername + ':' + urlPassword;
}
if (auth && headerNames.authorization) {
delete headers[headerNames.authorization];
}
var isHttpsRequest = isHttps.test(protocol);
var agent = isHttpsRequest ? config.httpsAgent : config.httpAgent;
var options = {
path: buildURL(parsed.path, config.params, config.paramsSerializer).replace(/^\?/, ''),
method: config.method.toUpperCase(),
headers: headers,
agent: agent,
agents: { http: config.httpAgent, https: config.httpsAgent },
auth: auth
};
if (config.socketPath) {
options.socketPath = config.socketPath;
} else {
options.hostname = parsed.hostname;
options.port = parsed.port;
}
var proxy = config.proxy;
if (!proxy && proxy !== false) {
var proxyEnv = protocol.slice(0, -1) + '_proxy';
var proxyUrl = process.env[proxyEnv] || process.env[proxyEnv.toUpperCase()];
if (proxyUrl) {
var parsedProxyUrl = url.parse(proxyUrl);
var noProxyEnv = process.env.no_proxy || process.env.NO_PROXY;
var shouldProxy = true;
if (noProxyEnv) {
var noProxy = noProxyEnv.split(',').map(function trim(s) {
return s.trim();
});
shouldProxy = !noProxy.some(function proxyMatch(proxyElement) {
if (!proxyElement) {
return false;
}
if (proxyElement === '*') {
return true;
}
if (proxyElement[0] === '.' &&
parsed.hostname.substr(parsed.hostname.length - proxyElement.length) === proxyElement) {
return true;
}
return parsed.hostname === proxyElement;
});
}
if (shouldProxy) {
proxy = {
host: parsedProxyUrl.hostname,
port: parsedProxyUrl.port,
protocol: parsedProxyUrl.protocol
};
if (parsedProxyUrl.auth) {
var proxyUrlAuth = parsedProxyUrl.auth.split(':');
proxy.auth = {
username: proxyUrlAuth[0],
password: proxyUrlAuth[1]
};
}
}
}
}
if (proxy) {
options.headers.host = parsed.hostname + (parsed.port ? ':' + parsed.port : '');
setProxy(options, proxy, protocol + '//' + parsed.hostname + (parsed.port ? ':' + parsed.port : '') + options.path);
}
var transport;
var isHttpsProxy = isHttpsRequest && (proxy ? isHttps.test(proxy.protocol) : true);
if (config.transport) {
transport = config.transport;
} else if (config.maxRedirects === 0) {
transport = isHttpsProxy ? https : http;
} else {
if (config.maxRedirects) {
options.maxRedirects = config.maxRedirects;
}
transport = isHttpsProxy ? httpsFollow : httpFollow;
}
if (config.maxBodyLength > -1) {
options.maxBodyLength = config.maxBodyLength;
}
if (config.insecureHTTPParser) {
options.insecureHTTPParser = config.insecureHTTPParser;
}
// Create the request
var req = transport.request(options, function handleResponse(res) {
if (req.aborted) return;
// uncompress the response body transparently if required
var stream = res;
// return the last request in case of redirects
var lastRequest = res.req || req;
// if no content, is HEAD request or decompress disabled we should not decompress
if (res.statusCode !== 204 && lastRequest.method !== 'HEAD' && config.decompress !== false) {
switch (res.headers['content-encoding']) {
/*eslint default-case:0*/
case 'gzip':
case 'compress':
case 'deflate':
// add the unzipper to the body stream processing pipeline
stream = stream.pipe(zlib.createUnzip());
// remove the content-encoding in order to not confuse downstream operations
delete res.headers['content-encoding'];
break;
}
}
var response = {
status: res.statusCode,
statusText: res.statusMessage,
headers: res.headers,
config: config,
request: lastRequest
};
if (config.responseType === 'stream') {
response.data = stream;
settle(resolve, reject, response);
} else {
var responseBuffer = [];
var totalResponseBytes = 0;
stream.on('data', function handleStreamData(chunk) {
responseBuffer.push(chunk);
totalResponseBytes += chunk.length;
// make sure the content length is not over the maxContentLength if specified
if (config.maxContentLength > -1 && totalResponseBytes > config.maxContentLength) {
// stream.destoy() emit aborted event before calling reject() on Node.js v16
rejected = true;
stream.destroy();
reject(createError('maxContentLength size of ' + config.maxContentLength + ' exceeded',
config, null, lastRequest));
}
});
stream.on('aborted', function handlerStreamAborted() {
if (rejected) {
return;
}
stream.destroy();
reject(createError('error request aborted', config, 'ERR_REQUEST_ABORTED', lastRequest));
});
stream.on('error', function handleStreamError(err) {
if (req.aborted) return;
reject(enhanceError(err, config, null, lastRequest));
});
stream.on('end', function handleStreamEnd() {
try {
var responseData = responseBuffer.length === 1 ? responseBuffer[0] : Buffer.concat(responseBuffer);
if (config.responseType !== 'arraybuffer') {
responseData = responseData.toString(config.responseEncoding);
if (!config.responseEncoding || config.responseEncoding === 'utf8') {
responseData = utils.stripBOM(responseData);
}
}
response.data = responseData;
} catch (err) {
reject(enhanceError(err, config, err.code, response.request, response));
}
settle(resolve, reject, response);
});
}
});
// Handle errors
req.on('error', function handleRequestError(err) {
if (req.aborted && err.code !== 'ERR_FR_TOO_MANY_REDIRECTS') return;
reject(enhanceError(err, config, null, req));
});
// set tcp keep alive to prevent drop connection by peer
req.on('socket', function handleRequestSocket(socket) {
// default interval of sending ack packet is 1 minute
socket.setKeepAlive(true, 1000 * 60);
});
// Handle request timeout
if (config.timeout) {
// This is forcing a int timeout to avoid problems if the `req` interface doesn't handle other types.
var timeout = parseInt(config.timeout, 10);
if (isNaN(timeout)) {
reject(createError(
'error trying to parse `config.timeout` to int',
config,
'ERR_PARSE_TIMEOUT',
req
));
return;
}
// Sometime, the response will be very slow, and does not respond, the connect event will be block by event loop system.
// And timer callback will be fired, and abort() will be invoked before connection, then get "socket hang up" and code ECONNRESET.
// At this time, if we have a large number of request, nodejs will hang up some socket on background. and the number will up and up.
// And then these socket which be hang up will devoring CPU little by little.
// ClientRequest.setTimeout will be fired on the specify milliseconds, and can make sure that abort() will be fired after connect.
req.setTimeout(timeout, function handleRequestTimeout() {
req.abort();
var transitional = config.transitional || defaults.transitional;
reject(createError(
'timeout of ' + timeout + 'ms exceeded',
config,
transitional.clarifyTimeoutError ? 'ETIMEDOUT' : 'ECONNABORTED',
req
));
});
}
if (config.cancelToken || config.signal) {
// Handle cancellation
// eslint-disable-next-line func-names
onCanceled = function(cancel) {
if (req.aborted) return;
req.abort();
reject(!cancel || (cancel && cancel.type) ? new Cancel('canceled') : cancel);
};
config.cancelToken && config.cancelToken.subscribe(onCanceled);
if (config.signal) {
config.signal.aborted ? onCanceled() : config.signal.addEventListener('abort', onCanceled);
}
}
// Send the request
if (utils.isStream(data)) {
data.on('error', function handleStreamError(err) {
reject(enhanceError(err, config, null, req));
}).pipe(req);
} else {
req.end(data);
}
});
}
at encode (/workspace/node_modules/firebase-functions/lib/common/providers/https.js:162:11)
at encode (/workspace/node_modules/firebase-functions/lib/common/providers/https.js:156:22)
at encode (/workspace/node_modules/firebase-functions/lib/common/providers/https.js:156:22)
at encode (/workspace/node_modules/firebase-functions/lib/common/providers/https.js:156:22)
at /workspace/node_modules/firebase-functions/lib/common/providers/https.js:334:22
at processTicksAndRejections (internal/process/task_queues.js:97:5)
First logger logs the parameter i gave correctly and the logger that logs data is in this format:
...["api.coingecko.com:443::::::::::::::::::"]},"keepAliveMsecs":1000,"maxFreeSockets":256,"scheduling":"fifo","keepAlive":false,"maxSockets":null},"_removedConnection":false,"writable":true},"status":200,"data":{"prices":[[1615345414698,37.27069164629981],[1615349310788,36.95627388647297],[1615352802175,37.48630338203377],[1615356202751,37.46442850999597],[1615360079361,37.642735963063906],[1615363905145,38.29435586902702],[1615367492353,38.313292928237594],[1615370461299,38.75503558097479],[1615374138056,38.24406575020552],[1615377815960,38.237026584388175],[1615381321332,38.93964664468625],[1615384813000,39.262646397955635],[1615388739874,39.15882057568881],[1615392094129,38.94488140309047],[1615395966875,38.79820936257378],[1615399312625,38.51637055616189],[1615403055037,38.59237008394828],[1615406529740,38.44087305010874],[1615410281814,37.71855645797291],[1615414278815,38.374824600586976],[1615417716420,38.4538669693684],[1615421045728,37.62772478442999],[1615425672990,36.8826465121472],[1615429587089,37.41958697414903],[1615432278494,37.34865694722488],[1615435254265,37.16289143388951],[1615439122292,37.14731463575248],[1615442523394,36.801517989796814],[1615446290102,37.02248224990424],[1615450361470,36.164787531097126],[1615453299572,36.46191265162147],[1615457172317,36.174755169666334],[1615460886498,37.05778010952229],[1615464298322,37.336909500902365],[1615469586325,37.56497212211488],[1615472126260,37.83046394206218],[1615474882979,37.252561357731096],[1615478498201,36.56190097084664],[1615482336185,36.83824760787625],[1615485957910,36.89351702770813],[1615489642151,37.589229946501746],[1615493390438,37.33184737771527],[1615496666244,37.29234576242379],[1615500577712,37.284260441548426],[1616866645601,1137195941.0307472],[1616870299925,1089416195.9864128],[1616873841648,1074341877.495249],[1616877368137,1061555457.3375872],[1616880970910,1077775411.1216433],[1616884693948,1064594490.6022671],[1616887998472,1087481667.611567],[1616891397951,1068140794.5197278],[1616894759953,1078753362.1719048],[1616898371565,1053546315.1245787],[1616902002474,1052498816.7223371],[1616905584364,1026915395.5541993],[1616909101481,1022271206.3215427],[1616912730390,997185793.1210617],[1616916434482,972130048.6316774],[1616919928611,988711196.2721183],[1616923534317,987299160.6191593],[1616926264719,975360472.6011684],[1616930074136,958327264.7346151],[1616933292776,935085970.8922312],[1616936940791,896217168.3654604],[1616940936847,878876312.6707534],[1616944090304,890504985.5476977],[1616948321869,896715385.5657766],[1616952007508,870767231.0865391],[1616955544207,880601758.4610194],[1616958381375,896794852.1077055],[1616962022167,929362788.5783823],[1616966479654,927502494.4691795],[1616969648773,880385481.5284289],[1616973545649,862329007.9935848],[1616977463095,840138544.6360805],[1616980359587,849727926.595521],[1616984356096,820616225.3306137],[1616987602367,898085663.0760688],[1616990444958,890215727.4112909],[1616995470635,914823340.6343507],[1616999032159,890922230.685704],[1617002651977,937214914.0703756],[1617005329558,976030203.3879734],[1617009370471,1061898884.4388478],[1617012348377,1111994349.2592206],[1617015705482,1175310227.1595278],[1617019895549,1217044915.3900926],[1617022941451,1204239369.9336267],[1617027118715,1225123359.1178432],[1617031210170,1191418570.9198012],[1617033728601,1257085051.9742537],[1617037882992,1261291734.3667347],[1617041858553,1265805909.4506621],[1617044547418,1261869965.5784621],[1617049418534,1225924891.220988],[1617052450394,1200646247.466799],[1617055896172,1209247034.0807025],[1617059684123,1249662106.3996315],[1617062561979,837849935.5380555],[1617066155823,1261094295.2039979],[1617070572708,1244044711.3556864],[1617074210159,1178503497.252399],[1617077106612,1184744920.254339],[1617080571662,1219164970.9205332],[1617084836477,1174744890.1399443],[1617087739776,1236332180.5454476],[1617092763739,1121685108.4046226],[1617096303391,1074005978.1362224],[1617100013739,1075898891.906641],[1617102136947,1041120230.0169744],[1617106411165,1021062028.7444541],[1617110588848,1004207600.6385714],[1617114148509,983098685.435342],[1617117449987,983878432.6976557],[1617120868725,943893192.0239582],[1617123806180,948379973.8680001],[1617128347360,948328240.0510467],[1617131244094,923477307.6495335],[1617134866719,918321070.6284192],[1617138697011,960178009.2986945],[1617142067857,974105207.7725881],[1617146083923,973959760.0729104],[1617149999086,959500047.5209063],[1617153094367,1007753562.6156206],[1617156698445,1021534121.1115336],[1617160175611,1028067427.0339341],[1617163928330,1007755251.8882328],[1617166924538,1023240773.0466446],[1617171886674,1037535813.1806505],[1617175133694,1101375379.7094195],[1617178435173,1136688478.90344],[1617182857658,1208366620.2561867],[1617185353773,1208823054.3509212],[1617188828477,1234197192.568771],[1617193393471,1707076315.380663],[1617196301983,1845668637.7358408],[1617199516026,1901877634.1385415],[1617203681947,2015292037.1305778],[1617207515426,2141098631.115179],[1617210224998,2343473154.2871637],[1617214323265,2329074198.4966955],[1617217968405,2461828129.1798186],[1617221653017,2493042958.539376],[1617224582971,2532015555.7692595],[1617228589364,2508661361.110037],[1617232204720,2590057969.924583],[1617235260464,2749780924.550207],[1617239367664,2791689438.967896],[1617243152558,2778422749.5901804],[1617246573894,2802892972.2612605],[1617250114952,2795446026.902383],[1617253276300,2837092221.188881],[1617257741390,2957061611.281718],[1617261111556,3025594776.954216],[1617264301698,3140730366.12618],[1617267704421,3230797741.627739],[1617272276500,3247001347.7404704],[1617275862720,3182990384.8873067],[1617279129292,2889317168.9977646],[1617283053665,2753527702.506779],[1617287046529,2700392654.8781624],[1617290204012,2616296684.424929],[1617293298853,2494255828.9768047],[1617296557242,2383424694.8900166],[1617301325511,2288268623.177356],[1617303766777,2297155897.636895],[1617307669347,2314935325.319679],[1617311721980,2259716784.056617],[1617314946823,2267889595.9127536],[1617319572007,2174169254.528509],[1617323182318,2097690604.8152165],[1617326033792,2110975746.1916978],[1617329489226,2126100629.800452],[1617332409284,2193182655.044224],[1617337211709,2199847063.5248647],[1617340611316,2167549077.601362],[1617344146863,2110348803.8388174],[1617347361962,2023115590.5637138],[1617351380142,1864316761.5098753],[1617354151186,1788973202.0040677],[1617359277447,1731207666.0376515],[1617361312976,1418566500.3106787],[1617366169158,1693859181.5518322],[1617369860769,1656689094.290342],[1617372306072,1660176536.7450612],[1617376754676,1722154482.4234965],[1617379285817,1915067128.493045],[1617383311995,1982773491.2907202],[1617387963188,1985155493.939231],[1617391564495,1827213471.6221747],[1617395202777,1932891922.7380657],[1617398214973,1937931474.560893],[1617401809690,1961473630.4188676],[1617405699909,1952347409.661483],[1617409553080,2172811188.054834],[1617412963837,2431917537.219363],[1617416445822,2666886575.1140027],[1617420431122,2769520722.4907126],[1617422613890,2797409323.779513],[1617427393260,2895546310.6951184],[1617431058021,2894169435.883223],[1617433696700,2651591430.614699],[1617437513773,3448548871.8910036],[1617441138039,3537764498.5278754],[1617444820385,3662623380.0181885],[1617448128419,3729999481.3895626],[1617452094944,3741683833.307362],[1617457034540,3761774670.321721],[1617460631688,3809173022.555833],[1617464335978,3711591162.8519845],[1617467879738,3759143118.4621553],[1617471447610,3693936894.7524076],[1617474960418,3833857114.2069917],[1617478639837,3888109113.59996],[1617482233320,3857034438.9984646],[1617485821346,3898924734.2645984],[1617489477282,3952661186.2182713],[1617493109729,4002501827.9437523],[1617495709286,3872814933.0218143],[1617499443431,3939579930.8108554],[1617503699037,3663106636.5813146],[1617507443725,3808705623.491391],[1617510706891,3786240536.055139],[1617512446242,3717882675.3539762],[1617516040645,3722966733.2957063],[1617519813304,3482249884.952562],[1617523351916,3345586253.508183],[1617526909722,3327000473.8244348],[1617530664916,3181835266.2617188],[1617534176048,3094776290.1306324],[1617537924632,3064167829.684326],[1617541493704,3112790145.252149],[1617545018360,2989449570.670528],[1617548594506,3016965749.017692],[1617552471191,2973530338.557288],[1617555933696,2759208177.1915674],[1617559387440,2662906186.1813793],[1617563034515,2521716547.9565806],[1617566483711,2454800946.788864],[1617570325792,2412175803.4922743],[1617573668989,2381142461.766321],[1617577282876,2228904400.2017546],[1617580896737,2203439508.717633],[1617584514686,2083961834.3200803],[1617588367701,1922511436.832222],[1617591869391,1816453643.1859522],[1617595346098,1783362433.1356776],[1617599069131,1767878927.408502],[1617602711113,1782121869.0062866],[1617606278078,1784322317.8294444],[1617609891135,1785304724.1970084],[1617613319383,1792007217.4012969],[1617617302304,1808002080.6732872],[1617620901014,1821923720.87615],[1617624265084,1769426364.6123836],[1617629555312,1731155926.337212],[1617631504259,1735378701.9021676],[1617635133537,1942437073.2385755],[1617638780500,1938122743.6976163],[1617642119732,1932182393.8447528],[1617645707597,1918416705.3436842],[1617649325384,1925855235.7182896],[1617653252063,1944708214.0244768],[1617656889033,1932665022.73478],[1617660329160,1943687775.1192245],[1617663683699,1971924479.2343264],[1617667435208,2101421530.2666874],[1617672769205,2175322213.812557],[1617674524812,2168578229.7784457],[1617678186353,2149217571.1759067],[1617681915267,2132725563.885806],[1617685469475,1907950838.2268875],[1617689189705,2026223167.4473426],[1617692670953,1991840998.8517568],[1617696101989,1958389716.0448081],[1617699877898,2027665770.2623076],[1617703590445,2045913908.1590445],[1617707076556,2057724347.183567],[1617710622851,1722203248.9530182],[1617714225215,2160140597.446546],[1617717905528,2192080372.5552874],[1617721488585,2199844279.449877],[1617724918808,2244159138.5689125],[1617728548093,2263548854.897557],[1617732187891,2106855536.9938018],[1617735969816,2268365061.664965],[1617739538518,1863113060.588111],[1617742875565,2296819840.9881096],[1617746516853,2308037223.56185],[1617750327052,2297405821.9954567],[1617754017835,2215648462.217197],[1617758617023,2112353884.9607923],[1617761085616,2094123582.0260437],[1617764518134,2101292245.7045105],[1617768287923,2104106865.0792534],[1617771810289,2127056476.4717],[1617775566730,2152196953.3590703],[1617778865860,2160666464.579131],[1617782881414,2201171213.1865735],[1617786249160,2203934869.139618],[1617789807394,2329117281.806726],[1617793383957,2333039138.899913],[1617796986959,2491205752.3653517],[1617800521125,2652604590.3673797],[1617804331429,2692817000.168284],[1617807822435,2121796914.212729],[1617811418506,2538097921.330415],[1617815037057,2572049083.87979],[1617818698211,2550478468.4248347],[1617822347031,2541491737.3311806],[1617825969097,2609118564.630648],[1617829326876,2651351577.1099257],[1617833305171,2429954572.560337],[1617837011298,2435043578.3313527],[1617840572965,2394428204.082167],[1617843841041,2446826032.07983],[1617848315742,2395082349.188743],[1617850339793,2376349751.741466],[1617852591890,2385498650.2366877],[1617855126472,2380054416.699361],[1617858732962,2424505564.216302],[1617862619633,2434391633.272485],[1617865876330,2410962812.9744062],[1617869233838,2516114320.406959],[1617872539799,2437748581.3302546],[1617876293610,2247205079.171164],[1617880005259,2149347865.150653],[1617883394235,1893777066.5168178],[1617886836203,1757412804.559377],[1617892197847,1668727963.8671286],[1617894162445,1631584545.4824028],[1617897737215,1596293896.725426],[1617901282046,1525523967.3370435],[1617905003853,1370316987.26801],[1617908631874,1358993841.079183],[1617912335250,1404691449.9164236],[1617915995319,1379405950.1047523],[1617919567600,1366246502.7408085],[1617923270275,1289254721.1461022],[1617926622919,1386402238.6371279],[1617930228859,1384851642.1789908],[1617933705891,1365548610.2907162],[1617937372258,1357266138.9978309],[1617941122560,1335764096.6047564],[1617944870896,1322495289.1105938],[1617948462328,1283751933.8339043],[1617951863802,1272489837.990008],[1617955666499,1259096045.8789752],[1617958890026,1247182948.0102005],[1617962609987,1220448763.9536679],[1617966256703,1222538618.147044],[1617969964555,1148194206.4734476],[1617973333279,1199996169.7479842],[1617977646106,1154935691.529977],[1617980504476,1144564005.003322],[1617984273306,1132822242.6037295],[1617987925282,1136733019.0246003],[1617991396077,1139090847.1565342],[1617994822351,1133169530.4839995],[1617998615234,1113274570.5832539],[1618002141094,1094805189.6349592],[1618005876460,1035579604.067034],[1618009282025,1090335224.3969038],[1618013035782,1063984405.5106469],[1618016519119,1058097513.8615906],[1618020114108,1065381128.0365001]]}}
When this code invoked it logs data correctly but i can not return it at last. Anyone can help?

The problem appears to be that you're trying to return the entire Axios response. This cannot be serialised as JSON due to circular references.
Simply return the response data instead. You can also make your URL construction simpler (and safer) using the params option
exports.getDatas = functions.https.onCall(async ({ name, ts1, ts2 }) => {
functions.logger.log(name);
// 👇 note the destructure here
const { data } = await axios.get(
`https://api.coingecko.com/api/v3/coins/${encodeURIComponent(name)}/market_chart/range`,
{
params: {
vs_currency: "usd",
from: ts1,
to: ts2,
}
}
);
functions.logger.log(data);
return { data };
});

Upload byte array from axios to Node server

Background
Javascript library for Microsoft Office add-ins allows you to get raw content of the DOCX file through getFileAsync() api, which returns a slice of up to 4MB in one go. You keep calling the function using a sliding window approach till you have reed entire content. I need to upload these slices to the server and the join them back to recreate the original DOCX file.
My attempt
I'm using axios on the client-side and busboy-based express-chunked-file-upload middleware on my node server. As I call getFileAsync recursively, I get a raw array of bytes that I then convert to a Blob and append to FormData before posting it to the node server. The entire thing works and I get the slice on the server. However, the chunk that gets written to the disk on the server is much larger than the blob I uploaded, normally of the order of 3 times, so it is obviously not getting what I sent.
My suspicion is that this may have to do with stream encoding, but the node middleware does not expose any options to set encoding.
Here is the current state of code:
Client-side
public sendActiveDocument(uploadAs: string, sliceSize: number): Promise<boolean> {
return new Promise<boolean>((resolve) => {
Office.context.document.getFileAsync(Office.FileType.Compressed,
{ sliceSize: sliceSize },
async (result) => {
if (result.status == Office.AsyncResultStatus.Succeeded) {
// Get the File object from the result.
const myFile = result.value;
const state = {
file: myFile,
filename: uploadAs,
counter: 0,
sliceCount: myFile.sliceCount,
chunkSize: sliceSize
} as getFileState;
console.log("Getting file of " + myFile.size + " bytes");
const hash = makeId(12)
this.getSlice(state, hash).then(resolve(true))
} else {
resolve(false)
}
})
})
}
private async getSlice(state: getFileState, fileHash: string): Promise<boolean> {
const result = await this.getSliceAsyncPromise(state.file, state.counter)
if (result.status == Office.AsyncResultStatus.Succeeded) {
const data = result.value.data;
if (data) {
const formData = new FormData();
formData.append("file", new Blob([data]), state.filename);
const boundary = makeId(12);
const start = state.counter * state.chunkSize
const end = (state.counter + 1) * state.chunkSize
const total = state.file.size
return await Axios.post('/upload', formData, {
headers: {
"Content-Type": `multipart/form-data; boundary=${boundary}`,
"file-chunk-id": fileHash,
"file-chunk-size": state.chunkSize,
"Content-Range": 'bytes ' + start + '-' + end + '/' + total,
},
}).then(async res => {
if (res.status === 200) {
state.counter++;
if (state.counter < state.sliceCount) {
return await this.getSlice(state, fileHash);
}
else {
this.closeFile(state);
return true
}
}
else {
return false
}
}).catch(err => {
console.log(err)
this.closeFile(state)
return false
})
} else {
return false
}
}
else {
console.log(result.status);
return false
}
}
private getSliceAsyncPromise(file: Office.File, sliceNumber: number): Promise<Office.AsyncResult<Office.Slice>> {
return new Promise(function (resolve) {
file.getSliceAsync(sliceNumber, result => resolve(result))
})
}
Server-side
This code is totally from the npm package (link above), so I'm not supposed to change anything in here, but still for reference:
makeMiddleware = () => {
return (req, res, next) => {
const busboy = new Busboy({ headers: req.headers });
busboy.on('file', (fieldName, file, filename, _0, _1) => {
if (this.fileField !== fieldName) { // Current field is not handled.
return next();
}
const chunkSize = req.headers[this.chunkSizeHeader] || 500000; // Default: 500Kb.
const chunkId = req.headers[this.chunkIdHeader] || 'unique-file-id'; // If not specified, will reuse same chunk id.
// NOTE: Using the same chunk id for multiple file uploads in parallel will corrupt the result.
const contentRangeHeader = req.headers['content-range'];
let contentRange;
const errorMessage = util.format(
'Invalid Content-Range header: %s', contentRangeHeader
);
try {
contentRange = parse(contentRangeHeader);
} catch (err) {
return next(new Error(errorMessage));
}
if (!contentRange) {
return next(new Error(errorMessage));
}
const part = contentRange.start / chunkSize;
const partFilename = util.format('%i.part', part);
const tmpDir = util.format('/tmp/%s', chunkId);
this._makeSureDirExists(tmpDir);
const partPath = path.join(tmpDir, partFilename);
const writableStream = fs.createWriteStream(partPath);
file.pipe(writableStream);
file.on('end', () => {
req.filePart = part;
if (this._isLastPart(contentRange)) {
req.isLastPart = true;
this._buildOriginalFile(chunkId, chunkSize, contentRange, filename).then(() => {
next();
}).catch(_ => {
const errorMessage = 'Failed merging parts.';
next(new Error(errorMessage));
});
} else {
req.isLastPart = false;
next();
}
});
});
req.pipe(busboy);
};
}
Update
So it looks like I have found the problem at least. busboy appears to be writing my array of bytes as text in the output file. I get 80,75,3,4,20,0,6,0,8,0,0,0,33,0,44,25 (as text) when I upload the array of bytes [80,75,3,4,20,0,6,0,8,0,0,0,33,0,44,25]. Now need to figure out how to force it to write it as a binary stream.

Figured out. Just in case it helps anyone, there was no problem with busboy or office.js or axios. I just had to convert the incoming chunk of data to Uint8Array before creating a blob from it. So instead of:
formData.append("file", new Blob([data]), state.filename);
like this:
const blob = new Blob([ new Uint8Array(data) ])
formData.append("file", blob, state.filename);
And it worked like a charm.

GraphQL file upload plain JS "Must provide query string."

I am uploading files to GraphQL API with plain JS. I've been doing this from the same origin for months now and now am trying to implement the exact same thing externally with NodeJS.
My code looks something like this:
const FormData = require('form-data');
const fs = require('fs')
const axios = require('axios')
const payload = generateRequest(input)
axios.post(apiBaseUrl + "/graphql", payload, {
headers: {...payload.getHeaders()}
}).then(response => {
let res = response.data
if (res.data.triggerPcWorkflow.status === 200) {
console.log("success!")
} else {
console.error(res.data.triggerPcWorkflow.message)
}
})
.catch(err => {
if (err.response) {
console.log(err.response.data);
console.log(err.response.status);
console.log(err.response.headers);
}
})
With the generateRequest function generating the multipart payload (https://github.com/jaydenseric/graphql-multipart-request-spec).
I have two identical versions of the Backend running on localhost:5000 and mycooldomain.com. When setting apiBaseUrl to http://localhost:5000 everything works flawlessly. However just by changing the URL to https://www.mycooldmain.com I get a 400 error thrown at me with { errors: [ { message: 'Must provide query string.' } ] }
BTW: A simple query works with both URLs...
Here is my generateRequest function:
const mutation = `
mutation MyMutation($xyz: String) {
doSomething(myInput: $xyz) {
status
message
}
}
`
let sendData = new FormData();
const fileNull = [];
// map
files = generateRequestInput.files
let map = '{'
for (let i = 0; i < files.length; i++) {
fileNull.push(null);
map += `"${i}": ["variables.files.${i}"], `
}
map = map.substring(0, map.length-2);
map += '}'
// operations
const operations = JSON.stringify({
query: mutation,
variables: {
"xyz": "Hello"
}
});
// build payload
sendData.append("operations", operations)
sendData.append("map", map)
for (let i = 0; i < files.length; i++) {
sendData.append(i, files[i]);
}
return sendData
I know that map looks a bit ugly but thats not the point (unless it is).
Has anyone had a similar problem or knows what my error is here?
Thanks!

I skipped on the axios dependency and implemented the request with FormData directly.
The code below works.
function makeRequest(formData, options) {
formData.submit(options, (err, res) => {
if (err) {
console.error(err.message)
return
} else {
if (res.statusCode < 200 || res.statusCode > 299) {
console.error(`HTTP status code ${res.statusCode}`)
}
const body = []
res.on('data', (chunk) => body.push(chunk))
res.on('end', () => {
const resString = Buffer.concat(body).toString()
console.log(resString)
})
}
})
}
const options = {
host: 'mycooldomain.com',
port: 443,
path: '/graphql',
method: 'POST',
protocol: 'https:'
}
makeRequest(payload, options)

How to respond to post request in Node.js?

My client-side code is something like:
let data = "Some data";
let request = new XMLHttpRequest();
let link = new URL("http://someurl.com");
link.searchParams.set("data", data);
request.onreadystatechange = function() {
if (this.readyState === 4 && this.status === 200) {
//some code
}
}
request.open("POST", link, true);
request.send();
How can I retrieve "data" on the sever with node.js?

The solution is wouldn't be simple.
First of all, you should use createServer from one of the native libraries http or https. Also, you should know how does stream and events work in Node.js
const https = require('https');
const srv = https.createServer(...);
const SRV_HOST = '127.0.0.1';
const SRV_PORT = 5000;
const MAX_PAYLOAD = 2048;
// You should provide your custom code for these events
srv.on('connection', onClientConnection);
srv.on('request', onClientRequest);
srv.on('clientError', onClientError);
srv.on('close', onServerClose);
// Start listening...
srv.listen(SRV_PORT, SRV_HOST, onStartListening);
function onClientRequest(req, res) {
const connectionId = `${req.client.remoteAddress}:${req.client.remotePort}`;
const dataPocket = {
chunks: [],
bytes: 0
};
req.on('data', chunk => onClientData.call(req, res, chunk, dataPocket));
req.on('end', onClientEnd.bind(this, req, res, dataPocket));
req.on('error', onClientRequestError);
res.on('close', onClientClose.bind(res, connectionId));
}
function onClientData(res, chunk, dataPocket) {
dataPocket.bytes += chunk.length;
if (dataPocket.bytes <= MAX_PAYLOAD) {
dataPocket.chunks.push(chunk);
} else {
const message = `Data limit exceeded. Maximum ${MAX_PAYLOAD} bytes are expected`;
const error = new Error(message);
error.code = 'ECONNRESET';
return this.emit('error', error);
}
}
function onClientEnd(req, res, dataPocket) {
const data = collectData(req, dataPocket.chunks);
// Process your data here
const responseData = { success: true, data: data };
const jsonData = JSON.stringify(responseData);
// Set headers and status
res.setHeader('Content-Type', 'application/json');
res.statusCode = 200;
// Send data
res.end(jsonData);
// Cleanup data
dataPocket.chunks = [];
dataPocket.bytes = 0;
}
You can look an example in my GitHub profile

You can use the http package. Nowadays you can use an async iterator on streams. That makes reading the data and error handling more easy and less likely to cause memory leaks.
const { createServer } = require('http');
createServer(function (req, res) {
if (req.method === 'POST') {
req.setEncoding('utf-8');
(async (readable) => {
for await (let chunk of readable) {
// do here with the data what you want
console.log(chunk);
}
res.writeHead(201);
res.end();
})(req).catch((err) => {
res.writeHead(500);
return res.end(JSON.stringify(err));
});
} else {
res.writeHead(404);
res.end()
}
}).listen(8080);
If you are interested in piping the output to another stream, you could also use the pipeline method for that.
You can also check this repl to get a live demo.