My client-side code is something like:
let data = "Some data";
let request = new XMLHttpRequest();
let link = new URL("http://someurl.com");
link.searchParams.set("data", data);
request.onreadystatechange = function() {
if (this.readyState === 4 && this.status === 200) {
//some code
}
}
request.open("POST", link, true);
request.send();
How can I retrieve "data" on the sever with node.js?
The solution is wouldn't be simple.
First of all, you should use createServer from one of the native libraries http or https. Also, you should know how does stream and events work in Node.js
const https = require('https');
const srv = https.createServer(...);
const SRV_HOST = '127.0.0.1';
const SRV_PORT = 5000;
const MAX_PAYLOAD = 2048;
// You should provide your custom code for these events
srv.on('connection', onClientConnection);
srv.on('request', onClientRequest);
srv.on('clientError', onClientError);
srv.on('close', onServerClose);
// Start listening...
srv.listen(SRV_PORT, SRV_HOST, onStartListening);
function onClientRequest(req, res) {
const connectionId = `${req.client.remoteAddress}:${req.client.remotePort}`;
const dataPocket = {
chunks: [],
bytes: 0
};
req.on('data', chunk => onClientData.call(req, res, chunk, dataPocket));
req.on('end', onClientEnd.bind(this, req, res, dataPocket));
req.on('error', onClientRequestError);
res.on('close', onClientClose.bind(res, connectionId));
}
function onClientData(res, chunk, dataPocket) {
dataPocket.bytes += chunk.length;
if (dataPocket.bytes <= MAX_PAYLOAD) {
dataPocket.chunks.push(chunk);
} else {
const message = `Data limit exceeded. Maximum ${MAX_PAYLOAD} bytes are expected`;
const error = new Error(message);
error.code = 'ECONNRESET';
return this.emit('error', error);
}
}
function onClientEnd(req, res, dataPocket) {
const data = collectData(req, dataPocket.chunks);
// Process your data here
const responseData = { success: true, data: data };
const jsonData = JSON.stringify(responseData);
// Set headers and status
res.setHeader('Content-Type', 'application/json');
res.statusCode = 200;
// Send data
res.end(jsonData);
// Cleanup data
dataPocket.chunks = [];
dataPocket.bytes = 0;
}
You can look an example in my GitHub profile
You can use the http package. Nowadays you can use an async iterator on streams. That makes reading the data and error handling more easy and less likely to cause memory leaks.
const { createServer } = require('http');
createServer(function (req, res) {
if (req.method === 'POST') {
req.setEncoding('utf-8');
(async (readable) => {
for await (let chunk of readable) {
// do here with the data what you want
console.log(chunk);
}
res.writeHead(201);
res.end();
})(req).catch((err) => {
res.writeHead(500);
return res.end(JSON.stringify(err));
});
} else {
res.writeHead(404);
res.end()
}
}).listen(8080);
If you are interested in piping the output to another stream, you could also use the pipeline method for that.
You can also check this repl to get a live demo.
Related
index.js file
import http from 'http';
function terminalConsole() {
console.log(5);
}
const server = http.createServer((req, res) => {
terminalConsole();
res.statusCode = 200;
res.setHeader('Content-Type', 'text/plain');
res.end('Hello');
});
server.listen(5000, '127.0.0.1', () => {
console.log(`server listening on localhost:5000`);
});
And when called from terminal, getting two 5 output instead of one.
Stackblitz link - https://stackblitz.com/edit/node-t5zngd?file=index.js
You can use a function to log only once, based on the content of the log:
const logOnceClass = () => {
const cache = new Map()
return (string, logFunction = console.log) => {
// if its already logged, return
if (cache.has(string)) return
logFunction(string)
cache.set(string, true)
}
}
const logOnce = logOnceClass()
logOnce(1) // 1
logOnce(1) // (No log, "1" has been already logged)
logOnce(2) // 2
Use req.on node.js event in-order to log to console on a specific event.
Here is the fixed version of your code.
const http = require("http");
function terminalConsole() {
console.log(5);
}
const server = http.createServer((req, res) => {
res.setHeader("Content-Type", "text/plain");
res.statusCode = 200;
req.on('end', () => {
terminalConsole();
});
res.end("Hello");
});
server.listen(3001, () => {
console.log(`server listening on localhost:3001`);
});
What happens here is that every time a request has been end, it would log the message to console.
Background
Javascript library for Microsoft Office add-ins allows you to get raw content of the DOCX file through getFileAsync() api, which returns a slice of up to 4MB in one go. You keep calling the function using a sliding window approach till you have reed entire content. I need to upload these slices to the server and the join them back to recreate the original DOCX file.
My attempt
I'm using axios on the client-side and busboy-based express-chunked-file-upload middleware on my node server. As I call getFileAsync recursively, I get a raw array of bytes that I then convert to a Blob and append to FormData before posting it to the node server. The entire thing works and I get the slice on the server. However, the chunk that gets written to the disk on the server is much larger than the blob I uploaded, normally of the order of 3 times, so it is obviously not getting what I sent.
My suspicion is that this may have to do with stream encoding, but the node middleware does not expose any options to set encoding.
Here is the current state of code:
Client-side
public sendActiveDocument(uploadAs: string, sliceSize: number): Promise<boolean> {
return new Promise<boolean>((resolve) => {
Office.context.document.getFileAsync(Office.FileType.Compressed,
{ sliceSize: sliceSize },
async (result) => {
if (result.status == Office.AsyncResultStatus.Succeeded) {
// Get the File object from the result.
const myFile = result.value;
const state = {
file: myFile,
filename: uploadAs,
counter: 0,
sliceCount: myFile.sliceCount,
chunkSize: sliceSize
} as getFileState;
console.log("Getting file of " + myFile.size + " bytes");
const hash = makeId(12)
this.getSlice(state, hash).then(resolve(true))
} else {
resolve(false)
}
})
})
}
private async getSlice(state: getFileState, fileHash: string): Promise<boolean> {
const result = await this.getSliceAsyncPromise(state.file, state.counter)
if (result.status == Office.AsyncResultStatus.Succeeded) {
const data = result.value.data;
if (data) {
const formData = new FormData();
formData.append("file", new Blob([data]), state.filename);
const boundary = makeId(12);
const start = state.counter * state.chunkSize
const end = (state.counter + 1) * state.chunkSize
const total = state.file.size
return await Axios.post('/upload', formData, {
headers: {
"Content-Type": `multipart/form-data; boundary=${boundary}`,
"file-chunk-id": fileHash,
"file-chunk-size": state.chunkSize,
"Content-Range": 'bytes ' + start + '-' + end + '/' + total,
},
}).then(async res => {
if (res.status === 200) {
state.counter++;
if (state.counter < state.sliceCount) {
return await this.getSlice(state, fileHash);
}
else {
this.closeFile(state);
return true
}
}
else {
return false
}
}).catch(err => {
console.log(err)
this.closeFile(state)
return false
})
} else {
return false
}
}
else {
console.log(result.status);
return false
}
}
private getSliceAsyncPromise(file: Office.File, sliceNumber: number): Promise<Office.AsyncResult<Office.Slice>> {
return new Promise(function (resolve) {
file.getSliceAsync(sliceNumber, result => resolve(result))
})
}
Server-side
This code is totally from the npm package (link above), so I'm not supposed to change anything in here, but still for reference:
makeMiddleware = () => {
return (req, res, next) => {
const busboy = new Busboy({ headers: req.headers });
busboy.on('file', (fieldName, file, filename, _0, _1) => {
if (this.fileField !== fieldName) { // Current field is not handled.
return next();
}
const chunkSize = req.headers[this.chunkSizeHeader] || 500000; // Default: 500Kb.
const chunkId = req.headers[this.chunkIdHeader] || 'unique-file-id'; // If not specified, will reuse same chunk id.
// NOTE: Using the same chunk id for multiple file uploads in parallel will corrupt the result.
const contentRangeHeader = req.headers['content-range'];
let contentRange;
const errorMessage = util.format(
'Invalid Content-Range header: %s', contentRangeHeader
);
try {
contentRange = parse(contentRangeHeader);
} catch (err) {
return next(new Error(errorMessage));
}
if (!contentRange) {
return next(new Error(errorMessage));
}
const part = contentRange.start / chunkSize;
const partFilename = util.format('%i.part', part);
const tmpDir = util.format('/tmp/%s', chunkId);
this._makeSureDirExists(tmpDir);
const partPath = path.join(tmpDir, partFilename);
const writableStream = fs.createWriteStream(partPath);
file.pipe(writableStream);
file.on('end', () => {
req.filePart = part;
if (this._isLastPart(contentRange)) {
req.isLastPart = true;
this._buildOriginalFile(chunkId, chunkSize, contentRange, filename).then(() => {
next();
}).catch(_ => {
const errorMessage = 'Failed merging parts.';
next(new Error(errorMessage));
});
} else {
req.isLastPart = false;
next();
}
});
});
req.pipe(busboy);
};
}
Update
So it looks like I have found the problem at least. busboy appears to be writing my array of bytes as text in the output file. I get 80,75,3,4,20,0,6,0,8,0,0,0,33,0,44,25 (as text) when I upload the array of bytes [80,75,3,4,20,0,6,0,8,0,0,0,33,0,44,25]. Now need to figure out how to force it to write it as a binary stream.
Figured out. Just in case it helps anyone, there was no problem with busboy or office.js or axios. I just had to convert the incoming chunk of data to Uint8Array before creating a blob from it. So instead of:
formData.append("file", new Blob([data]), state.filename);
like this:
const blob = new Blob([ new Uint8Array(data) ])
formData.append("file", blob, state.filename);
And it worked like a charm.
I am uploading files to GraphQL API with plain JS. I've been doing this from the same origin for months now and now am trying to implement the exact same thing externally with NodeJS.
My code looks something like this:
const FormData = require('form-data');
const fs = require('fs')
const axios = require('axios')
const payload = generateRequest(input)
axios.post(apiBaseUrl + "/graphql", payload, {
headers: {...payload.getHeaders()}
}).then(response => {
let res = response.data
if (res.data.triggerPcWorkflow.status === 200) {
console.log("success!")
} else {
console.error(res.data.triggerPcWorkflow.message)
}
})
.catch(err => {
if (err.response) {
console.log(err.response.data);
console.log(err.response.status);
console.log(err.response.headers);
}
})
With the generateRequest function generating the multipart payload (https://github.com/jaydenseric/graphql-multipart-request-spec).
I have two identical versions of the Backend running on localhost:5000 and mycooldomain.com. When setting apiBaseUrl to http://localhost:5000 everything works flawlessly. However just by changing the URL to https://www.mycooldmain.com I get a 400 error thrown at me with { errors: [ { message: 'Must provide query string.' } ] }
BTW: A simple query works with both URLs...
Here is my generateRequest function:
const mutation = `
mutation MyMutation($xyz: String) {
doSomething(myInput: $xyz) {
status
message
}
}
`
let sendData = new FormData();
const fileNull = [];
// map
files = generateRequestInput.files
let map = '{'
for (let i = 0; i < files.length; i++) {
fileNull.push(null);
map += `"${i}": ["variables.files.${i}"], `
}
map = map.substring(0, map.length-2);
map += '}'
// operations
const operations = JSON.stringify({
query: mutation,
variables: {
"xyz": "Hello"
}
});
// build payload
sendData.append("operations", operations)
sendData.append("map", map)
for (let i = 0; i < files.length; i++) {
sendData.append(i, files[i]);
}
return sendData
I know that map looks a bit ugly but thats not the point (unless it is).
Has anyone had a similar problem or knows what my error is here?
Thanks!
I skipped on the axios dependency and implemented the request with FormData directly.
The code below works.
function makeRequest(formData, options) {
formData.submit(options, (err, res) => {
if (err) {
console.error(err.message)
return
} else {
if (res.statusCode < 200 || res.statusCode > 299) {
console.error(`HTTP status code ${res.statusCode}`)
}
const body = []
res.on('data', (chunk) => body.push(chunk))
res.on('end', () => {
const resString = Buffer.concat(body).toString()
console.log(resString)
})
}
})
}
const options = {
host: 'mycooldomain.com',
port: 443,
path: '/graphql',
method: 'POST',
protocol: 'https:'
}
makeRequest(payload, options)
I'm uploading files from the browser via a multipart request to a GraphQL-API which is powered by graphql-yoga which is powered by express.
Now I want to forward this exact same request body to another GraphQL-API.
const fetch = require('node-fetch');
async passThrough(args, opts) {
const { body, getRawBody, headers, method } = opts.request;
var rawBody;
if (body.files && body.files.length) {
rawBody = await getRawBody;
} else {
rawBody = typeof body == 'string' ? body : JSON.stringify(body)
}
let options = {
body: rawBody,
method, headers
};
var res = await fetch(otherApiUrl, options).then((res) => {
return res.json();
});
return res;
}
In this function I get the body as an object. But it includes "files" as promises which I can't simply forward (Couldn't find anything to do it). So I tried to get the raw body through a express middleware and access it like above with await getRawBody.
function getRawBody(req, res, next) {
req.getRawBody = new Promise(resolve => {
var buf = '';
req.on('data', x => buf += x);
req.on('end', () => {
resolve(buf);
});
});
next();
}
server.express.use(getRawBody);
It passes the request to the other API but the files are no valid jpegs anymore. I found out, that the uploaded file is shifted some bits from the original file. What am I maybe doing wrong?
I found a solution here and adapted the function to get the raw body. Now the file contents are not shifted anymore on the target host.
const concatStream = require('concat-stream');
function getRawBody(req, res, next) {
req.getRawBody = new Promise(resolve => {
req.pipe(concatStream(function (data) {
resolve(data);
}));
});
next();
}
basically what I want to achieve is check in a middleware whether an uploaded file has the correct image type (png for example). This is what I have come up with till now:
export const fileCheckMiddleware = (req, res, next) => {
const acceptedImageTypes = ["image/gif", "image/jpeg", "image/png"];
const oldWrite = res.write;
const oldEnd = res.end;
const chunks = [];
res.write = (...restArgs) => {
chunks.push(new Buffer(restArgs[0]));
oldWrite.apply(res, restArgs);
};
res.end = async (...restArgs) => {
if (restArgs[0]) {
chunks.push(new Buffer(restArgs[0]));
}
const body = Buffer.concat(chunks).toString("utf8");
try {
let parsedBody = {};
try {
parsedBody = JSON.parse(body);
} catch (err) {
parsedBody = { data: { unparsedBody: body } };
}
const { variables } = req.body;
console.log("\x1b[1m%s\x1b[0m", "LOG variables", variables.file);
if (variables.file) {
console.log("\x1b[1m%s\x1b[0m", "LOG type", typeof variables.file);
}
} catch (err) {}
oldEnd.apply(res, restArgs);
};
next();
};
The logged type of variables.file is an object. And the result of the console.log is this:
LOG variables Promise {
{ filename: 'trump.jpeg',
mimetype: 'image/jpeg',
encoding: '7bit',
createReadStream: [Function: createReadStream] } }
So how can I access the mimetype here? I tried to map over the keys, variables.file["Promise"],...
Promise is not a key of variables.file, it's the type of variables.file. That means your code starts executing as soon as the HTTP request starts, and the file is received asynchronously, so you have to do something like:
variables.file.then(file => {
// Do whatever you want with the file
next();
});
Or declare the surrounding function as async and do this:
const file = await variables.file;
// Do whatever you want with the file
next();