I need a nodejs stream (http://nodejs.org/api/stream.html) implementation that sends data to a string. Do you know anyone?
To be direct I'm trying to pipe a request response like this:
request('http://google.com/doodle.png').pipe(fs.createWriteStream('doodle.png'))
FROM https://github.com/mikeal/request
Thanks
It would not be difficult to write a class that conforms to the Stream interface; here's an example that implements the very basics, and seems to work with the request module you linked:
var stream = require('stream');
var util = require('util');
var request = require('request');
function StringStream() {
stream.Stream.call(this);
this.writable = true;
this.buffer = "";
};
util.inherits(StringStream, stream.Stream);
StringStream.prototype.write = function(data) {
if (data && data.length)
this.buffer += data.toString();
};
StringStream.prototype.end = function(data) {
this.write(data);
this.emit('end');
};
StringStream.prototype.toString = function() {
return this.buffer;
};
var s = new StringStream();
s.on('end', function() {
console.log(this.toString());
});
request('http://google.com').pipe(s);
You might find the class Sink in the pipette module to be handy for this use case. Using that you can write:
var sink = new pipette.Sink(request(...));
sink.on('data', function(buffer) {
console.log(buffer.toString());
}
Sink also handles error events coming back from the stream reasonably gracefully. See https://github.com/Obvious/pipette#sink for details.
[Edit: because I realized I used the wrong event name.]
Related
Base problem: display a H264 live stream in a browser.
Solution: let's just convert it to fragmented mp4 and load chunk-by-chunk via websocket (or XHR) into MSE.
Sounds too easy. But I want to do the fragmentation on client side with pure JS.
So I'm trying to use MP4Box.js. On its readme page it states: it has a demo: "A player that performs on-the-fly fragmentation".
That's the thing I need!
However the onSegment callbacks which should feed MSE are not called at all:
var ws; //for websocket
var mp4box; //for fragmentation
function startVideo() {
mp4box = MP4Box.createFile();
mp4box.onError = function(e) {
console.log("mp4box failed to parse data.");
};
mp4box.onMoovStart = function () {
console.log("Starting to receive File Information");
};
mp4box.onReady = function(info) {
console.log(info.mime);
mp4box.onSegment = function (id, user, buffer, sampleNum) {
console.log("Received segment on track "+id+" for object "+user+" with a length of "+buffer.byteLength+",sampleNum="+sampleNum);
}
var options = { nbSamples: 1000 };
mp4box.setSegmentOptions(info.tracks[0].id, null, options); // I don't need user object this time
var initSegs = mp4box.initializeSegmentation();
mp4box.start();
};
ws = new WebSocket("ws://a_websocket_server_which_serves_h264_file");
ws.binaryType = "arraybuffer";
ws.onmessage = function (event) {
event.data.fileStart = 0; //tried also with event.data.byteOffset, but resulted error.
var nextBufferStart = mp4box.appendBuffer(event.data);
mp4box.flush(); //tried commenting out - unclear documentation!
};
}
window.onload = function() {
startVideo();
}
Now putting this into an HTML file would result this in the JavaScript console:
Starting to receive File Information
video/mp4; codecs="avc1.4d4028"; profiles="isom,iso2,avc1,iso6,mp41"
But nothing happens afterwards. Why is the onSegment not called here? (the h264 file which the websocket-server serves is playable in VLC - however it is not fragmented)
The problem was using the nextBufferStart in a wrong way.
This should be the correct one:
var nextBufferStart = 0;
...
ws.onmessage = function (event) {
event.data.fileStart = nextBufferStart;
nextBufferStart = mp4box.appendBuffer(event.data);
mp4box.flush();
};
I'm trying to write a node module that accepts an incoming piped binary (or base-64-encoded) stream, but frankly I don't even know where to start. I can't see any examples in the Node docs about handling incoming streams; I only see examples on consuming them?
Say for example I want to be able to do this:
var asset = new ProjectAsset('myFile', __dirname + '/image.jpg')
var stream = fs.createReadStream(__dirname + '/image.jpg', { encoding: 'base64' }).pipe(asset)
stream.on('finish', function() {
done()
})
I've gotten ProjectAsset looking like this, but I'm at a loss of where to go next:
'use strict'
var stream = require('stream'),
util = require('util')
var ProjectAsset = function() {
var self = this
Object.defineProperty(self, 'binaryData', {
configurable: true,
writable: true
})
stream.Stream.call(self)
self.on('pipe', function(src) {
// does it happen here? how do I set self.binaryData?
})
return self
}
util.inherits(ProjectAsset, stream.Stream)
module.exports = ProjectAsset
module.exports.DEFAULT_FILE_NAME = 'file'
It is possible to inherit from stream.Stream and make it work, however based on what's available in the documentation I would suggest inheriting from stream.Writable. Piping into a stream.Writable you'll need to have _write(chunk, encoding, done) defined to handle the piping. Here is an example:
var asset = new ProjectAsset('myFile', __dirname + '/image.jpg')
var stream = fs.createReadStream(__dirname + '/image.jpg', { encoding: 'base64' }).pipe(asset)
stream.on('finish', function() {
console.log(asset.binaryData);
})
Project Asset
'use strict'
var stream = require('stream'),
util = require('util')
var ProjectAsset = function() {
var self = this
self.data
self.binaryData = [];
stream.Writable.call(self)
self._write = function(chunk, encoding, done) {
// Can handle this data however you want
self.binaryData.push(chunk.toString())
// Call after processing data
done()
}
self.on('finish', function() {
self.data = Buffer.concat(self.binaryData)
})
return self
}
util.inherits(ProjectAsset, stream.Writable)
module.exports = ProjectAsset
module.exports.DEFAULT_FILE_NAME = 'file'
If you're looking to also read from the stream, take a look at inheriting from stream.Duplex and also including the _read(size) method.
There's also the simplified constructors api if you're doing something simpler.
Im not sure if this is exaclty what you were looking for but i think you could handle it using the buffer api with Buffer.concat on an array of buffers that can be retrieved form chunk on the stream data listener
'use strict'
var stream = require('stream'),
util = require('util');
var ProjectAsset = function() {
var self = this
Object.defineProperty(self, 'binaryData', {
configurable: true,
writable: true
})
stream.Stream.call(self)
var data;
var dataBuffer=[];
self.on('data', function(chunk) {
dataBuffer.push(chunk);
}).on('end',function(){
data=Buffer.concat(dataBuffer);
});
self.binaryData=data.toString('binary');
return self
}
util.inherits(ProjectAsset, stream.Stream)
module.exports = ProjectAsset
module.exports.DEFAULT_FILE_NAME = 'file'
Since your using var asset = new ProjectAsset('myFile', __dirname + '/image.jpg') I suppose your ProjectAsset responsibility is to take some input stream do some transformations and write that to a file. You could implement a transform stream because you receive some input from a stream and generate some output of it that can be saved to a file or to some other write stream.
You could of course implement a transform stream by inheriting from node.js Transform Stream but inheriting is quite cumbersome so my implementation uses through2 to implement the transform stream:
module.exports = through2(function (chunk, enc, callback) {
// This function is called whenever a piece of data from the incoming stream is read
// Transform the chunk or buffer the chunk in case you need more data to transform
// Emit a data package to the next stream in the pipe or omit this call if you need more data from the input stream to be read
this.push(chunk);
// Signal through2 that you processed the incoming data package
callback();
}))
Usage
var stream = fs.createReadStream(__dirname + '/image.jpg', { encoding: 'base64' })
.pipe(projectAsset)
.pipe(fs.createWriteStream(__dirname + '/image.jpg'));
As you can see in this example implementing a stream pipeline fully decouples data transformation and saving of the data.
Factory Function
If you prefer to use a constructor like approach in the project asset module because you need to pass some values or things you could easily export a constructor function as shown below
var through2 = require('through2');
module.exports = function(someData) {
// New stream is returned that can use someData argument for doing things
return through2(function (chunk, enc, callback) {
// This function is called whenever a piece of data from the incoming stream is read
// Transform the chunk or buffer the chunk in case you need more data to transform
// Emit a data package to the next stream in the pipe or omit this call if you need more data from the input stream to be read
this.push(chunk);
// Signal through2 that you processed the incoming data package
callback();
});
}
Usage
var stream = fs.createReadStream(__dirname + '/image.jpg', { encoding: 'base64' })
.pipe(projectAsset({ foo: 'bar' }))
.pipe(fs.createWriteStream(__dirname + '/image.jpg'));
I need to capture microphone audio in IE10. So far I have two semi-working solutions:
getUserMedia from Microsoft's experimental WebRTC plugin:
http://www.html5labs.com/prototypes/media-capture-api-(2nd-updated)/media-capture-api-(2nd-update)/info
The issue with this is that while I can capture and replay the audio in the browser, I cannot send the audio to the server. In particular, it is not clear how to extract the audio data from the "blob" object:
function msStopRecordCallback(blob) {
console.log(blob) // outputs {}
console.dir(blob) // outputs {}
playMediaObject.Play(blob); // This works!
}
jRecorder: http://www.sajithmr.me/jrecorder-jquery The issue with this is that it relies on Flash to capture the audio, which is something I would like to avoid.
Are there any other ways to capture audio in IE10?
I recognize that my answer a bit late, but...
You may upload a blob to a server as following (Javascript):
function saveBlob(blob)
{
var uploader = new CustomXMLHttpRequest();
uploader.onpartreceived = function (response)
{
// TODO: handle the server response here
};
var base = window.location.toString();
var uploadService = base.substr(0, base.lastIndexOf("/")) + "/api/upload";
uploader.open("POST", uploadService, true);
uploader.responseType = "text";
var form = new FormData();
form.append("fname", blob, "audio.wav");
uploader.send(form);
}
On the server side, you may treat this blob as a file attachment, e.g. (C#):
public class UploadController : ApiController
{
public async Task<HttpResponseMessage> PostFile()
{
// Check if the request contains multipart/form-data.
if (!Request.Content.IsMimeMultipartContent())
{
throw new HttpResponseException(HttpStatusCode.UnsupportedMediaType);
}
var root = HttpContext.Current.Server.MapPath("~/App_Data");
var provider = new MultipartFormDataStreamProvider(root);
try
{
// Read the form data and return an async task.
await Request.Content.ReadAsMultipartAsync(provider);
var fileName = "";
// get the uploaded files.
foreach (var data in provider.FileData)
{
var file = new FileInfo(data.LocalFileName);
// TODO: handle received file here
}
if (string.IsNullOrEmpty(fileName))
{
return Request.CreateResponse(HttpStatusCode.UnsupportedMediaType);
}
return Request.CreateResponse(HttpStatusCode.OK);
}
catch (System.Exception e)
{
return Request.CreateErrorResponse(HttpStatusCode.InternalServerError, e);
}
}
}
Hope this will help.
I am trying to use browserify to access a local binary file (that is, the binary file is in the same directory as the javascript file, which is in the user's computer).
I haven't succeeded. Here's what I tried and what I know:
~) I know fs won't work...
0) I tried using the require('html') but it says 'ajax not supported in this browser' [I am using chromium... but I'd assume it's roughly the same thing as chrome].
1) I tried using 'browser-request'. This reads the binary file... as a string.
It is based on 'request' so I should be able to configure the options, including encoding: null, which would solve all my problems but...looking at the source code, you'll see that no support for the encoding option is present. Not even a warning.
2) I used xmlhttprequest, which required the 'html' module... so again, I get the same error as in 0) Strangely enough, 'browser-request' uses this module and it works... and I have absolutely no idea why.
3) At this point, I looked into html5 file system support. It would work but I don't want the user to specify a file... seeing as I really ONLY want to get the buffer to memory. Is there any other way to access the file? Perhaps using --allow-file-access when starting chromium?
4) If all else fails, I just want a way to get the Buffer into my code. I guess I could just use node on shell and copy paste the result of reading the file into memory...
Is there any hope at all?
Here's what somewhat works:
function toArrayBuffer(buffer) {
var ab = new ArrayBuffer(buffer.length);
var view = new Uint8Array(ab);
for (var i = 0; i < buffer.length; ++i) {
view[i] = buffer[i];
}
return ab;
}
// node: readFileSync + toArrayBuffer
// browser: ajax http request
function readFile(filename, doneCb) {
var isNode =
typeof global !== "undefined" &&
global.toString() == '[object global]';
if (isNode) {
var fs = require('fs');
var buffer = fs.readFileSync(filename);
buffer = toArrayBuffer(buffer);
doneCb(buffer);
} else {
var http = require('http');
var buf;
var req = http.get({ path : '/'+ filename }, function (res) {
res.on('data', function (chunk) {
buf = chunk;
});
res.on('end', function () {
doneCb(buf);
});
});
req.xhr.responseType = 'arraybuffer';
}
}
It requires a server and I'm strugging with on how to make it work in testling.
Another approach I can think of is to use brfs with base64 encoding:
var base64 = fs.readFileSync('file.bin', enc='base64');
var buf = new Buffer(base64, 'base64');
var ab = toArrayBuffer(buf);
It is simpler, but it is not dynamic and cannot be refactored to self-contained function.
If it's not dynamic use brfs transform.
I have a C++ function which once called consumes input from stdin. Exporting this function to javascript using emscripten causes calls to window.prompt.
Interacting with browser prompt is really tedious task. First of all you can paste only one line at time. Secondly the only way to indicate EOF is by pressing 'cancel'. Last but not least the only way (in case of my function) to make it stop asking user for input by window.prompt is by checking the checkbox preventing more prompts to pop up.
For me the best input method would be reading some blob. I know I can hack library.js but I see some problems:
Reading blob is asynchronous.
To read a blob, first you have to open a file user has to select first.
I don't really know how to prevent my function from reading this blob forever - there is no checkbox like with window.prompt and I'm not sure if spotting EOF will stop it if it didn't in window.prompt case (only checking a checkbox helped).
The best solution would be some kind of callback but I would like to see sime hints from more experienced users.
A way would be to use the Emscripten Filesystem API, for example by calling FS.init in the Module preRun function, passing a custom function as the standard input.
var Module = {
preRun: function() {
function stdin() {
// Return ASCII code of character, or null if no input
}
var stdout = null; // Keep as default
var stderr = null; // Keep as default
FS.init(stdin, stdout, stderr);
}
};
The function is quite low-level: is must deal with one character at a time. To read some data from a blob, you could do something like:
var data = new Int8Array([1,2,3,4,5]);
var blob = new Blob([array], {type: 'application/octet-binary'});
var reader = new FileReader();
var result;
reader.addEventListener("loadend", function() {
result = new Int8Array(reader.result);
});
var i = 0;
var Module = {
preRun: function() {
function stdin() {
if (if < result.byteLength {
var code = result[i];
++i;
return code;
} else {
return null;
}
}
var stdout = null; // Keep as default
var stderr = null; // Keep as default
FS.init(stdin, stdout, stderr);
}
};
Note (as you have hinted), due to the asynchronous nature of the reader, there could be a race condition: the reader must have loaded before you can expect the data at the standard input. You might need to implement some mechanism to avoid this in a real case. Depending on your exact requirements, you could make it so the Emscripten program doesn't actually call main() until you have the data:
var fileRead = false;
var initialised = false;
var result;
var array = new Int8Array([1,2,3,4,5]);
var blob = new Blob([array], {type: 'application/octet-binary'});
var reader = new FileReader();
reader.addEventListener("loadend", function() {
result = new Int8Array(reader.result);
fileRead = true;
runIfCan();
});
reader.readAsArrayBuffer(blob);
var i = 0;
var Module = {
preRun: function() {
function stdin() {
if (i < result.byteLength)
{
var code = result[i];
++i;
return code;
} else{
return null;
}
}
var stdout = null;
var stderr = null;
FS.init(stdin, stdout, stderr);
initialised = true;
runIfCan();
},
noInitialRun: true
};
function runIfCan() {
if (fileRead && initialised) {
// Module.run() doesn't seem to work here
Module.callMain();
}
}
Note: this is a version of my answer at Providing stdin to an emscripten HTML program? , but with focus on the standard input, and adding parts about passing data from a Blob.
From what I understand you could try the following:
Implement selecting a file in Javascript and access it via Javascript Blob interface.
Allocate some memory in Emscripten
var buf = Module._malloc( blob.size );
Write the content of your Blob into the returned memory location from Javascript.
Module.HEAPU8.set( new Uint8Array(blob), buf );
Pass that memory location to a second Emscripten compiled function, which then processes the file content and
Deallocate allocated memory.
Module._free( buf );
Best to read the wiki first.