how to move heavy geocoding to a web-worker? - javascript

I have very big list of geo points which I want to translate to coordinates with the geocoder and the best way I think is to move the task to a web worker, otherwise the Firefox times out and never loads the page.
// the main html file:
var myWorker = new Worker('datapointscollection.js');
myWorker.onmessage = function(e) {
document.getElementById('loadingStatus').innerHTML = count + " elements from " + all + "are ready.";
if (count == all) {
myWorker.terminate();
myWorker = undefined;
}
};
myWorker.postMessage([geocodingParams]);
// the worker js file:
self.importScripts("http://js.api.here.com/v3/3.0/mapsjs-core.js");
self.importScripts("http://js.api.here.com/v3/3.0/mapsjs-service.js");
self.importScripts("http://js.api.here.com/v3/3.0/mapsjs-ui.js");
self.importScripts("http://js.api.here.com/v3/3.0/mapsjs-mapevents.js");
self.addEventListener(
'message',
function(e) {
var count = 0;
var all = 0;
// Initialize the platform object:
var platform = new H.service.Platform({
'app_id': 'myappID',
'app_code': 'myappCODE'
});
var geocoder = platform.getGeocodingService();
var onResult = function(result) {};
var findLocations = function(geocodingParams) {
var i=0;
all = geocodingParams.length;
for (i=0; i<geocodingParams.length; i++) {
geocoder.geocode(
geocodingParams[i],
onResult,
function(e){
alert(e);
} );
count = i;
self.postMessage(count, all);
}
};
findLocations(e.data[0]);
},
false);
I tried different approaches, but executing the worker script fails with different errors. The last problem is ReferenceError: document is not defined in mapsjs-core.js:158:623. and after a while another error:
NetworkError: A network error occurred. from datapointscollection.js:1

For huge number of geocodes, you should consider batch geocoding.
Check developer’s guide at developer.here.com

It looks like the files you are importing into your worker depend on a DOM existing. Your web worker doesn't have a DOM, so you will have to use dependencies that don't need a DOM (if it will work in node, it will work without a DOM). Check the documentation for your dependencies to see if there a version that works in node or doesn't need a DOM, and use that version in your web worker. (It might be just the mapsjs-service.js file. see if you can get away with just that).
See also Web Workers API

Related

forge autodesk viewer api , onDocumentLoadFailure() - errorCode:4

i want to use the autodesk's viewer in my application so i used the forge ph client sdk,i made the 5 steps that are herehttps://forge.autodesk.com/blog/basic-usage-forge-sdk-php everything worked good.
but now, i want to view files in the viewer but it doesn't work i have thies error in my browser's concole :onDocumentLoadFailure() - errorCode:4
function viewObject(access,urn,divId){
var viewer;
var viewerDivId;
var options = {
env: 'AutodeskProduction',
accessToken: access
};
function onDocumentLoadSuccess(doc) {
// A document contains references to 3D and 2D viewables.
var viewables = Autodesk.Viewing.Document.getSubItemsWithProperties(doc.getRootItem(), {'type':'geometry'}, true);
if (viewables.length === 0) {
console.error('Document contains no viewables.');
return;
}
// Choose any of the avialble viewables
var initialViewable = viewables[0];
var svfUrl = doc.getViewablePath(initialViewable);
var modelOptions = {
sharedPropertyDbPath: doc.getPropertyDbPath()
};
var viewerDiv = document.getElementById('viewerDivId');
viewer = new Autodesk.Viewing.Private.GuiViewer3D(viewerDiv);
viewer.start(svfUrl, modelOptions, onLoadModelSuccess, onLoadModelError);
}
function onDocumentLoadFailure(viewerErrorCode) {
console.error('onDocumentLoadFailure() - errorCode:' + viewerErrorCode);
}
function onLoadModelSuccess(model) {
console.log('onLoadModelSuccess()!');
console.log('Validate model loaded: ' + (viewer.model === model));
console.log(model);
}
function onLoadModelError(viewerErrorCode) {
console.error('onLoadModelError() - errorCode:' + viewerErrorCode);
}
var documentId = 'urn:'+urn;
viewerDivId = divId;
Autodesk.Viewing.Initializer(options, function onInitialized(){
Autodesk.Viewing.Document.load(documentId, onDocumentLoadSuccess, onDocumentLoadFailure);
});
}
</script>
<script src="https://ajax.googleapis.com/ajax/libs/jquery/3.2.1 /jquery.min.js"></script>
<script>
function buttonViewClicked() {
var access = $('#token').val();
var urn = $('#urn').val();
viewObject(access, urn, "MonViewer");
}
</script>
Error code 4 refers to situations where the viewer gets 403 Access Denied when trying to download files from Forge. Make sure that the access token you're providing to the viewer is valid and that it has access to the model you're trying to view.
If you're still having issues, feel free to shoot us an email to forge (dot) help (at) autodesk (dot) com, and include as many details about your project, for example, how does the access token look like, the URN of your model, your Forge app ID, etc.

indexedDb objectStore.openCursor so slow

The first time using the index query takes too long.
usage scenarios: mobile use webview.
After the data is saved into indexedDb, the first time you open the page query is extremely slow.
query code:
var startTime = new Date().getTime();
var request = indexedDB.open("yfg");
request.onerror = function(event) {
alert("Why didn't you allow my web app to use IndexedDB?!");
};
request.onsuccess = function(event) {
var table = [];
var db = request.result;
var objectStore = db.transaction("table").objectStore("table");
//objectStore.openCursor().onsuccess = function(event) {
objectStore.openCursor(null,IDBCursor.NEXT).onsuccess = function(event) {
var cursor = event.target.result;
if (cursor) {
table.push(cursor.value);
cursor.continue();
}else {
//alert("No more entries!");
console.log(table);
var endTime = new Date().getTime();
console.log("总耗时:",(endTime-startTime)/1000);
}
};
};
Execute on console:
The first result:
The second result:
Browser version:
The specific content of each piece of data:
This table sheet has these data:
Try using IDBObjectStore.prototype.getAll. getAll generates an array, similar to what you are doing, but involves fewer function calls than openCursor and is therefore more performant.
However, unlike openCursor, getAll may not be supported because it is a newer feature of indexedDB. The linked page shows a compatibility table but that may also be out of date.

Disable the possibility to upload files for user

I'm building a service for users where I must have private files.
Actually, with Cloud Code, I can control the download flux, through a function. But, how I can prevent a hacker to use the javascript console and upload his files ? He will get a link, which he can share with anyone without restriction and at my charges.
const file = Parse.File('hackerFile', hackerFileArray);
file.save().then(() => console.log(file.url)) // Now, he have a free file hosting.
Is there a way to completely remove this feature for everyone, except the master key ?
Example of hosting a file on http://todolist.parseapp.com/
Open the console in your browser then
var script = document.createElement('script');
script.src = '//www.parsecdn.com/js/parse-1.6.14.min.js'; // Because of their version.
document.head.appendChild(script);
Parse.initialize("0Oq3tTp9JMvd72LOrGN25PiEq9XgVHCxo57MQbpT", "vUFy2o7nFx3eeKVlZneYMPI2MBoxT5LhWNoIWPja"); // Found in their sources
var reader = new FileReader();
var input = document.createElement('input');
input.type = 'file';
document.body.appendChild(input);
// Then choose a file from the browser. I choosen a picture.
reader.onloadend = function() {
var file = new Parse.File('hackFile', {base64: reader.result});
file.save().then(function() {
console.log(file.url());
})
};
reader.readAsDataURL(input.files[0]);
Then you have a link. I got http://files.parsetfss.com/ae2ddbce-9cc0-4e1a-a16d-52ec5fdb7570/tfss-8fccfba0-ccf7-41cd-8f42-75f0a3478262-hackFile
Haven't tried this, but think it could work:
1) Add a beforeSave function on whatever class you're looking to prevent this behavior on.
2) In the beforeSave, check request.object.dirtyKeys() and iterate through each of those keys on the newly created object.
3) If the value associated with one of those dirtyKeys is a file, don't allow the file to save: response.error
Parse.Cloud.beforeSave(Parse.User, function(request, response) {
var dirtyKeys = request.object.dirtyKeys();
for (var i = 0; i < dirtyKeys.length; ++i) {
var dirtyKey = dirtyKeys[i];
if (isUnwantedFile(request.object, dirtyKey)) {
response.error("User is not allowed to store files");
return;
}
}
response.success();
});
//note this function is untested -- I'm not sure what type a user-created file would be,
//but basically if you can figure that out, substitute it in here
function isUnwantedFile(obj, key){
return typeof obj[dirtyKey] === Parse.File
}

document generation only works the first time

I'm using openxml in my HTML5 mobile app to generate word documents on the mobile device.
In general openxml works fine and straight forward, but I'm struggling with an annyoing problem.
The document generation only works the first time after I've started the app. This time I can open and view the document. Restart the app means:
- Redeploy from development machine
- Removing the app from the task pane (pushing aside; I assume the app is removed then?)
The second time I get the message the document is corrupted and I'm unable to view the file
UPDATE:
I can't reproduce this behaviour when I'm running the app connected to the remote debugger without having a breakpoint set. Doing it this way I always get a working document.
I doesn't make a difference wether I do any changes on the document or not. Simply open and saving reproduce this error.
After doing some research I've found that structure of the docx.zip file of the working and the corrupt file is the same. They also have the same file length. But in the corrupt docx there are some files I've found some files having a wrong/invalid CRC. See here an example when trying to get a corrupt file out of the zip. Other files are working as expected.
The properties for this file are->
(CRC in a working version is: 44D3906C)
Code for processing the doc-template:
/*
* Process the template
*/
function processTemplate(doc64, callback)
{
"use strict";
console.log("PROCESS TEMPLATE");
var XAttribute = Ltxml.XAttribute;
var XCData = Ltxml.XCData;
var XComment = Ltxml.XComment;
var XContainer = Ltxml.XContainer;
var XDeclaration = Ltxml.XDeclaration;
var XDocument = Ltxml.XDocument;
var XElement = Ltxml.XElement;
var XName = Ltxml.XName;
var XNamespace = Ltxml.XNamespace;
var XNode = Ltxml.XNode;
var XObject = Ltxml.XObject;
var XProcessingInstruction = Ltxml.XProcessingInstruction;
var XText = Ltxml.XText;
var XEntity = Ltxml.XEntity;
var cast = Ltxml.cast;
var castInt = Ltxml.castInt;
var W = openXml.W;
var NN = openXml.NoNamespace;
var wNs = openXml.wNs;
var doc = new openXml.OpenXmlPackage(doc64);
// add a paragraph to the beginning of the document.
var body = doc.mainDocumentPart().getXDocument().root.element(W.body);
var tpl_row = ((doc.mainDocumentPart().getXDocument().descendants(W.tbl)).elementAt(1).descendants(W.tr)).elementAt(2);
var newrow = new XElement(tpl_row);
doc.mainDocumentPart().getXDocument().descendants(W.tbl).elementAt(1).add(newrow);
// callback(doc);
var mod_file = null;
var newfile;
var path;
if (doc != null && doc != undefined ) {
mod_file = doc.saveToBlob();
// Start writing document
path = "Templates";
newfile = "Templates/Bau.docx";
console.log("WRITE TEMPLATE DOCUMENT");
fs.root.getFile("Templates/" + "MyGenerated.docx", {create: true, exclusive: false},
function(fileEntry)
{
fileEntry.createWriter(
function(fileWriter)
{
fileWriter.onwriteend = function(e) {
console.log("TEMPLATE DOCUMENT WRITTEN:"+e.target.length);
};
fileWriter.onerror = function(e) {
console.log("ERROR writing DOCUMENT:" + e.code + ";" + e.message);
};
var blobreader = new FileReader();
blobreader.onloadend = function()
{
fileWriter.write(blobreader.result); // reader.result contains the contents of blob as a typed array
};
blobreader.readAsArrayBuffer(mod_file);
},
null);
}, null);
};
Any ideas what I'm doing wrong?
Thanks for posting about the error. There were some issues with jszip.js that I encountered when I was developing the Open XML SDK for JavaScript.
At the following link, there is a sample javascript app that demonstrates generating a document.
Open XML SDK for JavaScript Demo
In that app you can save multiple DOCXs, one after another, and they are not corrupted.
In order to work on this issue, I need to be able to re-produce locally. Maybe you can take that little working web app and replace parts with your parts until it is generating invalid files?
Cheers, Eric
P.S. I am traveling and have intermittent access to internet. If you can continue the thread on OpenXmlDeveloper.org, then it will help me to answer quicker. :-)
What made it work for me, was changing the way of adding images (Parts) to the document. I was using the type "binary" for adding images to document. I changed this to "base64"
So I changed the source from:
mydoc.addPart( "/word/"+reltarget, openXml.contentTypes.png, "binary", fotodata ); // add Image Part to doc
to:
mydoc.addPart( "/word/"+reltarget, openXml.contentTypes.png, "base64", window.btoa(fotodata) ); // add Image Part to doc

How to change emscripten browser input method from window.prompt to something more sensible?

I have a C++ function which once called consumes input from stdin. Exporting this function to javascript using emscripten causes calls to window.prompt.
Interacting with browser prompt is really tedious task. First of all you can paste only one line at time. Secondly the only way to indicate EOF is by pressing 'cancel'. Last but not least the only way (in case of my function) to make it stop asking user for input by window.prompt is by checking the checkbox preventing more prompts to pop up.
For me the best input method would be reading some blob. I know I can hack library.js but I see some problems:
Reading blob is asynchronous.
To read a blob, first you have to open a file user has to select first.
I don't really know how to prevent my function from reading this blob forever - there is no checkbox like with window.prompt and I'm not sure if spotting EOF will stop it if it didn't in window.prompt case (only checking a checkbox helped).
The best solution would be some kind of callback but I would like to see sime hints from more experienced users.
A way would be to use the Emscripten Filesystem API, for example by calling FS.init in the Module preRun function, passing a custom function as the standard input.
var Module = {
preRun: function() {
function stdin() {
// Return ASCII code of character, or null if no input
}
var stdout = null; // Keep as default
var stderr = null; // Keep as default
FS.init(stdin, stdout, stderr);
}
};
The function is quite low-level: is must deal with one character at a time. To read some data from a blob, you could do something like:
var data = new Int8Array([1,2,3,4,5]);
var blob = new Blob([array], {type: 'application/octet-binary'});
var reader = new FileReader();
var result;
reader.addEventListener("loadend", function() {
result = new Int8Array(reader.result);
});
var i = 0;
var Module = {
preRun: function() {
function stdin() {
if (if < result.byteLength {
var code = result[i];
++i;
return code;
} else {
return null;
}
}
var stdout = null; // Keep as default
var stderr = null; // Keep as default
FS.init(stdin, stdout, stderr);
}
};
Note (as you have hinted), due to the asynchronous nature of the reader, there could be a race condition: the reader must have loaded before you can expect the data at the standard input. You might need to implement some mechanism to avoid this in a real case. Depending on your exact requirements, you could make it so the Emscripten program doesn't actually call main() until you have the data:
var fileRead = false;
var initialised = false;
var result;
var array = new Int8Array([1,2,3,4,5]);
var blob = new Blob([array], {type: 'application/octet-binary'});
var reader = new FileReader();
reader.addEventListener("loadend", function() {
result = new Int8Array(reader.result);
fileRead = true;
runIfCan();
});
reader.readAsArrayBuffer(blob);
var i = 0;
var Module = {
preRun: function() {
function stdin() {
if (i < result.byteLength)
{
var code = result[i];
++i;
return code;
} else{
return null;
}
}
var stdout = null;
var stderr = null;
FS.init(stdin, stdout, stderr);
initialised = true;
runIfCan();
},
noInitialRun: true
};
function runIfCan() {
if (fileRead && initialised) {
// Module.run() doesn't seem to work here
Module.callMain();
}
}
Note: this is a version of my answer at Providing stdin to an emscripten HTML program? , but with focus on the standard input, and adding parts about passing data from a Blob.
From what I understand you could try the following:
Implement selecting a file in Javascript and access it via Javascript Blob interface.
Allocate some memory in Emscripten
var buf = Module._malloc( blob.size );
Write the content of your Blob into the returned memory location from Javascript.
Module.HEAPU8.set( new Uint8Array(blob), buf );
Pass that memory location to a second Emscripten compiled function, which then processes the file content and
Deallocate allocated memory.
Module._free( buf );
Best to read the wiki first.

Categories

Resources