Through a REST API endpoint, I get rather big CSV files with the following structure (JSON inside CSV file):
A,B,C,D
1,2,3,{"E":1,"F":2,"G":3}
1,2,3,{"E":1,"H":2}
For a different tool, I need a CSV with a flat structure (no nested JSON). So, in the end, I'd like to have a CSV that looks like that.
A,B,C,E,F,G,H
1,2,3,1,2,3,
1,2,3,1,,,2
(Although the column headlines look structured, this is not important for my use case)
As the CSV files are rather big, I'm looking for a relatively performant way to do so. I'll be writing this in JavaScript (Node.JS) (as that's the language that's used for all other parts of the script). However, for now I'm just looking for a theoretical way / fake code to do so in a performant matter.
As far as I can tell, I'll probably have to loop over the CSV files twice. The first time I just have to get all JSON keys. The second time, I can then create a new CSV file and set all values. However, how would I properly find out in which column I have to write the values?
Or, is it more performant to "convert" the CSV file to an array of objects in one loop and then use something like the CSV parser (http://csv.adaltas.com/) to convert that back into a CSV?
Here is a solution using jq
If the file filter.jq contains
[
split("\n") # split string into lines
| (.[0] | split(",")) as $headers # split header
| (.[1:][] | split(",")) # split data rows
| select(length>0) # get rid of empty lines
| $headers[:-1] as $h1 # fixed headers
| .[:($h1|length)] as $p1 # fixed part
| .[($h1|length):] as $p2 # variable part
| (
[ [ $h1, $p1 ] # \
| transpose[] # \ assemble fixed object
| {key:.[0], value:.[1]|tonumber} # / from fixed keys and values
] | from_entries # /
) + (
$p2 | join(",") | fromjson # assemble variable object
)
]
| (map(keys) | add | unique) as $all # compute final headers
| [$all] + ( # add headers to
map(. as $b | reduce $all[] as $a ([];. + [$b[$a]])) # objects with all keys
| map(map(if . == null then "" else tostring end)) # convert values to strings
)
| .[] # scan final array
| #csv # convert to csv
and your data is in a file called data then
jq -M -R -s -r -f filter.jq data
will generate
"A","B","C","E","F","G","H"
"1","2","3","1","2","3",""
"1","2","3","1","","","2"
var express = require('express');
var app = express();
var bodyParser = require('body-parser');
var mysql=require('mysql');
var fs= require('fs');
var csv = require('fast-csv');
var formidable = require('formidable');
var urlencodedParser = bodyParser.urlencoded({ extended: false })
var con=mysql.createConnection({
host:'localhost',
user:'dheeraj',
password:'123',
database:'dheeraj'
});
app.use('/assets',express.static('assets'));
app.get('/d', function (req, res) {
res.sendFile( __dirname + "/" + "/d.html" );
})
app.post('/file_upload', urlencodedParser, function (req, res) {
//{
var form = new formidable.IncomingForm();
form.parse(req, function (err, fields, files) {
res.write('File uploaded');
//console.log(files.filetoupload);
fs.createReadStream(files.filetoupload.name)
.pipe(csv())
.on('data',function(data){
var d1=data[0];
var d2=data[1];
var d3=data[2];
var d4=data[3];
var d5=data[4];
con.query('insert into demo values(\''+d1+'\',\''+d2+'\',\''+d3+'\',\''+d4+'\',\''+d5+'\')',function(err,result)
{
console.log('inserted');
})
console.log(data);
})
.on('end',function(data){
console.log('read finished');
});
res.end();
})
})
var server = app.listen(8081, function () {
var host = server.address().address
var port = server.address().port
console.log("Example app listening at http://%s:%s", host, port)
})
Related
My .env has, say, two simple variables:
USERNAME:"myusername"
PASSWORD:"mypassword&7"
The thing is, when I try to use shell.exec to pass a git clone command, it seems to be ignoring the '&7'from my password variable.
shell.exec(`git clone https://${process.env.USERNAME}:${process.env.USERNAME}#github.com/my-repo/xyz-git-ops.git`);
it outputs:
/bin/sh: 7#gmy-repo/xyz-git-ops.git: No such file or directory
Cloning into 'mypassword'...
fatal: unable to access 'https://myusername:mypassword/': URL using bad/illegal format or missing URL
I notice a few weird stuff:
1 - it ignores the last 2 characters of my password value, the '&7'and the git clone output replaces it with a '/'instead.
2 - if I do console.log(process.env.USERNAME), it prints the value perfectly: mypassword&7
All that makes me wonder if is there a way of either escaping the '&' char from the password value or if my approach to pass credential via shell.exec() is absolutely mistaken. Bellow is the full content of my .js file
const nodeCron = require("node-cron");
const shell = require('shelljs');
const rpath = '/Users/myuser/Documents/Git Ops Cron/repos';
require('dotenv').config();
const start = Date.now();
const username = process.env.USERNAME
const password = process.env.PASSWORD
async function xyzGitOps(){
console.log("Running scheduled job", start);
shell.cd(rpath);
shell.exec(`git clone https://${username}:${password}#github.com/my-repo/xyz-git-ops.git`);
return console.log("Job finished");
}
const job = nodeCron.schedule("* * * * *", xyzGitOps);
The username/password component of a URL should be percent encoded.
The node:url URL class will do this for you
const repo = new URL(`https://github.com/my-repo/xyz-git-ops.git`)
repo.username = process.env.USERNAME
repo.password = process.env.PASSWORD
The URL's .toString() encodes the values:
> String(repo)
'https://userw:pass%%2F%40%23$#github.com/my-repo/xyz-git-ops.git'
> `${repo}`
'https://userw:pass%%2F%40%23$#github.com/my-repo/xyz-git-ops.git'
I'm trying to verify a webhook coming from Plaid in NodeJS by calculating the Sha256 of the webhook body and I'm following a Python code here where the code is showing :
# Compute the has of the body.
m = hashlib.sha256()
m.update(body.encode())
body_hash = m.hexdigest()
What's the alternative of body.encode() in Javascript before passing it to the Sha256 function please ? Note that the body I'm getting is an object containing the following data :
{ error: null, item_id: '4zAGyokJ1XiWP63QNl1RuLZV76o55nudVXzNG',
new_transactions: 0, webhook_code: 'DEFAULT_UPDATE', webhook_type:
'TRANSACTIONS' }
However I'm trying to get this hash :
b05ef560b59e8d8e427433c5e0f6a11579b5dfe6534257558b896f858007385a
So, if the body is JSON (NOT JSON STRING) then you need to stringify it and put it in the .update function As the m.body takes a string. If you have your body as STRING then just put it in as is.
This is from the Crypto Example here:
const crypto = require('crypto');
const hash = crypto.createHash('sha256');
const stringBody = JSON.stringify(body);
hash.update(stringBody);
console.log(hash.digest('hex'));
Edit:
If the hash is not same then maybe you need to correct the newlines or whitespaces. You need to make both bodies exactly the same. Here In the below example I am using same exact string and encoding using Python AND NodeJS.
import hashlib
body = '{"error":null,"item_id":"4zAGyokJ1XiWP63QNl1RuLZV76o55nudVXzNG","new_transactions":0,"webhook_code":"DEFAULT_UPDATE","webhook_type":"TRANSACTIONS"}'
m = hashlib.sha256()
m.update(body.encode())
body_hash = m.hexdigest()
print(body_hash)
Output:
python3 file.py
26f1120ccaf99a383b7462b233e18994d0c06d4585e3fe9a91a449e97a1c03ba
And Using NodeJS:
const crypto = require('crypto');
const hash = crypto.createHash('sha256');
const body = {
error: null,
item_id: '4zAGyokJ1XiWP63QNl1RuLZV76o55nudVXzNG',
new_transactions: 0,
webhook_code: 'DEFAULT_UPDATE',
webhook_type: 'TRANSACTIONS'
}
const stringBody = JSON.stringify(body);
hash.update(stringBody);
console.log(hash.digest('hex'));
Output:
node file.js
26f1120ccaf99a383b7462b233e18994d0c06d4585e3fe9a91a449e97a1c03ba
I'm using nodejs mysql-events to parse mysql db updates.
To avoid parsing all logs each time I relaunch my script, I want to use binlogName and nextPosition as read from documentation
filename string Begin reading events from this binlog file. If
specified together with position, will take precedence over
startAtEnd.
position integer Begin reading events from this position. Must be
included with filename.
The below code works like a charm:
const mysql = require('mysql');
const MySQLEvents = require('#rodrigogs/mysql-events');
// Connect to mysql instance
const connection = mysql.createConnection({
host: 'host',
user: 'user',
password: '*****'
});
const mysqlInstance = new MySQLEvents(connection, {
startAtEnd: true
});
await mysqlInstance.start();
Values are valid and copied from vscode debugger.
mysql> show master status;
+------------+----------+--------------+------------------+-------------------+
| File | Position | Binlog_Do_DB | Binlog_Ignore_DB | Executed_Gtid_Set |
+------------+----------+--------------+------------------+-------------------+
| bin.000015 | 34837697 | | | |
+------------+----------+--------------+------------------+-------------------+
1 row in set (0.00 sec)
Once I change the mysqlInstance settings, it ignores it and start from first log.
const mysqlInstance = new MySQLEvents(connection, {
binlogName: 'bin.000015',
nextPosition: 18540004
});
Should I specify something else?
I want to get something like
[06:32:35] [Server thread/INFO]: [0;36;1m | [0;36;22m|__) [0;32;22mLuckPerms [0;36;1mv4.3.73[m
[06:32:35] [Server thread/INFO]: [0;36;1m |___ [0;36;22m| [0;30;1mRunning on Bukkit - CraftBukkit[m
but I get
[06:05:02] [Server thread/INFO]: | |__) LuckPerms v4.3.73
[06:05:02] [Server thread/INFO]: |___ | Running on Bukkit - CraftBukkit
When running minecraft server using child_process
prcs.stdout.on("data", function(d) {
console.log(d.toString());
});
Without knowing exactly how d is shaped, here is something that complies with your example, probably not exactly behaving as you need but you can always try and upgrade it (and at least it doesn't require any dependency):
const versionRegExp = /v[0-9]+(\.[0-9]+)*$/;
d.toString().split("\n").forEach((line) => {
// no idea what the spaces are made of
const exploded = line.trim().split(/[ \t]+/);
// add paddings to the first two structures
const first = exploded.shift().padEnd(5, ' ');
const second = exploded.shift().padEnd(7, ' ');
// work out the content
// condition based on `second`, or should it be remainder.match(versionRegExp) ?
const remainder = 0 === second.indexOf('|__)')
? `[0;30;1m${exploded.join(' ').replace(versionRegExp, '[0;36;1m$&')}[m`
: `[0;32;22m${exploded.join(' ')}[m`
;
// format line and display
console.log(`[0;36;1m${first}[0;36;22m${second}${remainder}`);
});
I am new to encryption and i am trying to get some symmetric encryption usign AES256 going from a mobile app to a webpage through websockets.
I Encrypt the data using RNCryptor default settings
IOS CODE
NSString* message = #"testmessage";
NSData* pubData = [message dataUsingEncoding:NSUTF8StringEncoding];
NSData *encryptedData = [RNEncryptor encryptData:pubData
withSettings:kRNCryptorAES256Settings
password:#"test"
error:&error];
if(error) {
NSLog(#"Error encrypting %#", [error localizedDescription]);
}
NSString* encryptedString = [encryptedData base64Encoding];
NSLog(#"Sending message %#", encryptedString);
[self.session publishData:[encryptedString dataUsingEncoding:NSUTF8StringEncoding] onTopic:#"test12345"];
Here is the output going out over websockets
AgEBnXPPvAkJb7YVapwCVNd5SQw4JwqU7BfLsEXNZyKy9SazfJT8w16Y/hYY7aKxuz3Kuy2tAXXX/cHCc3PMhvG+fzSfrslRVMKvD6L+oWvXLg==
JAVASCRIPT CODE - I receive the message and i try to parse it and display it
function onMessageArrived(message) {
var rawData = base64.decode(message.payloadString);
var encryptionSalt = rawData.substr(2,8);
var hmacSalt = rawData.substr(10,8);
var iv = rawData.substr(18, 16);
var ciphertext = rawData.substr(34, rawData.length-34-32);
var hmac = rawData.substr(rawData.length-32, 32);
var password = "test";
var key = CryptoJS.PBKDF2(password, encryptionSalt, { keySize: 256 / 32, iterations: 10000});
var plaintextArray = CryptoJS.AES.decrypt(
{ ciphertext: CryptoJS.enc.Utf8.parse(ciphertext) },
CryptoJS.enc.Hex.parse(key),
{ iv: CryptoJS.enc.Latin1.parse(iv) }
);
showScreen('<span style="color: blue;">User: ' + CryptoJS.enc.Latin1.stringify(plaintextArray) + '</span>');
};
For some reason the code gets stuck on generating the key (maybe 10k iterations are too much for CryptoJS??? thats the iterations used on IOS )
I have tried a bunch of different things with this and the output is garbage im not actually getting the message decrypted. Any help would be greatly appreciated. If you are going to recommend SJCL please provide some code . RNCryptor uses its own message format. I use it because of the randomized iv it provides . Please recommend a different library if you know of any as well.
THanks for reading.
Please follow the documentation for the RNCryptor file format: https://github.com/RNCryptor/RNCryptor-Spec/blob/master/RNCryptor-Spec-v3.md
You should be able to derive all the necessary data from file and add your shared secret...
Specification for RNCryptor data format version 3
Byte: | 0 | 1 | 2-9 | 10-17 | 18-33 | <- ... -> | n-32 - n |
Contents: | version | options | encryptionSalt | HMACSalt | IV | ... ciphertext ... | HMAC |
version (1 byte): Data format version. Currently 3.
options (1 byte):
bit 0 - uses password
encryptionSalt (8 bytes): iff option includes
"uses password"
HMACSalt (8 bytes): iff options includes "uses
password"
IV (16 bytes) ciphertext (variable) -- Encrypted in CBC
mode HMAC (32 bytes)
All data is in network order (big-endian).