How to get a string from a BS4 scrape - javascript

recently I scraped a text/javascript that contains the following code:
var spConfigDisabledProducts = [-1
, '294653', '294655', '294656', '294657', '294658', '294659', '294660', '294661', '294662', '294663', '294664', '294666', '294667', '294668', '294669', '294670', '294671', '294672', '294673' ];
{"attributes":{"959":{"id":"959","code":"aw_taglia","label":"Taglia","options":[{"id":"1717","label":"15","price":"0","oldPrice":"0"...
I just want to get all the numbers inside var spConfigDisabledProducts excluded -1, so I tried this:
js = soup.find_all('script')[25].text.replace(',}', '}').replace(',]', ']').strip()
js = json.dumps(js)
obj = json.loads(js)
data_oos = obj.split('var spConfigDisabledProducts = [-1,')
data_oos = data_oos[1].split("];")
But it returns the entire javascript, not only var spConfigDisabledProducts.
How can I fix this?
Thanks in advance

You could regex out string representation of list and convert to actual list then slice
import re, json, ast
s = '''var spConfigDisabledProducts = [-1
, '294653', '294655', '294656', '294657', '294658', '294659', '294660', '294661', '294662', '294663', '294664', '294666', '294667', '294668', '294669', '294670', '294671', '294672', '294673' ];
{"attributes":{"959":{"id":"959","code":"aw_taglia","label":"Taglia","options":[{"id":"1717","label":"15","price":"0","oldPrice":"0"'''
p = re.compile(r'spConfigDisabledProducts = (\[[\s\S]*?\])')
data = ast.literal_eval(p.findall(re.sub('\n|\s{2,}','',s))[0])
print(data[1:])

Related

javascript - convert string to json array

I was using s3 select to fetch selective data and display them on my front end .
I converted array of byte to buffer and then to string like below as string
let dataString = Buffer.concat(records).toString('utf8');
the result i got was string like below
{"id":"1","obj1":"191.25","obj2":"11.81","obj3":"3.44","obj4":"15.62"}
{"id":"2","obj1":"642.00","obj2":"4.33","obj3":"0.00","obj4":"11.33"}
{"id":"3","obj1":"153.76","obj2":"94.77","obj3":"16.80","obj4":"29.79"}
{"id":"4","obj1":"61.71","obj2":"0.43","obj3":"0.00","obj4":"8.14"}
{"id":"5","obj1":"194.33","obj2":"108.89","obj3":"14.13","obj4":"168.60"}
{"id":"6","obj1":"204.31","obj2":"137.41","obj3":"34.76","obj4":"193.16"}
{"id":"7","obj1":"199.53","obj2":"34.53","obj3":"16.29","obj4":"26.56"}
{"id":"8","obj1":"77.33","obj2":"5.00","obj3":"12.50","obj4":"0.00"}
{"id":"9","obj1":"128.54","obj2":"101.60","obj3":"15.76","obj4":"46.23"}
{"id":"10","obj1":"107.00","obj2":"116.67","obj3":"34.42","obj4":"8.75"}
{"id":"12","obj1":"206.05","obj2":"155.03","obj3":"36.96","obj4":"148.99"}
{"id":"13","obj1":"133.93","obj2":"142.79","obj3":"39.91","obj4":"98.30"}
Now i want to convert them to json array , i got a solution like below
let dataArray = dataString.split('\n');
//remove white spaces and commas etc
dataArray = dataArray.filter(d=> d.length >2);
//change string to json
dataArray = dataArray.map(d=> JSON.parse(d));
Now the problem is that i have splitted them with new line and wont work if the json is compressed or data itself can have new line.
What is the best way to handle this situation. i want the output like below
[{"id":"1","obj1":"191.25","obj2":"11.81","obj3":"3.44","obj4":"15.62"},
{"id":"2","obj1":"642.00","obj2":"4.33","obj3":"0.00","obj4":"11.33"},
{"id":"3","obj1":"153.76","obj2":"94.77","obj3":"16.80","obj4":"29.79"},
{"id":"4","obj1":"61.71","obj2":"0.43","obj3":"0.00","obj4":"8.14"},
{"id":"5","obj1":"194.33","obj2":"108.89","obj3":"14.13","obj4":"168.60"},
{"id":"6","obj1":"204.31","obj2":"137.41","obj3":"34.76","obj4":"193.16"},
{"id":"7","obj1":"199.53","obj2":"34.53","obj3":"16.29","obj4":"26.56"},
{"id":"8","obj1":"77.33","obj2":"5.00","obj3":"12.50","obj4":"0.00"},
{"id":"9","obj1":"128.54","obj2":"101.60","obj3":"15.76","obj4":"46.23"},
{"id":"10","obj1":"107.00","obj2":"116.67","obj3":"34.42","obj4":"8.75"},
{"id":"12","obj1":"206.05","obj2":"155.03","obj3":"36.96","obj4":"148.99"},
{"id":"13","obj1":"133.93","obj2":"142.79","obj3":"39.91","obj4":"98.30"}
]
#sumit
please take a look at this solution.
let dataString=`{"id":"1","obj1":"191.25","obj2":"11.81","obj3":"3.44","obj4":"15.62"}
{"id":"2","obj1":"642.00","obj2":"4.33","obj3":"0.00","obj4":"11.33"}
{"id":"3","obj1":"153.76","obj2":"94.77","obj3":"16.80","obj4":"29.79"}
{"id":"4","obj1":"61.71","obj2":"0.43","obj3":"0.00","obj4":"8.14"}
{"id":"5","obj1":"194.33","obj2":"108.89","obj3":"14.13","obj4":"168.60"}
{"id":"6","obj1":"204.31","obj2":"137.41","obj3":"34.76","obj4":"193.16"}
{"id":"7","obj1":"199.53","obj2":"34.53","obj3":"16.29","obj4":"26.56"}
{"id":"8","obj1":"77.33","obj2":"5.00","obj3":"12.50","obj4":"0.00"}
{"id":"9","obj1":"128.54","obj2":"101.60","obj3":"15.76","obj4":"46.23"}
{"id":"10","obj1":"107.00","obj2":"116.67","obj3":"34.42","obj4":"8.75"}
{"id":"12","obj1":"206.05","obj2":"155.03","obj3":"36.96","obj4":"148.99"}
{"id":"13","obj1":"133.93","obj2":"142.79","obj3":"39.91","obj4":"98.30"}`;
let dataArray = dataString.match(/{(?:[^{}]*|(R))*}/g);
dataArray = dataArray.map(d=> JSON.parse(d));
console.log(dataArray);
Yeah, it is not a very good idea to concatenate objects into a string like that. If you don't have any other choice, however, something like that should do the trick:
const initialString = `{"id":"1","obj1":"191.25","obj2":"11.81","obj3":"3.44","obj4":"15.62"}
{"id":"2","obj1":"642.00","obj2":"4.33","obj3":"0.00","obj4":"11.33"}
{"id":"3","obj1":"153.76","obj2":"94.77","obj3":"16.80","obj4":"29.79"}
{"id":"4","obj1":"61.71","obj2":"0.43","obj3":"0.00","obj4":"8.14"}
{"id":"5","obj1":"194.33","obj2":"108.89","obj3":"14.13","obj4":"168.60"}
{"id":"6","obj1":"204.31","obj2":"137.41","obj3":"34.76","obj4":"193.16"}
{"id":"7","obj1":"199.53","obj2":"34.53","obj3":"16.29","obj4":"26.56"}
{"id":"8","obj1":"77.33","obj2":"5.00","obj3":"12.50","obj4":"0.00"}
{"id":"9","obj1":"128.54","obj2":"101.60","obj3":"15.76","obj4":"46.23"}
{"id":"10","obj1":"107.00","obj2":"116.67","obj3":"34.42","obj4":"8.75"}
{"id":"12","obj1":"206.05","obj2":"155.03","obj3":"36.96","obj4":"148.99"}
{"id":"13","obj1":"133.93","obj2":"142.79","obj3":"39.91","obj4":"98.30"}`;
const json = `[${initialString.replace(/}\s*{/g, '},{')}]`;
const array = JSON.parse(json);

POSTing a array from android to Google apps script

I am trying to send an array to google script to put into google sheets.
What I have for the google script:
function insert(e, sheet) {
//var scannedData = e.parameter.sOrder;
var scannedData = JSON.parse(e.parameter.sOrder);
var orderLocation = e.parameter.sLocation;
var d = new Date();
var ctime = d.toLocaleString();
sheet.appendRow([scannedData, orderLocation, ctime]);
return ContentService
.createTextOutput("Success")
.setMimeType(ContentService.MimeType.JAVASCRIPT);
}
the results it gives me is:
[Ljava.lang.Object;#5c0b25d1 Shipping 25/07/2020, 22:32:21
what it should give me is:
0152502243 Shipping 24/07/2020, 18:20:37
my code on my apps side:
postDataArray = new JSONArray(Arrays.asList(finalData));
postDataParams.put("sOrder", postDataArray);
postDataParams.put("sLocation",orderLocation);
postDataParams.put("sheetName",sheetName);
Log.e("params",postDataParams.toString());
finalData is a String[] that consists of 2 entries.
"Location"
"Data"
if i send finalData[0] as a control then it picks up the first entry, but it gives me this error instead:
[Ljava.lang.Object;#5c0b25d1
The google script needs to take either an array straight or convert a string into an array, and I am stuck on this conversion.
so google script must take the array
finalData = {"Location","Data"}
and convert it into:
[Location]
[Data]
When sending and receiving structured data, it is preferable to send and receive as json.
Sheet#appendRow accepts a single argument of type array. This array should not be a nested array. Try
sheet.appendRow(scannedData);
or
sheet.appendRow([...scannedData, orderLocation, ctime]);
or
sheet.appendRow(scannedData.concat([orderLocation, ctime]);
Assuming a doPost(e)
doPost(e) {
...
var scannedData = e.parameter.sOrder;
var arr ="{"+ scannedData+"}";
var orderLocation = e.parameter.sLocation;
var d = new Date();
var ctime = d.toLocaleString();
var ss=SpreadsheetApp.openById('ssid')
var sheet=ss.getActiveSheet();
sheet.appendRow([arr,ctime]);

Double ToString keeping "en-US" format

Sounds simples - And I know it is... but i'm having issues with it and don't know why exactly..
I have a web application globalized (multilanguages).
When I click to change the language, this is my action:
public ActionResult ChangeCulture(string lang)
{
Thread.CurrentThread.CurrentCulture = new CultureInfo(lang);
Thread.CurrentThread.CurrentUICulture = Thread.CurrentThread.CurrentCulture;
var languageCookie = new HttpCookie("_language") { Value = lang, Expires = DateTime.UtcNow.AddYears(1) };
Response.Cookies.Remove("_language");
Response.SetCookie(languageCookie);
return Redirect(Request.UrlReferrer.PathAndQuery);
}
And I have a page to display some char (I'm using chart.js) and I need to bind a List<double> to a javascript array.
So this list looks like:
var list = new List<double> {144, 0, 540.23};
And I need a simples array in javascript:
var arr = [144, 0, 540.23];
Here is how I'm doing (razor):
var arr = [#string.Join(",", Model.ListWithDoubles.Select(x => Convert.ToString(x, new CultureInfo("en-US"))))]
The problem is:
When I'm using english language its works pretty. The others langues gives me integer numbers instead...
var arr = [144, 0, 540.23]; //en-US
var arr = [144, 0, 54023]; //pt-BR
var arr = [144, 0, 54023]; //it
var arr = [144, 0, 54023]; //es
Questions
Why?
How to fix?
Because in some other non en-US cultures the , and . have the exact opposite meaning and usage. If you are not displaying this data, only for chart purposes, then use CultureInfo.InvariantCulture when converting the double to string representation for the HTML. You should only convert to a culture specific string at the point you want to actually visually display that data value to the user.
var arr = [#string.Join(",", Model.ListWithDoubles.Select(x => Convert.ToString(x, CultureInfo.InvariantCulture)))]
The default format specifier for double is G so it will create output with only a decimal separator. As you want the raw number (not formatted for display) then this (CultureInfo.InvariantCulture) is what you need to pass, not the culture formatted string representation as that is for display only.
To illustrate that the code I posted works regardless of the culture of the current thread here is that code. Drop this into a console application and replace the Main method and run it. You will see that this works. Your issue lies elsewhere, not with this code.
static void Main(string[] args)
{
var cultures = new[] {"en-US", "pt-BR", "it", "es"};
var list = new List<double> {144, 0, 540.23};
Console.WriteLine("Without specifying a culture");
foreach (var culture in cultures.Select(isoCulture => new CultureInfo(isoCulture)))
{
System.Threading.Thread.CurrentThread.CurrentCulture = culture;
System.Threading.Thread.CurrentThread.CurrentUICulture = culture;
Console.WriteLine("Culture: " + culture.Name);
Console.WriteLine("Not defined: " + string.Join(",", list.Select(x => Convert.ToString(x))));
Console.WriteLine("CultureInfo.InvariantCulture: " + string.Join(",", list.Select(x => Convert.ToString(x, CultureInfo.InvariantCulture))));
}
Console.ReadLine(); // stops so you can see the results
}

Use python to add to javascript array in javascript file

I have this array stored in versions.js
var cc_versions = [
"2016-01-22",
"2016-01-21",
];
var cs_versions = [
"2016-01-23",
"2016-01-21",
];
I have been trying to figure out a way to write new data to the top of the array inside the javascript file with python. I run a python script on my computer almost every day and I want to update this array when I run it so that I can have a website load the versions. Is it possible to do this? I know I could write to the file, but it would just go to the bottom of the file outside of the array. Also there will be multiple arrays, so I'm trying to find some python script that can interact with a javascript file so I can target specific arrays.
You could also try using Genshi template language. One of the tags there is
<?python ... ?>
for example:
<?python
import json
dateInit = []
selectInit = []
for fi in form_items:
if (fi["type"] == "date"):
dateInit.append(fi["id"])
if "attrs" in fi and "format-select" in fi["attrs"]:
selectInit.append(fi["id"])
if not "attrs" in fi:
fi["attrs"] = {}
jsonDateInit = json.dumps(dateInit)
jsonSelectInit = json.dumps(selectInit)
?>
Your response should contain form_items dictionary processed somewhere on the back-end. Then in javascript you can 'accept' the variables using '$'-sign:
var dateitems = JSON.stringify($jsonDateInit);
var select_items = JSON.stringify($jsonSelectInit);
import json
NEW_VARIABLES = [1, 2, 3]
with open('your/json/file.json') as data_file:
json_data = json.load(data_file)
# insert at the beginning of the list
json_data['cc_version'].insert(0, 'something new')
json_data['cs_version'] = NEW_VARIABLES + json_data['cs_version']
# write to json file after modifying data
with open('your/json/file.json', 'w') as output_file:
json.dump(json_data, output_file)
I ended up using json as suggested in the comments
import json
with open("cc_versions.txt") as data:
json = json.load(data)
dates = json["dates"]
dates.append("2016-01-21")
dates = set(dates)
dates = sorted(dates, key=lambda d: map(int, d.split('-')))
dates = dates[::-1]
import json
with open("cc_versions.txt", "w") as outfile:
json.dump({'dates':dates}, outfile, indent=4)

JS/Jquery - using variable in json selector

I need to use a variable when selecting data from a json source like this.
The json is retrieved with jquery getJSON().
"prices":[{
"fanta":10,
"sprite":20,
}]
var beverage = fanta;
var beverage_price = data.prices.beverage;
Now beverage_price = 10
var beverage = sprite;
var beverage_price = data.prices.beverage;
Now beverage_price = 20
When I try to do it like in the examples, the script tries to look up the beverage entry in prices.
Thanks a lot!!
You can access it like:
var beverage = 'fanta';
var beverage_price = data.prices[0][beverage];
As VisioN mentioned in the comment, data.prices is an array, you need to access its first element with [0] which contains prices { "fanta":10, "sprite":20}
here is the working example : http://jsfiddle.net/2E8AH/
Or else you can make data.prices an object like below : (if it is in your control)
var data = {
"prices" :
{
"fanta":10,
"sprite":20,
}
};
and can access without [0] like this : http://jsfiddle.net/Y8KtT/1/

Categories

Resources