Related
I am using GoogleEarthEngine to look at precipitation data for the Mekong River basin. The GLDAS data are set every three hours per day. My goal is to extract the GLDAS data from 2000 to 2020 by adding up the data by month.
I want to extract the total monthly precipitation in GLDAS using the Google Earth Engine, but I cannot extract the CSV due to the error
Image.date: Image '120' has a 'system:time_start' property which is not a number: 2010-01-01T00:00:00
I think I can extract the CSV by converting "system:time_start", how should I change it?
var studyArea = ee.Geometry.Rectangle(102, 8.5, 107, 15);
Map.centerObject(mekong, 9);
// 解析対象年
var years = ee.List.sequence(2000, 2019);
var months = ee.List.sequence(1, 12);
var early = ('2010-01-01');
var late = ('2011-01-01');
// MOD11A1の取り出し
var image = ee.ImageCollection('NASA/GLDAS/V021/NOAH/G025/T3H')
.filterDate(early, late).filterBounds(mekong);
//print(image);
// mmに変換してmodLSTcに保存
var gldas_precipitation = image.select('Rainf_f_tavg');
var gldas_precipitation_mm = gldas_precipitation.map(function(img)
{return img.multiply(10080.0).copyProperties(img, ['system:time_start'])});
// 変数gldas_precipitation_mm_monthの中に月単位のメジアンデータを保存 ////////////////////////////////////////////
var gldas_precipitation_mm_month = ee.ImageCollection.fromImages(
years.map(function(y) {
return months.map(function(m) {
var monthly = gldas_precipitation_mm.filter(ee.Filter.calendarRange(y, y, 'year'))
.filter(ee.Filter.calendarRange(m, m, 'month'))
.sum()
.rename('precipitation_mm_month');
return monthly.set('year', y).set('system:time_start', ee.Date.fromYMD(y, 1, 1))
.set('month', y).set('system:time_start', ee.Date.fromYMD(y, m, 1));
});
}).flatten());
var gldas_precipitation_mm_month = gldas_precipitation_mm_month.filterBounds(mekong);
// TSLのポリゴン ///////////////////////////////////////////////////////////////////////////////////
var empty = ee.Image().byte();
// Paint all the polygon edges with the same number and width, display
var outline = empty.paint({
featureCollection: mekong,
color: 1,
width: 2
});
Map.addLayer(outline, {palette: 'FF0000'}, 'TSL');
//output_csv_precipitation
//Create variables and extract data
var scale = gldas_precipitation_mm_month.mean().projection().nominalScale().multiply(0.05); print(scale);
var gldas = gldas_precipitation_mm_month.filter(ee.Filter.listContains('system:band_names', gldas_precipitation_mm.mean().bandNames().get(0)));
var ft = ee.FeatureCollection(ee.List([]));
//Function to extract values from image collection based on point file and export as a table
var fill = function(img, ini) {
var inift = ee.FeatureCollection(ini);
var ft2 = img.reduceRegions(mekong, ee.Reducer.mean(), scale);
var date = img.date().format("YYYY/MM/dd");
var ft3 = ft2.map(function(f){return f.set('month', date)});
return inift.merge(ft3);
};
// Iterates over the ImageCollection
var profile = ee.FeatureCollection(gldas_precipitation_mm_month.iterate(fill, ft));
print(profile,'profile');
The value of the property system:time_start must be a number (even though it would make sense for it to be a Date, the system design didn't end up that way). You must change calls like
.set('system:time_start', ee.Date.fromYMD(y, m, 1))
to
.set('system:time_start', ee.Date.fromYMD(y, m, 1).millis())
While looking, I see other possible problems here:
return monthly.set('year', y).set('system:time_start', ee.Date.fromYMD(y, 1, 1))
.set('month', y).set('system:time_start', ee.Date.fromYMD(y, m, 1));
This is setting the month property value to the y variable (not m), and it's setting system:time_start twice (so only the second value will be used). Probably this is not what you meant. I have not looked at what you're intending to do with the collection, so you'll have to figure that part out yourself.
I have a websocket where i receive price of stock randomly like 300ms or sometimes 1 second.I want to calculate how many price I received in past 30 seconds only.
var arr = [];
function onReceive(price) {
var timestamp = Number(new Date());
arr[timestamp] = [];
arr[timestamp].push(price);
if (arrarr[timestamp].length > 1000) {
arr.shift();
}
}
Now I just want to count how many price is received in last 30 seconds , I cannot come up with any logic.
I tried something like slicing last 30 items in array and calculating difference between last time stamp and -30 timestamp , which tells me how much time it took to receive 30 price ticks ,but i dont know how to calculate how to find how many ticks received in past 30 seconds , any ideas please.thank you.
arr[timestamp][arr[timestamp].length-1].key-arr[timestamp][0].key;
Personally I would create some sort of named instance for a log item, holding the UNIX timestamp and the price.
To retrieve anything in the last X seconds, you'd get the current UNIX timestamp, subtract X * 1000 from it, and use .filter() do a reverse iteration to retrieve all items where the timestamp is greater than that.
EDIT: As Robby pointed out, there's no need to search through the entire array as the timestamps are guaranteed to be in increasing order. By iterating in reverse, we can exit the loop when we find the first result outside of the desired window.
var priceLog = [];
function PriceLogItem(price) {
this.price = price;
this.timestamp = new Date().getTime();
}
function onReceive(price) {
priceLog.push(new PriceLogItem(price));
if (priceLog.length > 1000) log.shift();
}
function getPriceLogsSince(secondsAgo) {
let millisecondsAgo = secondsAgo * 1000;
let time = new Date().getTime() - millisecondsAgo;
let result = [];
for (let i = priceLog.length - 1; i >= 0; i--) {
if (priceLog[i].timestamp >= time) result.push(priceLog[i]);
else break;
}
return result;
}
//Usage
let priceLogs = getPriceLogsSince(30); //Get all logs within past 30 seconds
console.log(priceLogs);
I'm developing in Google Chrome browser using node.js and p5.js. I want to read 5 .csv files and place the values into arrays, which has worked successfully for 2 out of the 5 files. To do this I have used the loadtable function to place the data into p5 table objects to then loop through to retrieve the values and place them into arrays. To loop through, my for loop is using the getrowcount function as a bound which returns the correct number of rows for the two working instances, however for the 3 that don't work, the row count returned is zero, even though when I look at the table objects I've loaded the csv data into, they are clearly populated with the records. I can see that the "columns" and "rows" value when expanding the table objects do read 0, however when traversing down another layer, the "columns" and "rows" values are correctly stored and I'm assuming the function is returning the first incorrect case of rows and columns. Please find attached links to screenshots as I'm sure my explanation is very confusing. The method I've used is very simple and almost identical each time so I'm unsure where I could have made a mistake, any ideas would be great.
console log screenshot https://imgur.com/HmcZmw9
code screenshot https://imgur.com/Ouv6OLX
code screenshot https://imgur.com/cxEkWVc
code screenshot https://imgur.com/3sPGXgb
code screenshot https://imgur.com/LlRdEhw
The first line in the console log shows a successful occurrence where the data table is loaded in fine, the second line shows the array being populated correctly and the third line prints the correct row count.
The second data table shows an incorrect time where the table is populated with values but the rows and columns count is incorrect, but when expanding the object once, you can see the correct rows and columns count is there. Below shows the empty array and row count of 0.
This is my first post so apologies if I've asked in the wrong way. Happy to give more information. Thanks.
The following is the code used. Most of the operations take place in function setup():
// This file retrieves and formats the data from smartgrid readings
// Create p5 Table objects
let wind_energy_table = new p5.Table();
let solar_energy_table = new p5.Table();
let weather_10_table = new p5.Table();
let weather_60_table = new p5.Table();
let weather_forecast_table = new p5.Table();
// Create arrays to handle and access values easier
const wind_energy = [];
const solar_energy = [];
const weather_10 = [];
const weather_60 = [];
const weather_forecast = [];
// Populate tables with values, preload runs before setup
function preload() {
wind_energy_table = loadTable("/data/wind_energy.csv", "csv");
solar_energy_table = loadTable("/data/solar_energy.csv", "csv");
weather_10_table = loadTable("/data/weather_data_10.csv", "csv");
weather_60_table = loadTable("/data/weather_data_60.csv", "csv");
weather_forecast_table = loadTable("/data/weather_forecast.csv", "csv");
}
function setup() {
let we_date, we_time, we_p1, we_p2, we_p3, we_pt;
// pi indicates power of ith wind farm (1,2,3), pt = power exchange (Megawatts)
let se_date, se_time, se_ps;
// ps = power in watts of subgrid
let w10_date, w10_time, w10_airtemp, w10_wd, w10_ws, w10_sd, w10_ap, w10_precip;
// wd = wind direction (in decadegrees Celsius), ws = wind speed (in 0.1 m/s), sd = sunshine duration in last 10 minutes,
// air pressure (in 0.1 hPa), precip = amount of precipitation in last 10 minutes
// decadegrees = 10 degrees
// hPa = hectopascal i.e. 100 pascal
let w60_date, w60_time, w60_cc, w60_wd, w60_ws, w60_at, w60_ap, w60_sd, w60_precip;
// cc = cloud coverage (in 1/8), wd = wind direction (in decadegrees Celsius), ws = wind speed (in 0.1 m/s),
// at = air temperature (in 0.1 degrees), ap = air pressure (in 0.1 hPa), sd = sunshine duration in last hour,
// precip = precipitation in last hour
// cloud coverage 8/8 = whole sky covered
let wf_date, wf_time, wf_validDate, wf_temp, wf_dp, wf_ws, wf_gs, wf_ap, wf_precipProb, wf_cc, wf_si, wf_wd, wf_ah, wf_ad
// validDate = date the forecast is valid for, dp = dew point (in Celsius), ws = wind speed (in m/s), gs = gust speed (in m/s),
// ap = air pressure (in hPa), precipProb = probability of precipitation (in %), cc = cloud coverage, si = solar irradiance (in kJ/m^2),
// wd = wind direction (in degrees), ah = air humidity (in %), ad = air density (in J/(kg*K))
// temp (in degrees Celsius), J = Joule, K = Kelvin
// Populate arrays with arrays for each values by looping through the tables in csv files
for (let i = 0; i < wind_energy_table.getRowCount(); i++) {
we_date = wind_energy_table.getRow(i).arr[0];
we_time = wind_energy_table.getRow(i).arr[1];
we_p1 = wind_energy_table.getRow(i).arr[2];
we_p2 = wind_energy_table.getRow(i).arr[3];
we_p3 = wind_energy_table.getRow(i).arr[4];
we_pt = wind_energy_table.getRow(i).arr[5];
wind_energy[i] = [we_date, we_time, we_p1, we_p2, we_p3, we_pt];
}
console.log(wind_energy_table);
console.log(wind_energy);
console.log(wind_energy_table.getRowCount());
for (let j = 0; j < solar_energy_table.getRowCount(); j++) {
se_date = solar_energy_table.getRow(j).arr[0];
se_time = solar_energy_table.getRow(j).arr[1];
se_ps = solar_energy_table.getRow(j).arr[2];
solar_energy[j] = [se_date, se_time, se_ps];
}
console.log(solar_energy_table);
console.log(solar_energy);
console.log(solar_energy_table.getRowCount());
console.log(solar_energy_table.getRow(0));
for (let k = 0; k < weather_10_table.getRowCount(); k++) {
w10_date = weather_10_table.getRow(k).arr[0];
w10_time = weather_10_table.getRow(k).arr[1];
w10_airtemp = weather_10_table.getRow(k).arr[2];
w10_wd = weather_10_table.getRow(k).arr[3];
w10_ws = weather_10_table.getRow(k).arr[4];
w10_sd = weather_10_table.getRow(k).arr[5];
w10_ap = weather_10_table.getRow(k).arr[6];
w10_precip = weather_10_table.getRow(k).arr[7];
weather_10[k] = [w10_date, w10_time, w10_airtemp, w10_wd, w10_ws, w10_sd, w10_ap, w10_precip];
}
console.log(weather_10_table);
console.log(weather_10);
for (let l = 0; l < weather_60_table.getRowCount(); l++) {
w60_date = weather_60_table.getRow(l).arr[0];
w60_time = weather_60_table.getRow(l).arr[1];
w60_cc = weather_60_table.getRow(l).arr[2];
w60_wd = weather_60_table.getRow(l).arr[3];
w60_ws = weather_60_table.getRow(l).arr[4];
w60_at = weather_60_table.getRow(l).arr[5];
w60_ap = weather_60_table.getRow(l).arr[6];
w60_sd = weather_60_table.getRow(l).arr[7];
w60_precip = weather_60_table.getRow(l).arr[8];
weather_60[l] = [w60_date, w60_time, w60_cc, w60_wd, w60_ws, w60_at,
w60_ap, w60_sd, w60_precip
];
}
console.log(weather_60_table);
console.log(weather_60);
for (let m = 0; m < weather_forecast_table.getRowCount(); m++) {
wf_date = weather_forecast_table.getRow(m).arr[0];
wf_time = weather_forecast_table.getRow(m).arr[1];
wf_validDate = weather_forecast_table.getRow(m).arr[2];
wf_temp = weather_forecast_table.getRow(m).arr[3];
wf_dp = weather_forecast_table.getRow(m).arr[4];
wf_ws = weather_forecast_table.getRow(m).arr[5];
wf_gs = weather_forecast_table.getRow(m).arr[6];
wf_ap = weather_forecast_table.getRow(m).arr[7];
wf_precipProb = weather_forecast_table.getRow(m).arr[8];
wf_cc = weather_forecast_table.getRow(m).arr[9];
wf_si = weather_forecast_table.getRow(m).arr[10];
wf_wd = weather_forecast_table.getRow(m).arr[11];
wf_ah = weather_forecast_table.getRow(m).arr[12];
wf_ad = weather_forecast_table.getRow(m).arr[13];
weather_forecast[m] = [wf_date, wf_time, wf_validDate, wf_temp, wf_dp, wf_ws,
wf_gs, wf_ap, wf_precipProb, wf_cc, wf_si, wf_wd, wf_ah, wf_ad
];
}
console.log(weather_forecast_table);
console.log(weather_forecast);
}
I am relatively new to JavaScript and Django and I am struggling with passing big data to my google chart.
I have a chart representing velocities for a given date and distance. In my django views I create list of distances, dates and according velocities. I also generate there a list with sorted values occuring in velocity list and a list with colors according to velocity's value.
I want to have a chart with velocity map with applied colortable like this :
http://i.imgur.com/9Tyv8Rn.jpg
So I used scatter chart with velocity series. The chart is dynamic, it's diffrent for every item selected by a user.
JS to generate rows and columns :
// Define data table rows:
var rows = [];
var rows_list = [];
var vl_max = vel_list.length;
for (i=0; i < vl_max; i+=1) {
var date_tmp = new Date(date_list[i].split(',')[0],date_list[i].split(',')[1]-1,date_list[i].split(',')[2]);
var date = [date_tmp];
var vel_tmp = vel_list[i];
var vtemp_max = vel_tmp.length;
var tooltip_dsname = dsname_list[i];
var tooltip_track = track_list[i];
for (j=0; j < vtemp_max; j+=1) {
var cell = [{v : date_tmp}];
for (k=0; k < vr_max; k+=1) {
var vel_full = vel_tmp[j];
var vel = vel_full.toFixed(1);
if (vel == vel_range[k]) {
// tooltip:
var dist = dist_list[j]/1000;
var yyyy = date_tmp.getFullYear().toString();
var mm = (date_tmp.getMonth()+1).toString(); // getMonth() is zero-based
var dd = date_tmp.getDate().toString();
var tooltip_date = yyyy + "-" + (mm[1]?mm:"0"+mm[0]) + "-" + (dd[1]?dd:"0"+dd[0]);
var tooltip = "<b>dataset: </b>"+tooltip_dsname+"<br><b>date: </b>"+tooltip_date+"<br><b>track: </b>"+tooltip_track+"<br><b>distance: </b>"+dist+" k"+mapunit+"<br><b> velocity: </b>"+vel_full.toFixed(2)+" m/d";
var color = color_list[k]
var style = "point { shape-type: square; fill-color: "+color+";}"
} else {
var dist = NaN;
var tooltip = "empty" ;
var style = "empty" ;
}
cell.push({v: dist},{v: tooltip},{v:style});
}
rows_list.push({c: cell});
}
};
Here is JSfiddle for chart generation with smaller data :
http://jsfiddle.net/joannao89/t26ooyrt/2/
The problem that I have is while the chart is working for smaller data, once I want to load it for a long distance and a wide date range, the browser keeps on popping us this line : "A website is slowing down your browser, what would you like to do ? {stop} {wait}"
I know that this is probably the problem of too large amount of rows, my website generates also 3 other charts like this, with the same data but in another X-Y axis combination (for example time on X-axis, velocity on Y-axis and distance as series) and it works perfectly fine. That's why I would like to pass the data to the chart in some faster way, but I have no clue how.
I already tried to use setTimeout, but it doesn't change a lot. I also tried doing a little less coding on JS side and more in django views, but it also didn't help.
So any suggestions about how to solve this will be very appreciated!
I am following this tutorial for creating multiple area charts with D3.js and I want to customize it to my data.
His data is:
Year, Variable1, Variable2, etc
My data is
YYYYMMDD, variable1, variable2, etc
I changed this part of the code to convert the date column to a Date object:
function createChart(data){
var countries = [];
var charts = [];
var maxDataPoint = 0;
/* Loop through first row and get each country
and push it into an array to use later */
for (var prop in data[0]) {
if (data[0].hasOwnProperty(prop)) {
if (prop != 'Day') {
countries.push(prop);
}
}
};
var countriesCount = countries.length;
var startYear = (data[0].Day).substr(0,4);
var endYear = (data[data.length - 1].Day).substr(0,4);
var chartHeight = height * (1 / countriesCount);
/* Let's make sure these are all numbers,
we don't want javaScript thinking it's text
Let's also figure out the maximum data point
We'll use this later to set the Y-Axis scale
*/
data.forEach(function(d) {
for (var prop in d) {
if (d.hasOwnProperty(prop)) {
d[prop] = parseFloat(d[prop]);
if (d[prop] > maxDataPoint) {
maxDataPoint = d[prop];
}
}
}
// D3 needs a date object, let's convert it just one time
var y = (d.Day).substr(0,4),
m = (d.Day).substr(4,2) - 1,
d = (d.Day).substr(6,2);
d.Day = new Date(y,m,d);
});
This is a sample of my data and there is around 400 rows
Day NYTimes Guardian The Peninsula Gulf Times
20101201 1 8 2 0
20101203 3 9 2 0
20101205 6 10 4 1
20101207 2 9 5 1
20101209 1 3 7 0
20101211 12 8 6 0
20101213 3 4 3 0
No graph is shown, I just get a blank page and no errors on the console, take a look here. What is wrong?
Somewhere in the code, the Day value is getting converted from a string ("20101201") to an integer (20101201). You are then getting a no method substr error when trying use .substr() on a integer.
Try converting the var y to a string using .toString() by replacing line:
var y = (d.Day).substr(0,4)
With
var y = (d.Day).toString().substr(0,4)
Or consider using the d3's built in time parsing functions