Puppeteer Web Scraper runs on local machine only but fails to deploy as Google Cloud Function - javascript

I have built a simple scraper with Puppeteer which I can run locally on my machine, but when I deploy it as a Google Cloud function, it's not working. The only error message I get from the Google Cloud Logs is:
Function failed on loading user code. This is likely due to a bug in
the user code.
Here are the steps I follow to deploy it the function; code is further below.
(Note: I'm outlining the process of zipping the files; I have tried the Cloud Function inline editor as well but am receiving the same error.)
Run npm install
Run zip -r my-app.zip *
Create new Google Cloud function
-- Name 'newFunction'
-- Memory: 1gb
-- Runtime: Node.js 14
-- Entry point: scrapeFunction
Upload Zip
index.js
const puppeteer = require('puppeteer');
const { BigQuery } = require('#google-cloud/bigquery');
async function scrapeFunction() {
const browser = await puppeteer.launch({ args: ['--no-sandbox', '--disable-setuid-sandbox'] });
const page = await browser.newPage();
await page.goto('<URL>', {waitUntil: 'load', timeout: 0});
await page.waitForSelector('span.text');
const info = await page.evaluate(() => {
return document.querySelector('span.text').innerText;
});
console.log(info);
// Write results to BigQuery table
const bigqueryClient = new BigQuery();
const dataset = bigqueryClient.dataset('xxx');
const table = dataset.table('yyy');
const rows = [{ info: info }];
await table.insert(rows);
await browser.close();
}
scrapeFunction();
package.json
{
"name": "newFunction",
"version": "1.0.0",
"description": "",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"keywords": [],
"author": "",
"license": "ISC",
"dependencies": {
"#google-cloud/bigquery": "^6.1.0",
"puppeteer": "^19.7.1"
}
}

Related

discord.js .setToken(...) is not a function

I am new in discord.js v14.0.3 and trying to start a simple command but I got an error.
I checked my BOT_TOKEN, BOT_CLIENT_ID, GUILD_ID is valid and I think the problem is not related with those token, how can I fix it?
Error Message
> node register.js
/register.js:11
(async () => {
^
TypeError: (intermediate value).setToken(...) is not a function
at Object.<anonymous> ....
My code
require("dotenv").config()
const { REST } = require('#discordjs/rest')
const { Routes } = require('discord.js')
const commands = [
{
name: 'ping',
description: 'Replies with Pong!',
},
];
const rest = new REST({ version: '10' }).setToken(process.env.BOT_TOKEN)
(async () => {
try {
console.log('Started refreshing application (/) commands.')
await rest.put(Routes.applicationGuildCommands(process.env.BOT_CLIENT_ID, process.env.DC_GUILD_ID), { body: commands });
console.log('Successfully reloaded application (/) commands.')
} catch (error) {
console.error(error)
}
})()
package.json
{
"version": "1.0.0",
"description": "",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1",
"start": "node index.js",
"build": "node register.js"
},
"dependencies": {
"discord.js": "^14.0.3",
"dotenv": "^16.0.1",
"ytdl-core": "^4.11.0"
}
}
Two things:
Looks like your package.json is missing #discordjs/rest. Install it with npm install #discordjs/rest or whatever your package manager is. Since you have no error on the require it may be is installed as phantom dependency.
You simply missing a semicolon in the line const rest = new REST({ version: '10' }).setToken(process.env.BOT_TOKEN) therfore the whole block with ...setToken...(async () is considered to be one(!) expression.

Returning reason for node-fetch query results not matching rather than giving undefined. nodejs node-fetch

Note the API key has had the last characters removed as I was told not to post api keys linked to my personal account.
I am working on getting weather results from openweathermap.org Everything seems to be working except if I input an invalid city name or area code it returns undefined. I would like to return a reason for the failure in the console.
below are the dependencies being used:
"name": "node-final-exercise",
"version": "1.0.0",
"description": "",
"main": "app.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"keywords": [],
"author": "",
"license": "ISC",
"dependencies": {
"node-fetch": "^2.6.1"
}
}
heres the code snippet I'm working on right now.
// Goals for project
// API's OpenWeatherMap api.openweathermap.org/data/2.5/weather?q=London,uk&APPID=8d0f96c686dbab4b20d5dda13
// API's zip code specific api.openweathermap.org/data/2.5/weather?zip={zip code},{country code}&appid={API key}
// a. use node app.js (areacode) to connect to weather api should also accept ( city ) and (state) like cleveland ohio
// b. parse data from weather api
// c. ouput JSON parsed data into a readable format in the console.
const fetch = require("node-fetch");
const city = process.argv.slice(2);
(async () => {
try {
await fetch(
`http://api.openweathermap.org/data/2.5/weather?q=${city}&units=metric&appid=8d0f96c686dbab4b20d5dda13`
)
.then((res) => res.json())
.then((body) => console.log(body.main))
.catch(error =>{console.error("first catch error : ", error)})
} catch (error) {
console.error("error : ",error.message);
}
})();
console results

TypeError: querySnapshot.forEach is not a function - Firebase Cloud Functions

I have a data structure made this way:
Posts(collection)
UserId(document)
Posts(collection)
postDoc (document)
And I'm setting a cloud function to change all Posts(subcollection) documents upon a certain event, and in order to do so I'm using collectionGroup queries:
This is how I set it up:
exports.changeIsVisibleFieldAfterDay = functions.pubsub
.schedule("every 2 minutes").onRun((context) => {
const querySnapshot = db.collectionGroup("Posts")
.where("isVisible", "==", true).get();
querySnapshot.forEach((doc) => {
console.log(doc.data());
});
});
In the Firebase Log I receive the following error:
TypeError: querySnapshot.forEach is not a function
Searching online I found out that there may be a problem with Firebase SDK version but mine is uptodate, I attach the package.json file here:
{
"name": "functions",
"description": "Cloud Functions for Firebase",
"scripts": {
"lint": "eslint .",
"serve": "firebase emulators:start --only functions",
"shell": "firebase functions:shell",
"start": "npm run shell",
"deploy": "firebase deploy --only functions",
"logs": "firebase functions:log"
},
"engines": {
"node": "12"
},
"main": "index.js",
"dependencies": {
"firebase-admin": "^9.2.0",
"firebase-functions": "^3.11.0"
},
"devDependencies": {
"eslint": "^7.6.0",
"eslint-config-google": "^0.14.0",
"firebase-functions-test": "^0.2.0"
},
"private": true
}
The get() method is asynchronous, so you either need to use then() to get the querySnapshot when the Promise returned by get() is fulfilled, or use async/await. More details on how to deal with asynchronous calls in this SO answer.
With then()
const functions = require('firebase-functions');
// The Firebase Admin SDK to access Firestore.
const admin = require('firebase-admin');
admin.initializeApp();
const db = admin.firestore();
exports.changeIsVisibleFieldAfterDay = functions.pubsub
.schedule("every 2 minutes").onRun((context) => {
return db.collectionGroup("Posts").where("isVisible", "==", true).get()
.then(querySnapshot => {
querySnapshot.forEach((doc) => {
console.log(doc.data());
});
return null;
})
});
With async/await
const functions = require('firebase-functions');
// The Firebase Admin SDK to access Firestore.
const admin = require('firebase-admin');
admin.initializeApp();
const db = admin.firestore();
exports.changeIsVisibleFieldAfterDay = functions.pubsub
.schedule("every 2 minutes").onRun(async (context) => {
const querySnapshot = await db.collectionGroup("Posts").where("isVisible", "==", true).get();
querySnapshot.forEach((doc) => {
console.log(doc.data());
});
return null;
});
Note the return null; at the end. See this doc item for more details on this key point.
Note also that if you want to update several docs within the forEach() loop, you will need to use Promise.all(). Many SO answers cover this case.

selenium test is failing using selenium javascript webdriver

I have installed javascript selenium webdriver and write a first test like this
I am writing a test for login. I enter correct username and password and try to login.
describe('Checkout Google.com', function () {
let driver;
before(async function () {
// driver = await new Builder().forBrowser('chrome').build();
});
it('Search on Google', async function () {
let driver = await new Builder().forBrowser('chrome').build();
await driver.get('http://alpdev.s3-website-us-east-1.amazonaws.com/');
// await driver.findElement(By.name('q')).sendKeys('selenium', Key.RETURN);
this.timeout(5000);
await driver.findElement(By.xpath("//input[#placeholder='Email']")).sendKeys('owais#demoshop.com');
await driver.findElement(By.xpath("//input[#placeholder='Password']")).sendKeys('test');
await driver.findElement(By.className('btn-submit')).click().then(()=> done());
// assert.equal(title, 'dalenguyen - Google Search');
});
// close the browser after running tests
after(() => driver && driver.quit());
})
And my package json is like this
{
"name": "selenium-01",
"version": "1.0.0",
"description": "",
"main": "index.js",
"scripts": {
"test": "mocha --recursive index.js"
},
"author": "",
"license": "ISC",
"dependencies": {
"mocha": "^7.2.0",
"selenium-webdriver": "^4.0.0-alpha.7"
}
}
Now when i run ng run test so browser opens and test is passed visually meaning that login is happening but on console it prints like this
What's wrong here that it's giving an error.
So, I could not fully reproduce your case as the url in the script is not available anymore but found a few flaws though and tried some fixes.
First is your driver was declared twice.
Second, I believe the main problem (the reason you are getting the error mentioned above) is that you're using the then & done promises which are not needed as you're using the async & await functions.
Also I would recommend you use the css locators instead of the xpaths.
Another thing is you could use the async in the after(); closure, and you could use the arrow functions (async () => {}); all around instead of using the function keyword.
Below is your example but with a few changes and I am quite positive it will work (though if this is all on the google search page, there are no inputs on that page so those steps will fail, you'll have to add some extra steps to load the login page with the input fields available):
describe('Checkout Google.com', function () {
let driver;
before(async () => {
driver = await new Builder().forBrowser('chrome').build();
});
it('Search on Google', async () => {
await driver.get('http://alpdev.s3-website-us-east-1.amazonaws.com/');
await driver.findElement(By.name('q')).sendKeys('selenium', Key.RETURN);
await driver.sleep(5000);
// the steps below will fail as per the description above
await driver.findElement(By.css("input[placeholder='Email']")).sendKeys('owais#demoshop.com');
await driver.findElement(By.css("input[placeholder='Password']")).sendKeys('test');
await driver.findElement(By.className('btn-submit')).click();
// assert.equal(title, 'dalenguyen - Google Search');
});
// close the browser after running tests
after(async () => {
await driver.quit();
});
});
I hope this helps, please let me know.

Unit testing of HttpsCallable cloud functions - online mode

I am designing a backend api (for android and iOS apps) around HttpsCallable cloud functions. It becomes quite cumbersome to test them through the app, so I wish to switch to unit testing the functions (before production deployment) using the firebase-functions-test tool. I have been following this unit testing tutorial.
I am having some issues running the unit tests in online mode. Let me provide some details.
Here is my `package.json' content:
{
"name": "functions",
"description": "Cloud Functions for Firebase",
"scripts": {
"lint": "eslint .",
"serve": "firebase serve --only functions",
"shell": "firebase functions:shell",
"start": "npm run shell",
"deploy": "firebase deploy --only functions",
"logs": "firebase functions:log",
"test": "mocha --reporter spec"
},
"engines": {
"node": "8"
},
"dependencies": {
"#google-cloud/tasks": "^1.9.0",
"#googlemaps/google-maps-services-js": "^2.0.2",
"chai": "^4.2.0",
"firebase-admin": "^8.6.0",
"firebase-functions": "^3.3.0",
},
"devDependencies": {
"eslint": "^5.12.0",
"eslint-plugin-promise": "^4.0.1",
"firebase-functions-test": "^0.1.7",
"mocha": "^7.1.1"
},
"private": true
}
I am using google APIs (Directions, Geocoding etc) from firebase backend, therefore to be able to access them while running my tests, I configured my tests located at test/index.test.js as recommended in the Unit testing tutorial as follows:
const firebaseConfig = {
apiKey: ...,
authDomain: ...,
databaseURL: "https://my-project.firebaseio.com",
projectId: "my-project",
storageBucket: "my-project.appspot.com",
messagingSenderId: ********,
appId: *********
};
const test = require('firebase-functions-test')(firebaseConfig
, 'path_to_json_keyfile/myproject.json');
Below is the sample test code. Note that all my callable functions return HttpsError, and the test below, simply checks for the code field of HttpsError, if its ok, the test passes. The functions I am testing reside in a separate rides.js file, which the index.js exports as exports.rides = require(./rides.js) (not shown below)
const chai = require('chai');
const assert = chai.assert;
describe('Cloud functions', () => {
let rideFunctions;
before(() => {
rideFunctions = require('../index.js');
});
after(() => {
test.cleanup();
});
describe('getRideEstimate', () => {
it('should return ok', async () => {
data = {
riderType: 1,
pickup_lat: 37.0,
pickup_lng: -122,
dropoff_lat: 37.01,
dropoff_lng: -122.01,
scheduledTime: 1585939000,
ts: 1585939000,
tz_id: "uslax"
}
context = {
auth: {
uid: AUTH_ID_FOR_USER_ACCOUNT
}
};
const wrappedGetRideEstimate = test.wrap(rideFunctions.rides.getRideEstimate);
let httpsError = await wrappedGetRideEstimate(data, context);
return assert.equal(httpsError.code, 'ok');
});
})
describe('testCallable', () => {
it('should return ok', async () => {
data = {}
context = {}
const wrappedTestCallable = test.wrap(rideFunctions.rides.testCallable);
let httpsError = await wrappedTestCallable(data, context);
return assert.equal(httpsError.code, 'ok');
})
})
})
Problem
My simple testCallable function of the form
exports.testCallable = functions.https.onCall((data, context) => {
console.log('testCallable');
return new functions.https.HttpsError('ok', '', '');
})
passes the test (as expected) but inexplicably, it seems it is running in the offline mode, as there is no record of it on cloud function logs in Firebase Console. Also, if I disable connectivity of my laptop, the test result remains the same suggesting that somehow this function is running in the offline mode.
My getRideEstimate function which calls the google Directions API, returns a lengthy 400 error indicating some issue with forming the request to Directions API. I don't know if this error is related to the first problem, but since the Directions API call is embedded deeper inside the getRideEstimate function, it does suggest that the function is getting called, but somehow the API call is failing.
Is there any other way to test HttpsCallable functions in online mode?
For me this works:
import * as firebaseSetup from 'firebase-functions-test';
export const testEnvironment = firebaseSetup({
databaseURL: "https://project-id.firebaseio.com",
projectId: 'project-id'
}, './service-account-project-id-firebase-adminsdk.json');
You can find a full YouTube tutorial under this: https://youtu.be/UDMDpdu5-rE?t=1122

Categories

Resources