Develop Reference

Async function immediately returns undefined yet output variable returns a value immediately before the return statement

I am writing a function that downloads and converts a pdf into individual jpg files by page. I am using the imagemagick library to do the conversion. I am having trouble with my processPDF() function as it immediately returns undefined. I put a console.log statement immediately before the function returns and it returns the exact value I expect yet that value doesn't seem to be getting outside of the function for some reason.
import im from 'imagemagick'
import { promises as fs } from 'fs'
import path from 'path'
import _ from 'lodash'
import axios from 'axios'
import { v4 as uuid } from 'uuid'
async function processPDF(pdfPath) {
let basename = path.basename(pdfPath, '.pdf')
let outputPath = "./img/" + basename + ".jpg";
console.log(`Converting ${pdfPath}`)
// Take PDF file and generate individual JPG files
await im.convert(["-density", 300, pdfPath, outputPath],async (err) => {
if (err) {
console.log(err)
throw `Couldn't Process ${pdfPath}`
}
else {
// Get every file in Temporary Image Directory
let files = await fs.readdir(`./img/`)
// Append directory into filenames
files = files.map(file => {
return "./img/" + file
})
// We only want the files that match the source pdf's name
files = files.filter((file) => {
return file.includes(basename)
})
console.log(`Getting ${basename} Buffer Data`)
// For each file, read and return the buffer data along with the path
let images = await Promise.all(files.map(async file => {
const contents = await fs.readFile(file)
return { path: file, buffer: contents }
}))
// Since we read the files asynchonously, Reorder the files
images = _.orderBy(images, (image) => {
let regex = /\d*.jpg/
let res = image.path.match(regex)[0]
res = path.basename(res, '.jpg')
return res
})
let output = { pdf: pdfPath, images }
// Returns a value
console.log(output)
// Returns undefined???
return output
}
})
}
export async function downloadAndProcessPDF(url) {
// Fetch PDF from server
let { data } = await axios.get(url, {
responseType: 'arraybuffer',
headers: {
'Content-Type': 'application/json',
'Accept': 'application/pdf'
}
}).catch(e=>{
console.log(e);
throw `Can't retrieve ${url}`
})
// Generate a Unique ID for the pdf since this is called asynchronously, this will be called many times simultaneously
let id = "./pdf/" + uuid() + ".pdf"
await fs.writeFile(id, data);
// tell processPDF to process the pdf in the ./pdf directory with the given filename
let pdfData = await processPDF(id);
// Returns undefined???
console.log(pdfData)
return pdfData
}
If I had to take a wild guess I'd think that im.convert is the function that is giving me trouble. Throughout my source code i'm using promises to handle asynchronous tasks yet im.convert() uses a callback function. I'm not super familiar with how concurrency works between promises and callback functions so I think that's what's probably the issue.

How to import multiple NodeJS packages together?

const metascraper = require('metascraper')([
require('metascraper-author')(),
require('metascraper-date')(),
require('metascraper-description')(),
require('metascraper-image')(),
require('metascraper-logo')(),
require('metascraper-clearbit')(),
require('metascraper-publisher')(),
require('metascraper-title')(),
require('metascraper-url')()
])
I have the following code, but const doesn't work so I have to use import
import metascraper from 'metascraper';
import title from 'metascraper-title';
import image from 'metascraper-image';
...
This does not work as expected and returns undefined - how can I import all those libraries underneath metascraper using import. Importing them manually with their own name does not work.
(async () => {
try {
// Use the got library to fetch the website content.
const targetUrl = 'http://www.bloomberg.com/news/articles/2016-05-24/as-zenefits-stumbles-gusto-goes-head-on-by-selling-insurance';
const { body: html, url } = await got(targetUrl);
// Extract the metadata from the website content.
const metascraper = await metascraper(title(),{ html, url });
// Return the metadata as JSON
console.log(JSON.stringify(metascraper));
} catch (err) {
console.log(err);
}
})()

The following should work:
import got from "got";
import _metascraper from "metascraper";
import title from "metascraper-title";
import image from "metascraper-image";
import author from "metascraper-author";
// ...
const metascraper = _metascraper([title(), image(), author()]);
try {
// Use the got library to fetch the website content.
const targetUrl = "https://wesbos.com/scoped-github-access-token/";
const { body: html, url } = await got(targetUrl);
// Extract the metadata from the website content.
const metadata = await metascraper({ html, url });
// Return the metadata as JSON
console.log(metadata);
} catch (err) {
console.log(err);
}

Getting some problem to read the CSV file inside the firebase functions

I am trying to read the csv file inside the Firebase functions so that i can send the mail to the all the records. I am planning to go with the following procedure
upload the csv
fire a on finalize function
read the file and send emails
Below is the function
import * as functions from "firebase-functions";
import * as mkdirp from "mkdirp-promise";
import * as os from "os";
import * as path from "path";
import csv = require('csvtojson');
const gcs = require('#google-cloud/storage')({ keyFilename: 'service-account-credentials.json' });
const csvDirectory = "csv";
export = functions.storage.object().onFinalize(async (object) => {
const filePath = object.name;
const contentType = object.contentType;
const fileDir = path.dirname(filePath);
if(fileDir.startsWith(csvDirectory) && contentType.startsWith("text/csv")) {
const bucket = gcs.bucket(object.bucket);
const file = bucket.file(filePath);
const fileName = path.basename(filePath);
const tempLocalFile = path.join(os.tmpdir(), filePath);
const tempLocalDir = path.dirname(tempLocalFile);
console.log("values", bucket, file, fileName, tempLocalDir, tempLocalFile);
console.log("csv file uploadedeeeed");
await mkdirp(tempLocalDir);
await bucket.file(filePath).download({
destination: tempLocalFile
});
console.log('The file has been downloaded to', tempLocalFile);
csv()
.fromFile(tempLocalFile)
.then((jsonObj) => {
console.log(jsonObj);
})
}
});
While running the code i am only getting csv file uploadeded which i have written inside the console.log and then i get the timeout after 1 minute .i am also not getting the The file has been downloaded to log . Can anybody look at the code and help me to get out of this.

You are mixing up the use of async/await together with a call to then() method. You should also use await for the fromFile() method.
The following should do the trick (untested):
export = functions.storage.object().onFinalize(async (object) => {
const filePath = object.name;
const contentType = object.contentType;
const fileDir = path.dirname(filePath);
try {
if (fileDir.startsWith(csvDirectory) && contentType.startsWith("text/csv")) {
//.....
await mkdirp(tempLocalDir);
await bucket.file(filePath).download({
destination: tempLocalFile
});
console.log('The file has been downloaded to', tempLocalFile);
const jsonObj = await csv().fromFile(tempLocalFile);
console.log(jsonObj);
return null;
} else {
//E.g. throw an error
}
} catch (error) {
//.....
}
});
Also note that (independently of the mixed use of async/await and then()), with the following line in your code
csv().fromFile(tempLocalFile).then(...)
you were not returning the Promise returned by the fromFile() method. This is a key point in Cloud Functions.
I would suggest you watch the official Video Series on Cloud Functions (https://firebase.google.com/docs/functions/video-series/) and in particular the videos on Promises titled "Learn JavaScript Promises".

Unable to use external API on Botpress (axios)

When trying to use axis to query an external Weather API, I get this error
ReferenceError: axios is not defined
at getTropicalCyclones (vm.js:16:9)
Here is my action for getTropicalCyclones {}
(of course I have to hide my client ID and secret)
const getTropicalCyclones = async () => {
const BASE_WEATHER_API = `https://api.aerisapi.com/tropicalcyclones/`
const CLIENT_ID_SECRET = `SECRET`
const BASIN = `currentbasin=wp`
const PLACE = `p=25,115,5,135` // rough coords for PH area of responsibility
const ACTION = `within` // within, closest, search, affects or ''
try {
let text = ''
let response = {}
await axios.get(
`${BASE_WEATHER_API}${ACTION}?${CLIENT_ID_SECRET}&${BASIN}&${PLACE}`
)
.then((resp) => {]
response = resp
text = 'Success retrieving weather!'
})
.catch((error) => {
console.log('!! error', error)
})
const payload = await bp.cms.renderElement(
'builtin_text',
{
text,
},
event.channel
)
await bp.events.replyToEvent(event, payload)
} catch (e) {
// Failed to fetch, this is where ReferenceError: axios is not defined comes from
console.log('!! Error while trying to fetch weather info', e)
const payload = await bp.cms.renderElement(
'builtin_text',
{
text: 'Error while trying to fetch weather info.',
},
event.channel
)
await bp.events.replyToEvent(event, payload)
}
}
return getTropicalCyclones()
So my question is, how do I import axios? I've tried
const axios = require('axios')
or
import axios from 'axios';
but this causes a different error:
Error processing "getTropicalCyclones {}"
Err: An error occurred while executing the action "getTropicalCyclones"
Looking at the package.json on GitHub, it looks like axios is already installed
https://github.com/botpress/botpress/blob/master/package.json
However, I cannot locate this package.json on my bot directory...
Secondly, based on an old version doc it looks like this example code just used axios straight
https://botpress.io/docs/10.31/recipes/apis/
How do I use axios on Botpress?
Any leads would be appreciated
Botpress: v11.0.0

Simply use ES6 import.
include this line at the top of your code.
import axios from 'axios';
Note: I'm expecting that the axios is already installed

How to call module with object Parameter in main file Index.js

I need to call module from main index.js File
Here is my module
const request = require('./rq.js');
const callback = require('./callback.js')
const url = `https://localhost.3000/${id}`;
request(url, callback)
.then(res => {
console.log(res);
})
.catch(err => {
console.log(err);
})
module.exports = page; //Tell me how to export all code from module
So here is my index.js file
const Methods = {
page: require('./page.js'),
}
module.exports = //What i need to code here?
File from what i give a call a module :
const main = require('./index.js');
main.page({id: 'id'})
.then(console.log);
So what I should change to call page.js file like that ?

Make the following changes to page.js since in your main file you expect a promise to be returned.
const request = require('./rq.js');
const callback = require('./callback.js')
function page({id}) {
const url = `https://localhost.3000/${id}`;
return request(url, callback)
}
module.exports = {page: page} //Tell me how to export all code from module
Make the following changes to Mehods.js
const Methods = {
page: require('./page.js').page,
}
module.exports = Methods;
Check if this works.