Develop Reference

Async function immediately returns undefined yet output variable returns a value immediately before the return statement

I am writing a function that downloads and converts a pdf into individual jpg files by page. I am using the imagemagick library to do the conversion. I am having trouble with my processPDF() function as it immediately returns undefined. I put a console.log statement immediately before the function returns and it returns the exact value I expect yet that value doesn't seem to be getting outside of the function for some reason.
import im from 'imagemagick'
import { promises as fs } from 'fs'
import path from 'path'
import _ from 'lodash'
import axios from 'axios'
import { v4 as uuid } from 'uuid'
async function processPDF(pdfPath) {
let basename = path.basename(pdfPath, '.pdf')
let outputPath = "./img/" + basename + ".jpg";
console.log(`Converting ${pdfPath}`)
// Take PDF file and generate individual JPG files
await im.convert(["-density", 300, pdfPath, outputPath],async (err) => {
if (err) {
console.log(err)
throw `Couldn't Process ${pdfPath}`
}
else {
// Get every file in Temporary Image Directory
let files = await fs.readdir(`./img/`)
// Append directory into filenames
files = files.map(file => {
return "./img/" + file
})
// We only want the files that match the source pdf's name
files = files.filter((file) => {
return file.includes(basename)
})
console.log(`Getting ${basename} Buffer Data`)
// For each file, read and return the buffer data along with the path
let images = await Promise.all(files.map(async file => {
const contents = await fs.readFile(file)
return { path: file, buffer: contents }
}))
// Since we read the files asynchonously, Reorder the files
images = _.orderBy(images, (image) => {
let regex = /\d*.jpg/
let res = image.path.match(regex)[0]
res = path.basename(res, '.jpg')
return res
})
let output = { pdf: pdfPath, images }
// Returns a value
console.log(output)
// Returns undefined???
return output
}
})
}
export async function downloadAndProcessPDF(url) {
// Fetch PDF from server
let { data } = await axios.get(url, {
responseType: 'arraybuffer',
headers: {
'Content-Type': 'application/json',
'Accept': 'application/pdf'
}
}).catch(e=>{
console.log(e);
throw `Can't retrieve ${url}`
})
// Generate a Unique ID for the pdf since this is called asynchronously, this will be called many times simultaneously
let id = "./pdf/" + uuid() + ".pdf"
await fs.writeFile(id, data);
// tell processPDF to process the pdf in the ./pdf directory with the given filename
let pdfData = await processPDF(id);
// Returns undefined???
console.log(pdfData)
return pdfData
}
If I had to take a wild guess I'd think that im.convert is the function that is giving me trouble. Throughout my source code i'm using promises to handle asynchronous tasks yet im.convert() uses a callback function. I'm not super familiar with how concurrency works between promises and callback functions so I think that's what's probably the issue.

Google Text-to-speech - Loading text from individual lines of a txt file

I am using the Google TextToSpeech API in Node.js to generate speech from text. I was able to get an output file with the same name as the text that is generated for the speech. However, I need to tweak this a bit. I wish I could generate multiple files at the same time. The point is that I have, for example, 5 words (or sentences) to generate, e.g. cat, dog, house, sky, sun. I would like to generate them each to a separate file: cat.wav, dog.wav, etc.
I also want the application to be able to read these words from the * .txt file (each word/sentence on a separate line of the * .txt file).
Is there such a possibility? Below I am pasting the * .js file code and the * .json file code that I am using.
*.js
const textToSpeech = require('#google-cloud/text-to-speech');
const fs = require('fs');
const util = require('util');
const projectId = 'forward-dream-295509'
const keyFilename = 'myauth.json'
const client = new textToSpeech.TextToSpeechClient({ projectId, keyFilename });
const YourSetting = fs.readFileSync('setting.json');
async function Text2Speech(YourSetting) {
const [response] = await client.synthesizeSpeech(JSON.parse(YourSetting));
const writeFile = util.promisify(fs.writeFile);
await writeFile(JSON.parse(YourSetting).input.text + '.wav', response.audioContent, 'binary');
console.log(`Audio content written to file: ${JSON.parse(YourSetting).input.text}`);
}
Text2Speech(YourSetting);
*.json
{
"audioConfig": {
"audioEncoding": "LINEAR16",
"pitch": -2,
"speakingRate": 1
},
"input": {
"text": "Text to Speech"
},
"voice": {
"languageCode": "en-US",
"name": "en-US-Wavenet-D"
}
}
I'm not very good at programming. I found a tutorial on google on how to do this and slightly modified it so that the name of the saved file was the same as the generated text.
I would be very grateful for your help.
Arek

Here ya go - I haven't tested it, but this should show how to read a text file, split into each line, then run tts over it with a set concurrency. It uses the p-any and filenamify npm packages which you'll need to add to your project. Note that google may have API throttling or rate limits that I didn't take into account here - may consider using p-throttle library if that's a concern.
// https://www.npmjs.com/package/p-map
const pMap = require('p-map');
// https://github.com/sindresorhus/filenamify
const filenamify = require('filenamify');
const textToSpeech = require('#google-cloud/text-to-speech');
const fs = require('fs');
const path = require('path');
const projectId = 'forward-dream-295509'
const keyFilename = 'myauth.json'
const client = new textToSpeech.TextToSpeechClient({ projectId, keyFilename });
const rawSettings = fs.readFileSync('setting.json', { encoding: 'utf8'});
// base data for all requests (voice, etc)
const yourSetting = JSON.parse(rawSettings);
// where wav files will be put
const outputDirectory = '.';
async function Text2Speech(text, outputPath) {
// include the settings in settings.json, but change text input
const request = {
...yourSetting,
input: { text }
};
const [response] = await client.synthesizeSpeech(request);
await fs.promises.writeFile(outputPath, response.audioContent, 'binary');
console.log(`Audio content written to file: ${text} = ${outputPath}`);
// not really necessary, but you could return something if you wanted to
return response;
}
// process a line of text - write to file and report result (success/error)
async function processLine(text, index) {
// create output path based on text input (use library to ensure it's filename safe)
const outputPath = path.join(outputDirectory, filenamify(text) + '.wav');
const result = {
text,
lineNumber: index,
path: outputPath,
isSuccess: null,
error: null
};
try {
const response = await Text2Speech(text, outputPath);
result.isSuccess = true;
} catch (error) {
console.warn(`Failed: ${text}`, error);
result.isSuccess = false;
result.error = error;
}
return result;
}
async function processInputFile(filepath, concurrency = 3) {
const rawText = fs.readFileSync(filepath, { encoding: 'utf8'});
const lines = rawText
// split into one item per line
.split(/[\r\n]+/)
// remove surrounding whitespace
.map(s => s.trim())
// remove empty lines
.filter(Boolean);
const results = await pMap(lines, processLine, { concurrency });
console.log('Done!');
console.table(results);
}
// create sample text file
const sampleText = `Hello World
cat
dog
another line of text`;
fs.writeFileSync('./my-text-lines.txt', sampleText);
// process each line in the text file, 3 at a time
processInputFile('./my-text-lines.txt', 3);

How to build a readable stream in Node.js and TypeScript?

I connected the typescript function to Azure Blobstorage through Rest-API and this works fine for me. Now I want to get each blob contents and read the contents of each blobs.
I try this with this code here, but it returns an error:
const blobServiceClient = new BlobServiceClient(`https://${accountName}.blob.core.windows.net?${sasToken}`,
pipeline)
const containerClient = blobServiceClient.getContainerClient(containerName)
console.log(containerClient)
if (!containerClient.exists()) {
console.log("the container does not exit")
await containerClient.create()
}
const client = containerClient.getBlockBlobClient(this.currentFile.name)
//name of uploded blob
console.log(this.currentFile.name)
//metaata from the blob
console.log(client)
//List each blobs in the container
for await (const blob of containerClient.listBlobsFlat()) {
console.log('\t', blob.name);
const blockBlobClient = containerClient.getBlockBlobClient(blob.name);
const downloadBlockBlobResponse = await blockBlobClient.download(0);
console.log('\nDownloaded blob content...');
console.log('\t', await streamToString(downloadBlockBlobResponse.readableStreamBody));
//end of loop
}
async function streamToString(readableStream) {
return new Promise((resolve, reject) => {
const chunks = [];
readableStream.on("data", (data) => {
chunks.push(data.toString());
});
readableStream.on("end", () => {
resolve(chunks.join(""));
});
readableStream.on("error", reject);
});
}
The error is :
ERROR Error: Uncaught (in promise): TypeError: Cannot read property 'on' of undefined TypeError: Cannot read property 'on' of undefined
So how to solve the problem?
Thanks

Download the official sample code.
It runs normally on my side. Check if your local lack of dependencies, or the permissions in the storage need to be set.
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
/*
Setup: Enter your storage account name and shared key in main()
*/
import {
BlobServiceClient,
StorageSharedKeyCredential,
BlobDownloadResponseModel
} from "#azure/storage-blob";
// Load the .env file if it exists
import * as dotenv from "dotenv";
dotenv.config();
export async function main() {
// Enter your storage account name and shared key
const account = process.env.ACCOUNT_NAME || "pans*****age";
const accountKey = process.env.ACCOUNT_KEY || "IHa48xxo+0anyKQ2GzQ2K*******ZBxgJ0VotCpGs/PMftkebb9UFqyg==";
// Use StorageSharedKeyCredential with storage account and account key
// StorageSharedKeyCredential is only available in Node.js runtime, not in browsers
const sharedKeyCredential = new StorageSharedKeyCredential(account, accountKey);
// ONLY AVAILABLE IN NODE.JS RUNTIME
// DefaultAzureCredential will first look for Azure Active Directory (AAD)
// client secret credentials in the following environment variables:
//
// - AZURE_TENANT_ID: The ID of your AAD tenant
// - AZURE_CLIENT_ID: The ID of your AAD app registration (client)
// - AZURE_CLIENT_SECRET: The client secret for your AAD app registration
//
// If those environment variables aren't found and your application is deployed
// to an Azure VM or App Service instance, the managed service identity endpoint
// will be used as a fallback authentication source.
// const defaultAzureCredential = new DefaultAzureCredential();
// You can find more TokenCredential implementations in the [#azure/identity](https://www.npmjs.com/package/#azure/identity) library
// to use client secrets, certificates, or managed identities for authentication.
// Use AnonymousCredential when url already includes a SAS signature
// const anonymousCredential = new AnonymousCredential();
// List containers
const blobServiceClient = new BlobServiceClient(
// When using AnonymousCredential, following url should include a valid SAS or support public access
`https://${account}.blob.core.windows.net`,
sharedKeyCredential
);
let i = 1;
for await (const container of blobServiceClient.listContainers()) {
console.log(`Container ${i++}: ${container.name}`);
}
// Create a container
const containerName = `newcontainer${new Date().getTime()}`;
const containerClient = blobServiceClient.getContainerClient(containerName);
const createContainerResponse = await containerClient.create();
console.log(`Create container ${containerName} successfully`, createContainerResponse.requestId);
// Create a blob
const content = "hello, 你好";
const blobName = "newblob" + new Date().getTime();
const blockBlobClient = containerClient.getBlockBlobClient(blobName);
const uploadBlobResponse = await blockBlobClient.upload(content, Buffer.byteLength(content));
console.log(`Upload block blob ${blobName} successfully`, uploadBlobResponse.requestId);
// List blobs
i = 1;
for await (const blob of containerClient.listBlobsFlat()) {
console.log(`Blob ${i++}: ${blob.name}`);
}
// Get blob content from position 0 to the end
// In Node.js, get downloaded data by accessing downloadBlockBlobResponse.readableStreamBody
// In browsers, get downloaded data by accessing downloadBlockBlobResponse.blobBody
const downloadBlockBlobResponse: BlobDownloadResponseModel = await blockBlobClient.download(0);
console.log(
"Downloaded blob content",
await streamToString(downloadBlockBlobResponse.readableStreamBody!)
);
// Delete container
await containerClient.delete();
console.log("deleted container");
}
// A helper method used to read a Node.js readable stream into string
async function streamToString(readableStream: NodeJS.ReadableStream) {
return new Promise((resolve, reject) => {
const chunks: string[] = [];
readableStream.on("data", (data) => {
chunks.push(data.toString());
});
readableStream.on("end", () => {
resolve(chunks.join(""));
});
readableStream.on("error", reject);
});
}
main().catch((err) => {
console.error("Error running sample:", err.message);
});

How to zip files PDF from a Storage in NodeJS

I need to create a zip file with any PDF what I recieved from Storage AWS, and I am trying do this with ADM-zip in NodeJS, but i cant read the final file.zip.
Here is the code.
var zip = new AdmZip();
// add file directly
var content = data.Body.buffer;
zip.addFile("test.pdf", content, "entry comment goes here");
// console.log(content)
// add local file
zip.addLocalFile(`./tmp/boletos/doc.pdf`);
// // get everything as a buffer
var willSendthis = zip.toBuffer();
console.log(willSendthis)
// // or write everything to disk
zip.writeZip("test.zip", `../../../tmp/boletos/${datastring}.zip`);
As it is this only creates a .zip for each file..zip

I was also facing this issue. I looked through a lot of SO posts. This is how I was able to create a zip with multiple files from download urls. Please keep in mind, I'm unsure this is best practice, or if this is going to blow up memory.
Create a zip folder from a list of id's of requested resources via the client.
const zip = new AdmZip();
await Promise.all(sheetIds.map(async (sheetId) => {
const downloadUrl = await this.downloadSds({ sheetId, userId, memberId });
if (downloadUrl) {
await new Promise((resolve) => https.get(downloadUrl, (res) => {
const data = [];
res.on('data', (chunk) => {
data.push(chunk);
}).on('end', () => {
const buffer = Buffer.concat(data);
resolve(zip.addFile(`${sheetId}.pdf`, buffer));
});
}));
} else {
console.log('could not download');
}
}));
const zipFile = zip.toBuffer();
I then used downloadjs in my React.js client to download.
const result = await sds.bulkDownloadSds(payload);
if (result.status > 399) return store.rejectWithValue({ errorMessage: result?.message || 'Error', redirect: result.redirect });
const filename = 'test1.zip';
const document = await result.blob();
download(document, filename, 'zip');

Problem with accessing file in Firebase Storage with Firebase Cloud Functions, file returns "[object Object] "

When trying to access an image in my home directory of Firebase storage with node.js functions, I'm getting [object Object] as a response. I guess I initialized the bucket incorrectly, but not sure where I'm going wrong.
That's the debug info in firebase functions:
ChildProcessError: `composite -compose Dst_Out [object Object] [object Object] /tmp/output_final2.png` failed with code 1
Here's my code:
const admin = require('firebase-admin');
admin.initializeApp();
const storage = admin.storage();
const os = require('os');
const path = require('path');
const spawn = require('child-process-promise').spawn;
exports.onFileChange= functions.storage.object().onFinalize(async object => {
const bucket = storage.bucket('myID.appspot.com/');
const contentType = object.contentType;
const filePath = object.name;
console.log('File change detected, function execution started');
if (object.resourceState === 'not_exists') {
console.log('We deleted a file, exit...');
return;
}
if (path.basename(filePath).startsWith('changed-')) {
console.log('We already changed that file!');
return;
}
const destBucket = bucket;
const tmpFilePath = path.join(os.tmpdir(), path.basename(filePath));
const border = bucket.file("border.png");
const mask1 = bucket.file("mask1.png");
const metadata = { contentType: contentType };
return destBucket.file(filePath).download({
destination: tmpFilePath
}).then(() => {
return spawn('composite', ['-compose', 'Dst_Out', mask1, border, tmpFilePath]);
}).then(() => {
return destBucket.upload(tmpFilePath, {
destination: 'changed-' + path.basename(filePath),
metadata: metadata
})
}); });```

If, with
const bucket = storage.bucket('myID.appspot.com/');
your goal is to initialize the default bucket, you should just do
const bucket = storage.bucket();
since you have declared storage as admin.storage()
UPDATE (following your comment about const border = bucket.file("border.png");)
In addition, by looking at the code of a similar Cloud Function (from the official samples, using ImageMagick and spawn) it appears that you should not pass to the spawn() method some File objects created through the file() method of the Cloud Storage Node.js Client API (i.e. const border = bucket.file("border.png");) but some files that you have previously saved to a temp directory.
Look at the following excerpt from the Cloud Function example referred to above. They define some temporary directory and file paths (using the path module), download the files to this directory and use them to call the spawn() method.
//....
const filePath = object.name;
const contentType = object.contentType; // This is the image MIME type
const fileDir = path.dirname(filePath);
const fileName = path.basename(filePath);
const thumbFilePath = path.normalize(path.join(fileDir, `${THUMB_PREFIX}${fileName}`)); // <---------
const tempLocalFile = path.join(os.tmpdir(), filePath); // <---------
const tempLocalDir = path.dirname(tempLocalFile); // <---------
const tempLocalThumbFile = path.join(os.tmpdir(), thumbFilePath); // <---------
//....
// Cloud Storage files.
const bucket = admin.storage().bucket(object.bucket);
const file = bucket.file(filePath);
const thumbFile = bucket.file(thumbFilePath);
const metadata = {
contentType: contentType,
// To enable Client-side caching you can set the Cache-Control headers here. Uncomment below.
// 'Cache-Control': 'public,max-age=3600',
};
// Create the temp directory where the storage file will be downloaded.
await mkdirp(tempLocalDir) // <---------
// Download file from bucket.
await file.download({destination: tempLocalFile}); // <---------
console.log('The file has been downloaded to', tempLocalFile);
// Generate a thumbnail using ImageMagick.
await spawn('convert', [tempLocalFile, '-thumbnail', `${THUMB_MAX_WIDTH}x${THUMB_MAX_HEIGHT}>`, tempLocalThumbFile], {capture: ['stdout', 'stderr']});
//.....

You can't pass Cloud Storage File type objects to spawn. You need to pass strings that will be used to create the command line. This means you need to download those files locally to /tmp before you can work with them - ImageMagick doesn't know how to work file in Cloud Storage.