I am in the process of slowly rewriting my blog and part of it is adding support for comments. In my mind I want to take the commenting a step further and allow people to use BBcode shortcodes.
Searching around for ready to use libraries was not a success, as most of these are either abandoned, bugged and so on. With that in mind I decided to spend some time and create the necessary regex patterns to replace the BB codes I want to allow in comments with their relevant substitutes.
My problem is that I need to execute all these in "one go" on the comment content if take makes sense, perhaps using some sort of array/map. The API is written using TypeScript and execution speed/CPU usage is vital as the API sits on serverless computing.
Please see what I have created below (special thanks to urlregex.com and devdreamz.com):
const STRONG = '(?:\\[b\\]([a-zA-Z\\s\\d]+)\\[\\/b\\])+';
const regexSTRONG = new RegExp(`${STRONG}`, 'gi');
const substSTRONG = '<strong>$1</strong>';
let str = `[b]bolded text[/b]`;
let result = str.replace(regexSTRONG, substSTRONG);
console.log('Substitution result: ', result);
const ITALIC = '(?:\\[i\\]([a-zA-Z\\s\\d]+)\\[\\/i\\])+';
const regexITALIC = new RegExp(`${ITALIC}`, 'gi');
const substITALIC = '<em>$1</em>';
str = `[i]italicized text[/i]`;
result = str.replace(regexITALIC, substITALIC);
console.log('Substitution result: ', result);
const UNDERLINED = '(?:\\[u\\]([a-zA-Z\\s\\d]+)\\[\\/u\\])+';
const regexUNDERLINED = new RegExp(`${UNDERLINED}`, 'gi');
const substUNDERLINED = '<ins>$1</ins>';
str = `[u]underlined text[/u]`;
result = str.replace(regexUNDERLINED, substUNDERLINED);
console.log('Substitution result: ', result);
const STRIKETHROUGH = '(?:\\[s\\]([a-zA-Z\\s\\d]+)\\[\\/s\\])+';
const regexSTRIKETHROUGH = new RegExp(`${STRIKETHROUGH}`, 'gi');
const substSTRIKETHROUGH = '<del>$1</del>';
str = `[s]strikethrough text[/s]`;
result = str.replace(regexSTRIKETHROUGH, substSTRIKETHROUGH);
console.log('Substitution result: ', result);
const uURL = '(?:\\[url\\]((([A-Za-z]{3,9}:(?:\\/\\/)?)(?:[\\-;:&=\\+\\$,\\w]+#)?[A-Za-z0-9\\.\\-]+|(?:www\\.|[\\-;:&=\\+\\$,\\w]+#)[A-Za-z0-9\\.\\-]+)((?:\\/[\\+~%\\/\\.\\w\\-_]*)?\\??(?:[\\-\\+=&;%#\\.\\w_]*)#?(?:[\\.\\!\\/\\\\\\w]*))?)\\[\\/url\\])+';
const regexURL = new RegExp(`${uURL}`, 'gi');
const substURL = '$1';
str = `[url]https://en.wikipedia.org[/url]`;
result = str.replace(regexURL, substURL);
console.log('Substitution result: ', result);
const uURLALT = '(?:\\[url=((([A-Za-z]{3,9}:(?:\\/\\/)?)(?:[\\-;:&=\\+\\$,\\w]+#)?[A-Za-z0-9\\.\\-]+|(?:www\\.|[\\-;:&=\\+\\$,\\w]+#)[A-Za-z0-9\\.\\-]+)((?:\\/[\\+~%\\/\\.\\w\\-_]*)?\\??(?:[\\-\\+=&;%#\\.\\w_]*)#?(?:[\\.\\!\\/\\\\\\w]*))?)\\]([a-zA-Z\\s\\d]+)\\[\\/url\\])+';
const regexURLALT = new RegExp(`${uURLALT}`, 'gi');
const substURLALT = '$5';
str = `[url=https://en.wikipedia.org]English Wikipedia[/url]`;
result = str.replace(regexURLALT, substURLALT);
console.log('Substitution result: ', result);
const IMG = '(?:\\[img=((([A-Za-z]{3,9}:(?:\\/\\/)?)(?:[\\-;:&=\\+\\$,\\w]+#)?[A-Za-z0-9\\.\\-]+|(?:www\\.|[\\-;:&=\\+\\$,\\w]+#)[A-Za-z0-9\\.\\-]+)((?:\\/[\\+~%\\/\\.\\w\\-_]*)?\\??(?:[\\-\\+=&;%#\\.\\w_]*\\.(?:jpg|jpeg|gif|png))#?(?:[\\.\\!\\/\\\\\\w]*))?)\\]([a-zA-Z\\s\\d]+)\\[\\/img\\])+';
const regexIMG = new RegExp(`${IMG}`, 'gi');
const substIMG = '<img src="$1" alt="$5" />';
str = `[img=https://upload.wikimedia.org/wikipedia/commons/7/70/Example.png]Example Image[/img]`;
result = str.replace(regexIMG, substIMG);
console.log('Substitution result: ', result);
const QUOTE = '(?:\\[quote\\]([a-zA-Z\\s\\d]+)\\[\\/quote\\])+';
const regexQUOTE = new RegExp(`${QUOTE}`, 'gi');
const substQUOTE = '<blockquote><p>$1</p></blockquote>';
str = `[quote]quoted text[/quote]`;
result = str.replace(regexQUOTE, substQUOTE);
console.log('Substitution result: ', result);
const QUOTEALT = '(?:\\[quote=\\"([a-zA-Z\\s\\d]+)\\"\\]([a-zA-Z\\s\\d]+)\\[\\/quote\\])+';
const regexQUOTEALT = new RegExp(`${QUOTEALT}`, 'gi');
const substQUOTEALT = '<blockquote><p>$2</p><cite>$1</cite></blockquote>';
str = `[quote="author"]quoted text[/quote]`;
result = str.replace(regexQUOTEALT, substQUOTEALT);
console.log('Substitution result: ', result);
const CODE = '(?:\\[code])([^[]*(?:\\[(?!\\/?code[\\]=])[^[]*)*)(?:\\[\\/code])+';
const regexCODE = new RegExp(`${CODE}`, 'gi');
const substCODE = '<code><pre>$1</pre></code>';
str = `[code]monospaced text[/code]`;
result = str.replace(regexCODE, substCODE);
console.log('Substitution result: ', result);
Unfortunately my JavaScript/TypeScript knowledge is quite limited and I am still in the process of getting used to them whilst I develop my blog.
Please let me know how I can achieve my goal.
Any advice is much appreciated!
I ended up using the following solution:
const STRONG = '(?:\\[b\\]([a-zA-Z\\s\\d,.\\\'";:\\-_=+*?!\\(\\)\\{\\}\\\\\\/|#]+)\\[\\/b\\])+';
const regexSTRONG = new RegExp(`${STRONG}`, 'gi');
const substSTRONG = '<strong>$1</strong>';
const ITALIC = '(?:\\[i\\]([a-zA-Z\\s\\d,.\\\'";:\\-_=+*?!\\(\\)\\{\\}\\\\\\/|#]+)\\[\\/i\\])+';
const regexITALIC = new RegExp(`${ITALIC}`, 'gi');
const substITALIC = '<em>$1</em>';
const UNDERLINED = '(?:\\[u\\]([a-zA-Z\\s\\d,.\\\'";:\\-_=+*?!\\(\\)\\{\\}\\\\\\/|#]+)\\[\\/u\\])+';
const regexUNDERLINED = new RegExp(`${UNDERLINED}`, 'gi');
const substUNDERLINED = '<ins>$1</ins>';
const STRIKETHROUGH = '(?:\\[s\\]([a-zA-Z\\s\\d,.\\\'";:\\-_=+*?!\\(\\)\\{\\}\\\\\\/|#]+)\\[\\/s\\])+';
const regexSTRIKETHROUGH = new RegExp(`${STRIKETHROUGH}`, 'gi');
const substSTRIKETHROUGH = '<del>$1</del>';
const uURL = '(?:\\[url\\]((([A-Za-z]{3,9}:(?:\\/\\/)?)(?:[\\-;:&=\\+\\$,\\w]+#)?[A-Za-z0-9\\.\\-]+|(?:www\\.|[\\-;:&=\\+\\$,\\w]+#)[A-Za-z0-9\\.\\-]+)((?:\\/[\\+~%\\/\\.\\w\\-_]*)?\\??(?:[\\-\\+=&;%#\\.\\w_]*)#?(?:[\\.\\!\\/\\\\\\w]*))?)\\[\\/url\\])+';
const regexURL = new RegExp(`${uURL}`, 'gi');
const substURL = '$1';
const uURLALT = '(?:\\[url=((([A-Za-z]{3,9}:(?:\\/\\/)?)(?:[\\-;:&=\\+\\$,\\w]+#)?[A-Za-z0-9\\.\\-]+|(?:www\\.|[\\-;:&=\\+\\$,\\w]+#)[A-Za-z0-9\\.\\-]+)((?:\\/[\\+~%\\/\\.\\w\\-_]*)?\\??(?:[\\-\\+=&;%#\\.\\w_]*)#?(?:[\\.\\!\\/\\\\\\w]*))?)\\]([a-zA-Z\\s\\d]+)\\[\\/url\\])+';
const regexURLALT = new RegExp(`${uURLALT}`, 'gi');
const substURLALT = '$5';
const IMG = '(?:\\[img=((([A-Za-z]{3,9}:(?:\\/\\/)?)(?:[\\-;:&=\\+\\$,\\w]+#)?[A-Za-z0-9\\.\\-]+|(?:www\\.|[\\-;:&=\\+\\$,\\w]+#)[A-Za-z0-9\\.\\-]+)((?:\\/[\\+~%\\/\\.\\w\\-_]*)?\\??(?:[\\-\\+=&;%#\\.\\w_]*\\.(?:jpg|jpeg|gif|png))#?(?:[\\.\\!\\/\\\\\\w]*))?)\\]([a-zA-Z\\s\\d]+)\\[\\/img\\])+';
const regexIMG = new RegExp(`${IMG}`, 'gi');
const substIMG = '<img src="$1" alt="$5" />';
const QUOTE = '(?:\\[quote\\]([a-zA-Z\\s\\d,.\\\'";:\\-_=+*?!\\(\\)\\{\\}\\\\\\/|#]+)\\[\\/quote\\])+';
const regexQUOTE = new RegExp(`${QUOTE}`, 'gi');
const substQUOTE = '<blockquote><p>$1</p></blockquote>';
const QUOTEALT = '(?:\\[quote=\\"([a-zA-Z\\s\\d]+)\\"\\]([a-zA-Z\\s\\d,.\\\'";:\\-_=+*?!\\(\\)\\{\\}\\\\\\/|#]+)\\[\\/quote\\])+';
const regexQUOTEALT = new RegExp(`${QUOTEALT}`, 'gi');
const substQUOTEALT = '<blockquote><p>$2</p><cite>$1</cite></blockquote>';
const CODE = '(?:\\[code])([^[]*(?:\\[(?!\\/?code[\\]=])[^[]*)*)(?:\\[\\/code])+';
const regexCODE = new RegExp(`${CODE}`, 'gi');
const substCODE = '<code><pre>$1</pre></code>';
const NEWLINE = '(.+?)(\\n|$)+';
const regexNEWLINE = new RegExp(`${NEWLINE}`, 'gi');
const substNEWLINE = '<p>$1</p>';
const RegexArray: readonly RegExp[] = [regexSTRONG,
regexITALIC, regexUNDERLINED, regexSTRIKETHROUGH, regexURL, regexURLALT, regexIMG, regexQUOTE, regexQUOTEALT, regexCODE];
const SubstArray: readonly string[] = [substSTRONG,
substITALIC, substUNDERLINED, substSTRIKETHROUGH, substURL, substURLALT, substIMG, substQUOTE, substQUOTEALT, substCODE];
// https://github.com/dsblv/string-replace-async
// Fixed up a bit.
const replaceAsync = async (text: string, searchValue: RegExp, replacer: any): Promise<string> => {
try {
if (typeof replacer === "function") {
// 1. Run fake pass of `replace`, collect values from `replacer` calls
// 2. Resolve them with `Promise.all`
// 3. Run `replace` with resolved values
var values:any = [];
String.prototype.replace.call(text, searchValue, function () {
values.push(replacer.apply(undefined, arguments));
return "";
});
return Promise.all(values).then(function (resolvedValues) {
return String.prototype.replace.call(text, searchValue, function () {
return resolvedValues.shift();
});
});
} else {
return Promise.resolve(
String.prototype.replace.call(text, searchValue, replacer)
);
}
} catch (error) {
return Promise.reject(error);
}
}
const parseBBCode = async (text: string): Promise<string> => {
try{
for (var i = 0; i < RegexArray.length; i++) {
text = await replaceAsync(text, RegexArray[i], SubstArray[i]);
}
return text;
} catch (error) {
return Promise.reject(error);
}
};
async function test()
{
let text = `Hello World! This is just a [b]test[/b] to see how this idea [i]works[/i]. Maybe it [s]doesn't work at all[/s].
We all like to code: [code]var Banana = 0;[/code].
How are [url=https://stackoverflow.com/questions/72170180/execute-multiple-regexp-patterns-in-one-shot-using-typescript]links[/url] working?
How about images? [img=https://upload.wikimedia.org/wikipedia/commons/7/70/Example.png]Example Image[/img]`;
text = await parseBBCode(text);
const result = text.replace(regexNEWLINE, substNEWLINE);
console.log(result)
};
test();
It's async/await-able and works on replacements concurrently, overall pretty fast.
If anyone has any suggestions on how to improve this feel free to comment and I will update my answer.
I am trying to transfer SPL tokens and am getting the error from the function
mintToken.getOrCreateAssociatedAccountInfo(wallet.publicKey);
Error: Invalid seeds, address must fall off the curve
My wallet variable a an AnchorWallet
ToWallet is obtained via:
var toWallet = anchor.web3.Keypair.fromSecretKey(DEMO_TO_WALLET);
try {
if (wallet) {
const mintPublicKey = new anchor.web3.PublicKey("Token address");
const mintToken = new Token(
props.connection,
mintPublicKey,
TOKEN_PROGRAM_ID,
toWallet
);
const fromTokenAccount = await mintToken.getOrCreateAssociatedAccountInfo(
wallet.publicKey
);
const destPublicKey = new anchor.web3.PublicKey(toWallet);
// Get the derived address of the destination wallet which will hold the custom token
const associatedDestinationTokenAddr = await Token.getAssociatedTokenAddress(
mintToken.associatedProgramId,
mintToken.programId,
mintPublicKey,
destPublicKey
);
const receiverAccount = await props.connection.getAccountInfo(associatedDestinationTokenAddr);
const instructions: anchor.web3.TransactionInstruction[] = [];
if (receiverAccount === null) {
instructions.push(
Token.createAssociatedTokenAccountInstruction(
mintToken.associatedProgramId,
mintToken.programId,
mintPublicKey,
associatedDestinationTokenAddr,
destPublicKey,
wallet.publicKey
)
)
}
instructions.push(
Token.createTransferInstruction(
TOKEN_PROGRAM_ID,
fromTokenAccount.address,
associatedDestinationTokenAddr,
wallet.publicKey,
[],
1
)
);
const transaction = new anchor.web3.Transaction().add(...instructions);
transaction.feePayer = wallet.publicKey;
transaction.recentBlockhash = (await props.connection.getRecentBlockhash()).blockhash;
const transactionSignature = await props.connection.sendRawTransaction(
transaction.serialize(),
{ skipPreflight: true }
);
await props.connection.confirmTransaction(transactionSignature);
Please ensure that wallet.publicKey contains valid value.
console.log(wallet.publicKey);//I think this might be an empty string.
const fromTokenAccount = await mintToken.getOrCreateAssociatedAccountInfo(
wallet.publicKey
);
How can I get the data from an api request from the then() method, so I can process the data outside
of the function. Can I store the data in a state object for example and how to do that.
// openweathermap async await api request
async function getData(){
try{
const url = `http://api.openweathermap.org/data/2.5/forecast?
q=${city}&units=metric&appid=${key}`;
const data = await axios(url)
console.log(data)
return data;
}catch(error){
temperature.insertAdjacentHTML('afterend', '<div id="error">Oooops
something went wrong!</div>');
}
}
getData().then( data => {
const temp = data.data.list[0].main.temp;
temperature.textContent = temp.toFixed(1) + ' \xB0' + "C ";
const temp2 = data.data.list[1].main.temp;
temperature2.textContent = temp2.toFixed(1) + ' \xB0' + "C ";
const forecast = data.data.list[0].weather[0].description;
foreCast.textContent = forecast.toLowerCase();
const forecast2 = data.data.list[1].weather[0].description;
foreCast2.textContent = forecast2.toLowerCase();
const icon = data.data.list[0].weather[0].icon
weatherIcon.insertAdjacentHTML('afterbegin', `<img
src="http://openweathermap.org/img/w/${icon}.png">`)
const icon2 = data.data.list[1].weather[0].icon
weatherIcon2.insertAdjacentHTML('afterbegin', `<img
src="http://openweathermap.org/img/w/${icon2}.png">`)
day.textContent = newTime;
day2.textContent = newTime2;
})
Should axios(url) be axios.get(url)?
Then you can handle the data inside the callback function:
getData().then(data => console.log(data))
Get data is already an async function which means it returns a Promise. Just assign variable data by calling getData function. This should be done in an async function.
(async () => {
const data = await getData();
const temp = data.data.list[0].main.temp;
temperature.textContent = temp.toFixed(1) + ' \xB0' + "C ";
const temp2 = data.data.list[1].main.temp;
temperature2.textContent = temp2.toFixed(1) + ' \xB0' + "C ";
const forecast = data.data.list[0].weather[0].description;
foreCast.textContent = forecast.toLowerCase();
const forecast2 = data.data.list[1].weather[0].description;
foreCast2.textContent = forecast2.toLowerCase();
const icon = data.data.list[0].weather[0].icon
weatherIcon.insertAdjacentHTML('afterbegin', `<img src="http://openweathermap.org/img/w/${icon}.png">`)
const icon2 = data.data.list[1].weather[0].icon
weatherIcon2.insertAdjacentHTML('afterbegin', `<img src="http://openweathermap.org/img/w/${icon2}.png">`)
day.textContent = newTime;
day2.textContent = newTime2;
})();
Found a solution: Create a class with the getData() method, create a function to control the data and store it in the state object.
// Create class
class Weather {
constructor(){
}
async getData(){
try{
const url = `http://api.openweathermap.org/data/2.5/forecast?
q=${city}&units=metric&appid=${key}`;
const data = await axios(url)
this.data = data
}catch(error){
temperature.insertAdjacentHTML('afterend', '<div id="error">Oooops something
went wrong!</div>');
}
}
}
// Create state object, declare function to get data and store data in state object
const state = {}
const controlData = async () => {
state.data = new Weather()
await state.data.getData()
const temp = state.data.data.data.list[0].main.temp;
temperature.textContent = temp.toFixed(1) + ' \xB0' + "C ";
const temp2 = state.data.data.data.list[1].main.temp;
temperature2.textContent = temp2.toFixed(1) + ' \xB0' + "C ";
const forecast = state.data.data.data.list[0].weather[0].description;
foreCast.textContent = forecast.toLowerCase();
const forecast2 = state.data.data.data.list[1].weather[0].description;
foreCast2.textContent = forecast2.toLowerCase();
const icon = state.data.data.data.list[0].weather[0].icon
weatherIcon.insertAdjacentHTML('afterbegin', `<img
src="http://openweathermap.org/img/w/${icon}.png">`)
const icon2 = state.data.data.data.list[1].weather[0].icon
weatherIcon2.insertAdjacentHTML('afterbegin', `<img
src="http://openweathermap.org/img/w/${icon2}.png">`)
day.textContent = newTime;
day2.textContent = newTime2;
let sRise = convertTime(state.data.data.data.city.sunrise);
let sSet = convertTime(state.data.data.data.city.sunset);
sunRise.textContent = `Sunrise ${sRise}`;
sunSet.textContent = `Sunset ${sSet}`;
}
controlData()
Hey guys this is a follow on from my other question, i have created a Nodejs Scraper that doesnt seem to want to go through the pages, it stays on the first. my source code is below
const rp = require('request-promise');
const request = require('request');
const otcsv = require('objects-to-csv');
const cheerio = require('cheerio');
//URL To scrape
const baseURL = 'xxx';
const searchURL = 'xxxx';
//scrape info
const getCompanies = async () => {
// Pagination test
for (let index = 1; index <= 20; index = index + 1) {
const html = await rp.get(baseURL + searchURL + index);
const $ = await cheerio.load(html);
console.log("Loading Pages....");
console.log("At page number " + index);
// end pagination test
//const htmls = await rp(baseURL + searchURL);
const businessMap = cheerio('a.business-name', html).map(async (i, e) => {
const link = baseURL + e.attribs.href;
const innerHtml = await rp(link);
const emailAddress = cheerio('a.email-business', innerHtml).prop('href');
const name = e.children[0].data || cheerio('h1', innerHtml).text();
const phone = cheerio('p.phone', innerHtml).text();
return {
// link,
name,
emailAddress: emailAddress ? emailAddress.replace('mailto:', '') : '',
phone,
}
}).get();
return Promise.all(businessMap);
}
};
console.log("Finished Scraping.... Now Saving!")
//save to CSV
getCompanies()
.then(result => {
const transformed = new otcsv(result);
return transformed.toDisk('./output.csv');
})
.then(() => console.log('Scrape Complete :D '));
As you can see I have tried a few different ways to make this happen so any help will be gratefully appreciated.
I am trying following code to get all records from a paginated API in Zapier.
const limitPerPage = 20;
const apiUrl = "https://myurl.com/data";
var lastCursor = null;
var output = null;
const getContent = async function (cursor) {
let actualUrl = apiUrl + `?cursor=${cursor}&limit=${limitPerPage}`;
var apiResults = await fetch(actualUrl)
.then(resp => {
return resp.json;
});
}
const getEntireContentList = async function (cursor) {
const results = await getContent(cursor);
console.log("Retreiving data from API for cursor : " + cursor);
if (results.metadata.cursor !== "") {
return results.concat(await getEntireContentList(results.metadata.cursor));
} else {
return results;
}
};
(async() => {
const entireList = await getEntireContentList();
console.log(entireList);
output = entireList;
callback(null, entireList);
})();
I get error as
You did not define output! Try output = {id: 1, hello: await Promise.resolve("world")};
How can I fix this?
Your problem is that though you're awaiting in that function, the top-level carries on and execution ends before your code has had a chance to run.
The good news is, Zapier wraps your code in an async function already, so you can use await at the top level (per these docs).
Try this instead:
const limitPerPage = 20;
const apiUrl = "https://myurl.com/data";
let lastCursor = null;
// var output = null; // zapier does this for you already
const getContent = async function (cursor) {
const actualUrl = apiUrl + `?cursor=${cursor}&limit=${limitPerPage}`;
const rawResponse = await fetch(actualUrl)
return resp.json() // async function, you had it as a property
}
const getEntireContentList = async function (cursor) {
const results = await getContent(cursor);
console.log("Retreiving data from API for cursor : " + cursor);
if (results.metadata.cursor !== "") {
return results.concat(await getEntireUserList(results.metadata.cursor)); // should this be named getEntireContentList?
} else {
return results;
}
};
return {
results: await getEntireContentList()
}
I noticed this is a recursive approach. That's fine, but remember that you've got limited execution time. You also might hit memory limits (depending on how many objects you're returning), so keep an eye on that.