puppeteer#HTTPResponse TypeScript Examples
The following examples show how to use
puppeteer#HTTPResponse.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
| Source File: platform.ts From cli with Apache License 2.0 | 6 votes |
|
export function isSearchRequestOrResponse(
requestOrResponse: HTTPRequest | HTTPResponse
) {
const searchUrl = new URL(
'/rest/search/v2?organizationId',
process.env.PLATFORM_HOST
);
return requestOrResponse.url().startsWith(searchUrl.href);
}
Example #2
| Source File: solver.ts From FlareSolverr with MIT License | 5 votes |
|
async function gotoPage(params: V1Request, page: Page): Promise<HTTPResponse> {
let pageTimeout = params.maxTimeout / 3;
let response: HTTPResponse
try {
response = await page.goto(params.url, {waitUntil: 'domcontentloaded', timeout: pageTimeout});
} catch (e) {
// retry
response = await page.goto(params.url, {waitUntil: 'domcontentloaded', timeout: pageTimeout});
}
if (params.method == 'POST') {
// post hack
await page.setContent(
`
<!DOCTYPE html>
<html>
<body>
<script>
function parseQuery(queryString) {
var query = {};
var pairs = (queryString[0] === '?' ? queryString.substr(1) : queryString).split('&');
for (var i = 0; i < pairs.length; i++) {
var pair = pairs[i].split('=');
query[decodeURIComponent(pair[0])] = decodeURIComponent(pair[1] || '');
}
return query;
}
const form = document.createElement('form');
form.method = 'POST';
form.action = '${params.url}';
const params = parseQuery('${params.postData}');
for (const key in params) {
if (params.hasOwnProperty(key)) {
const hiddenField = document.createElement('input');
hiddenField.type = 'hidden';
hiddenField.name = key;
hiddenField.value = params[key];
form.appendChild(hiddenField);
}
}
document.body.appendChild(form);
form.submit();
</script>
</body>
</html>
`
);
await page.waitForTimeout(2000)
try {
await page.waitForNavigation({waitUntil: 'domcontentloaded', timeout: 2000})
} catch (e) {}
}
return response
}
Example #3
| Source File: cloudflare.ts From FlareSolverr with MIT License | 4 votes |
|
export default async function resolveChallenge(url: string, page: Page, response: HTTPResponse): Promise<HTTPResponse> {
// look for challenge and return fast if not detected
let cfDetected = response.headers().server &&
(response.headers().server.startsWith('cloudflare') || response.headers().server.startsWith('ddos-guard'));
if (cfDetected) {
if (response.status() == 403 || response.status() == 503) {
cfDetected = true; // Defected CloudFlare and DDoS-GUARD
} else if (response.headers().vary && response.headers().vary.trim() == 'Accept-Encoding,User-Agent' &&
response.headers()['content-encoding'] && response.headers()['content-encoding'].trim() == 'br') {
cfDetected = true; // Detected Custom CloudFlare for EbookParadijs, Film-Paleis, MuziekFabriek and Puur-Hollands
} else {
cfDetected = false;
}
}
if (cfDetected) {
log.info('Cloudflare detected');
} else {
log.info('Cloudflare not detected');
return response;
}
if (await findAnySelector(page, BAN_SELECTORS)) {
throw new Error('Cloudflare has blocked this request. Probably your IP is banned for this site, check in your web browser.');
}
// find Cloudflare selectors
let selectorFound = false;
let selector: string = await findAnySelector(page, CHALLENGE_SELECTORS)
if (selector) {
selectorFound = true;
log.debug(`Javascript challenge element '${selector}' detected.`)
log.debug('Waiting for Cloudflare challenge...')
while (true) {
try {
selector = await findAnySelector(page, CHALLENGE_SELECTORS)
if (!selector) {
// solved!
log.debug('Challenge element not found')
break
} else {
log.debug(`Javascript challenge element '${selector}' detected.`)
// new Cloudflare Challenge #cf-please-wait
const displayStyle = await page.evaluate((selector) => {
return getComputedStyle(document.querySelector(selector)).getPropertyValue("display");
}, selector);
if (displayStyle == "none") {
// spinner is hidden, could be a captcha or not
log.debug('Challenge element is hidden')
// wait until redirecting disappears
while (true) {
try {
await page.waitForTimeout(1000)
const displayStyle2 = await page.evaluate(() => {
return getComputedStyle(document.querySelector('#cf-spinner-redirecting')).getPropertyValue("display");
});
if (displayStyle2 == "none") {
break // hCaptcha detected
}
} catch (error) {
break // redirection completed
}
}
break
} else {
log.debug('Challenge element is visible')
}
}
log.debug('Found challenge element again')
} catch (error)
{
log.debug("Unexpected error: " + error);
if (!error.toString().includes("Execution context was destroyed")) {
break
}
}
log.debug('Waiting for Cloudflare challenge...')
await page.waitForTimeout(1000)
}
log.debug('Validating HTML code...')
} else {
log.debug(`No challenge element detected.`)
}
// check for CAPTCHA challenge
if (await findAnySelector(page, CAPTCHA_SELECTORS)) {
log.info('CAPTCHA challenge detected');
throw new Error('FlareSolverr can not resolve CAPTCHA challenges. Since the captcha doesn\'t always appear, you may have better luck with the next request.');
// const captchaSolver = getCaptchaSolver()
// if (captchaSolver) {
// // to-do: get the params
// log.info('Waiting to receive captcha token to bypass challenge...')
// const token = await captchaSolver({
// url,
// sitekey,
// type: captchaType
// })
// log.debug(`Token received: ${token}`);
// // to-do: send the token
// }
// } else {
// throw new Error('Captcha detected but no automatic solver is configured.');
// }
} else {
if (!selectorFound)
{
throw new Error('No challenge selectors found, unable to proceed.')
} else {
log.info('Challenge solved');
}
}
return response;
}
Example #4
| Source File: solver.ts From FlareSolverr with MIT License | 4 votes |
|
async function resolveChallenge(params: V1Request, session: SessionsCacheItem): Promise<ChallengeResolutionT | void> {
try {
let status = 'ok'
let message = ''
const page: Page = await session.browser.newPage()
// the Puppeter timeout should be half the maxTimeout because we reload the page and wait for challenge
// the user can set a really high maxTimeout if he wants to
await page.setDefaultNavigationTimeout(params.maxTimeout / 2)
// the user-agent is changed just for linux arm build
await page.setUserAgent(sessions.getUserAgent())
// set the proxy
if (params.proxy) {
log.debug(`Using proxy: ${params.proxy.url}`);
// todo: credentials are not working
// if (params.proxy.username) {
// await page.authenticate({
// username: params.proxy.username,
// password: params.proxy.password
// });
// }
}
// go to the page
log.debug(`Navigating to... ${params.url}`)
let response: HTTPResponse = await gotoPage(params, page);
// set cookies
if (params.cookies) {
for (const cookie of params.cookies) {
// the other fields in the cookie can cause issues
await page.setCookie({
"name": cookie.name,
"value": cookie.value
})
}
// reload the page
response = await gotoPage(params, page);
}
// log html in debug mode
log.html(await page.content())
// detect protection services and solve challenges
try {
response = await cloudflareProvider(params.url, page, response);
// is response is ok
// reload the page to be sure we get the real page
log.debug("Reloading the page")
try {
response = await gotoPage(params, page);
} catch (e) {
log.warn("Page not reloaded (do not report!): Cause: " + e.toString())
}
} catch (e) {
status = "error";
message = "Cloudflare " + e.toString();
}
const payload: ChallengeResolutionT = {
status,
message,
result: {
url: page.url(),
status: response.status(),
headers: response.headers(),
response: null,
cookies: await page.cookies(),
userAgent: sessions.getUserAgent()
}
}
if (params.returnOnlyCookies) {
payload.result.headers = null;
payload.result.userAgent = null;
} else {
payload.result.response = await page.content()
}
// make sure the page is closed because if it isn't and error will be thrown
// when a user uses a temporary session, the browser make be quit before
// the page is properly closed.
await page.close()
return payload
} catch (e) {
log.error("Unexpected error: " + e);
throw e;
}
}
Example #5
| Source File: browser.test.ts From web with MIT License | 4 votes |
|
describe('browser tests', function () {
this.timeout(10000);
let browser: Browser;
before(async () => {
browser = await launchPuppeteer();
});
after(async () => {
await browser.close();
});
it('should bubble when bubbles is true', async () => {
const { server, host } = await createTestServer({
rootDir: __dirname,
plugins: [
mockFiles({
'/foo.html': '<script src="/foo.js" type="module"></script>',
'/foo.js': `import '/bar.js'; import.meta.hot.accept();`,
'/bar.js': `import.meta.hot.accept({ bubbles: true })`,
}),
hmrPlugin(),
],
});
const { fileWatcher, webSockets } = server;
const stub = stubMethod(webSockets, 'send');
const page = await browser.newPage();
try {
await page.goto(`${host}/foo.html`, { waitUntil: 'networkidle0' });
fileWatcher.emit('change', pathUtil.join(__dirname, '/bar.js'));
expect(stub.callCount).to.equal(2);
expect(stub.getCall(0)!.args[0]).to.equal(
JSON.stringify({
type: 'hmr:update',
url: '/bar.js',
}),
);
expect(stub.getCall(1)!.args[0]).to.equal(
JSON.stringify({
type: 'hmr:update',
url: '/foo.js',
}),
);
} finally {
await page.close();
await server.stop();
}
});
it('should hot replace a module', async () => {
const files = {
'/foo.html': '<script src="/foo.js" type="module"></script>',
'/foo.js':
'import.meta.hot.accept(); document.body.appendChild(document.createTextNode(" a "));',
};
const { server, host } = await createTestServer({
rootDir: __dirname,
plugins: [mockFiles(files), hmrPlugin()],
});
const page = await browser.newPage();
const errors = trackErrors(page);
try {
await page.goto(`${host}/foo.html`, { waitUntil: 'networkidle0' });
expectIncludes(await page.content(), '<body> a </body>');
files['/foo.js'] = files['/foo.js'].replace('" a "', '" b "');
server.fileWatcher.emit('change', pathUtil.join(__dirname, '/foo.js'));
await page.waitForResponse((r: HTTPResponse) => r.url().startsWith(`${host}/foo.js`));
expectIncludes(await page.content(), '<body> a b </body>');
for (const error of errors) {
throw error;
}
} finally {
await page.close();
await server.stop();
}
});
it('should hot replace a bubbled module', async () => {
const files = {
'/foo.html': '<script src="/foo.js" type="module"></script>',
'/foo.js':
'import bar from "/bar.js"; import.meta.hot.accept(); document.body.appendChild(document.createTextNode(bar));',
'/bar.js': 'export default " a ";',
};
const { server, host } = await createTestServer({
rootDir: __dirname,
plugins: [mockFiles(files), hmrPlugin()],
});
const page = await browser.newPage();
const errors = trackErrors(page);
try {
await page.goto(`${host}/foo.html`, { waitUntil: 'networkidle0' });
expectIncludes(await page.content(), '<body> a </body>');
files['/bar.js'] = 'export default " b ";';
server.fileWatcher.emit('change', pathUtil.join(__dirname, '/bar.js'));
await page.waitForResponse((r: HTTPResponse) => r.url().startsWith(`${host}/bar.js`));
await new Promise(r => setTimeout(r, 1000));
expectIncludes(await page.content(), '<body> a b </body>');
for (const error of errors) {
throw error;
}
} finally {
await page.close();
await server.stop();
}
});
it('hot replaces multiple bubbled modules', async () => {
const files = {
'/foo.html': '<script type="module">import "/foo.js"; import "/bar.js";</script>',
'/foo.js':
'import baz from "/baz.js"; import.meta.hot.accept(); document.body.appendChild(document.createTextNode(" foo " + baz));',
'/bar.js':
'import baz from "/baz.js"; import.meta.hot.accept(); document.body.appendChild(document.createTextNode(" bar " + baz));',
'/baz.js': 'export default " a ";',
};
const { server, host } = await createTestServer({
rootDir: __dirname,
plugins: [mockFiles(files), hmrPlugin()],
});
const page = await browser.newPage();
const errors = trackErrors(page);
try {
await page.goto(`${host}/foo.html`, { waitUntil: 'networkidle0' });
// expectIncludes(await page.content(), '<body> foo a bar a </body>');
files['/baz.js'] = 'export default " b ";';
server.fileWatcher.emit('change', pathUtil.join(__dirname, '/baz.js'));
await Promise.all([
page.waitForResponse((r: HTTPResponse) => r.url().startsWith(`${host}/foo.js`)),
page.waitForResponse((r: HTTPResponse) => r.url().startsWith(`${host}/bar.js`)),
page.waitForResponse((r: HTTPResponse) => r.url().startsWith(`${host}/baz.js`)),
]);
await new Promise(r => setTimeout(r, 1000));
expectIncludes(await page.content(), '<body> foo a bar a foo b bar b </body>');
for (const error of errors) {
throw error;
}
} finally {
await page.close();
await server.stop();
}
});
it('reloads the page when a module has no hot replacable parent', async () => {
const files = {
'/foo.html':
'<script src="/foo.js" type="module"></script><script src="/baz.js" type="module"></script>',
'/foo.js':
'import bar from "/bar.js"; import.meta.hot.accept(); document.body.appendChild(document.createTextNode(bar));',
'/bar.js': 'export default " a ";',
'/baz.js': 'document.body.appendChild(document.createTextNode(" b "));',
};
const { server, host } = await createTestServer({
rootDir: __dirname,
plugins: [mockFiles(files), hmrPlugin()],
});
const page = await browser.newPage();
const errors = trackErrors(page);
try {
await page.goto(`${host}/foo.html`, { waitUntil: 'networkidle0' });
await page.evaluate('document.body.appendChild(document.createTextNode(" c "))');
expectIncludes(await page.content(), '<body> a b c </body>');
server.fileWatcher.emit('change', pathUtil.join(__dirname, '/baz.js'));
await page.waitForNavigation({ waitUntil: 'networkidle0' });
expectIncludes(await page.content(), '<body> a b </body>');
for (const error of errors) {
throw error;
}
} finally {
await page.close();
await server.stop();
}
});
});
