HTTP Requests in Node

Dr. Greg Bernstein

Updated: April 14th, 2020

HTTP Requests in Node.js

Why non-browser HTTP requests

  • Testing
  • Proxies, multi-server architectures such as micro-services
  • Web Scraping, web site monitoring…

Request Libraries

  1. The original Request - Simplified HTTP client “Request is designed to be the simplest way possible to make http calls. It supports HTTPS and follows redirects by default.”, Callback based library, approximately 15 Million downloads a week. Full documentation. Deprecated

  2. Promise based Request-Promise built on 1. but uses promises. ~2.4 Million downloads a week.

  3. Request-Promise-Native “This package is similar to request-promise but uses native ES6+ promises”. Uses 1, documented in 2. This is what we will use in this class.

Promise Based Examples

Request Library and Promises

We’ll use Request-Promise-Native

Installation:

npm install --save request
npm install --save request-promise-native

Request Promise Documentation

Request-Promise-Native has some nice API improvements see:

Request-Promise

Example 3 in order

threeInOrder.js

const rp = require('request-promise-native');
let site1 = {
    uri: 'https://www.grotto-networking.com',
    method: 'HEAD', // What does this do?
    resolveWithFullResponse: true
};

let site2 = {
    uri: 'http://www.google.com',
    method: 'HEAD',
    resolveWithFullResponse: true
};

let site3 = {
    uri: 'http://www.kiteboardingcairns.com.au/kitesurfing-australia/',
    method: 'HEAD',
    resolveWithFullResponse: true
};

let start = new Date();
rp(site1).then(res => {
    // console.log(`Grotto status: ${JSON.stringify(res)}`);
    let time = (new Date() - start)/1000;
    console.log(`Grotto status: ${res.statusCode}, time: ${time}`);
    return rp(site2);
}).then(res => {
    let time = (new Date() - start)/1000;
    console.log(`Google status: ${res.statusCode}, time: ${time}`);
    return rp(site3);
}).then(res => {
    let time = (new Date() - start)/1000;
    console.log(`Aus kiteboarding status: ${res.statusCode}, time: ${time}`);
})
console.log("Starting my web requests:");

Making Parallel Requests

Suppose we don’t care about the order in which the information comes back we just need to know when all our requests have been satisfied?

Example Node.js/Requests

threeInParallel.js

const rp = require('request-promise-native');
let site1 = {
    uri: 'https://www.grotto-networking.com',
    method: 'HEAD', // What does this do?
    resolveWithFullResponse: true
};

let site2 = {
    uri: 'http://www.google.com',
    method: 'HEAD',
    resolveWithFullResponse: true
};

let site3 = {
    uri: 'http://www.kiteboardingcairns.com.au/kitesurfing-australia/',
    method: 'HEAD',
    resolveWithFullResponse: true
};

let start = new Date();
let p1 = rp(site1).then(res => {
    // console.log(`Grotto status: ${JSON.stringify(res)}`);
    let time = (new Date() - start)/1000;
    return console.log(`Grotto status: ${res.statusCode}, time: ${time}`);});

let p2 = rp(site2).then(res => {
    let time = (new Date() - start)/1000;
    return console.log(`Google status: ${res.statusCode}, time: ${time}`);
});

let p3 = rp(site3).then(res => {
    let time = (new Date() - start)/1000;
    return console.log(`Aus kiteboarding status: ${res.statusCode}, time: ${time}`);
});

console.log("Starting my web requests:");
Promise.all([p1, p2, p3]).then(x=>{
    console.log("All Finished");
});

First Answer

Suppose we make a set of parallel requests but only want to wait till we get the first answer?

Racing Requests

threeInRace.js

const rp = require('request-promise-native');
let site1 = {
    uri: 'https://www.grotto-networking.com',
    method: 'HEAD', // What does this do?
    resolveWithFullResponse: true
};

let site2 = {
    uri: 'http://www.google.com',
    method: 'HEAD',
    resolveWithFullResponse: true
};

let site3 = {
    uri: 'http://www.kiteboardingcairns.com.au/kitesurfing-australia/',
    method: 'HEAD',
    resolveWithFullResponse: true
};

let start = new Date();
let p1 = rp(site1).then(res => {
    // console.log(`Grotto status: ${JSON.stringify(res)}`);
    let time = (new Date() - start)/1000;
    return console.log(`Grotto status: ${res.statusCode}, time: ${time}`);});

let p2 = rp(site2).then(res => {
    let time = (new Date() - start)/1000;
    return console.log(`Google status: ${res.statusCode}, time: ${time}`);
});

let p3 = rp(site3).then(res => {
    let time = (new Date() - start)/1000;
    return console.log(`Aus kiteboarding status: ${res.statusCode}, time: ${time}`);
});

console.log("Starting my web requests:");
Promise.race([p1, p2, p3]).then(x=>{
    console.log("was the winner! \n The rest: ");
});

Web requests in order async/await

From threeInOrderAsync.js in NodeRequests.zip

const rp = require('request-promise-native');
let site1 = {
    uri: 'https://www.grotto-networking.com',
    method: 'HEAD', // What does this do?
    resolveWithFullResponse: true
};

let site2 = {
    uri: 'http://www.google.com',
    method: 'HEAD',
    resolveWithFullResponse: true
};

let site3 = {
    uri: 'http://www.kiteboardingcairns.com.au/kitesurfing-australia/',
    method: 'HEAD',
    resolveWithFullResponse: true
};

let start = new Date();
async function inOrder() {
    let res = await rp(site1);
    let time = (new Date() - start)/1000;
    console.log(`Grotto status: ${res.statusCode}, time: ${time}`);
    res = await rp(site2);
    time = (new Date() - start)/1000;
    console.log(`Google status: ${res.statusCode}, time: ${time}`);
    res = await rp(site3);
    time = (new Date() - start)/1000;
    console.log(`Aus kiteboarding status: ${res.statusCode}, time: ${time}`);
}
console.log("Starting my web requests:");
inOrder();

Cookies and Such

Why Care About Cookies

  • Testing proper cookie use
  • Testing interfaces that require a “login”

Documentation

  • Request cookies are explained at the very end of the request documentation

  • Note that they use tough-cookie to store cookies but their interface is easier and slightly different.

Inspecting Cookies

Enhanced version of documentation example

let j = request.jar(); // Create cookiejar to save cookies in
let options = {url: 'http://www.google.com', jar: j}; // use it here
request(options), function () {
  // Two different ways to look at cookies in the jar j, use one of these
  var cookie_string = j.getCookieString(url); //Use this just to see
  // cookie name and value
  // "key1=value1; key2=value2; ..."
  var cookies = j.getCookies(url); // Use this to get all cookie info
  // [{key: 'key1', value: 'value1', domain: "www.google.com", ...}, ...]
})