Wappalyzer: Inconsistencies between Chrome extension and npm package

Created on 17 Aug 2018 · 16Comments · Source: AliasIO/wappalyzer

I found that for some websites the npm package produces different results than the Chrome extension. Specifically, the npm package is unable to detect Google Analytics while the Chrome extension is.

I'm using the 5.5.3 in Chrome and 5.5.1 for the npm package. I also tried different options for the package but nothing really helped.

Steps to reproduce

Visit http://runifico.com/ or http://www.der-autotester.de/
Check the results of the Chrome extension (it detects Google Analytics)
Run the following JavaScript snippet and check the results:

const options = {
  debug: true,
  delay: 0,
  maxDepth: 0,
  maxUrls: 10,
  maxWait: 5000,
  recursive: true,
  htmlMaxCols: Number.MAX_SAFE_INTEGER,
  htmlMaxRows: Number.MAX_SAFE_INTEGER,
  userAgent: 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36'
};

const wappalyzer = new Wappalyzer('http://runifico.com/', options);

wappalyzer.analyze().then(console.log);

The results will not contain Google Analytics

{
  "urls": {
    "http://runifico.com/": {
      "status": 200
    }
  },
  "applications": [
    {
      "name": "Apache",
      "confidence": "100",
      "version": "2.4.34",
      "icon": "Apache.svg",
      "website": "http://apache.org",
      "categories": [
        {
          "22": "Web Servers"
        }
      ]
    },
    {
      "name": "Font Awesome",
      "confidence": "100",
      "version": null,
      "icon": "Font Awesome.png",
      "website": "http://fontawesome.io",
      "categories": [
        {
          "17": "Font Scripts"
        }
      ]
    },
    {
      "name": "Google AdSense",
      "confidence": "100",
      "version": null,
      "icon": "Google AdSense.svg",
      "website": "https://www.google.fr/adsense/start/",
      "categories": [
        {
          "36": "Advertising Networks"
        }
      ]
    },
    {
      "name": "Google Font API",
      "confidence": "100",
      "version": null,
      "icon": "Google Font API.png",
      "website": "http://google.com/fonts",
      "categories": [
        {
          "17": "Font Scripts"
        }
      ]
    },
    {
      "name": "PHP",
      "confidence": "100",
      "version": "5.6.37",
      "icon": "PHP.svg",
      "website": "http://php.net",
      "categories": [
        {
          "27": "Programming Languages"
        }
      ]
    },
    {
      "name": "UNIX",
      "confidence": "100",
      "version": null,
      "icon": "UNIX.png",
      "website": "http://unix.org",
      "categories": [
        {
          "28": "Operating Systems"
        }
      ]
    },
    {
      "name": "WordPress",
      "confidence": "100",
      "version": " 4.9.6",
      "icon": "WordPress.svg",
      "website": "http://wordpress.org",
      "categories": [
        {
          "1": "CMS"
        },
        {
          "11": "Blogs"
        }
      ]
    },
    {
      "name": "Yoast SEO",
      "confidence": "100",
      "version": "4.1",
      "icon": "Yoast SEO.png",
      "website": "http://yoast.com",
      "categories": [
        {
          "54": "SEO"
        }
      ]
    },
    {
      "name": "jQuery",
      "confidence": "100",
      "version": "1.12.4",
      "icon": "jQuery.svg",
      "website": "https://jquery.com",
      "categories": [
        {
          "12": "JavaScript Frameworks"
        }
      ]
    },
    {
      "name": "jQuery Migrate",
      "confidence": "100",
      "version": "1.4.1",
      "icon": "jQuery.svg",
      "website": "https://github.com/jquery/jquery-migrate",
      "categories": [
        {
          "12": "JavaScript Frameworks"
        }
      ]
    },
    {
      "name": "prettyPhoto",
      "confidence": "100",
      "version": null,
      "icon": "prettyPhoto.png",
      "website": "http://no-margin-for-errors.com/projects/prettyphoto-jquery-lightbox-clone/",
      "categories": [
        {
          "12": "JavaScript Frameworks"
        }
      ]
    },
    {
      "name": "MySQL",
      "confidence": "0",
      "version": null,
      "icon": "MySQL.svg",
      "website": "http://mysql.com",
      "categories": [
        {
          "34": "Databases"
        }
      ]
    }
  ],
  "meta": {
    "language": "de-DE"
  }
}

Am I doing something wrong? Is there a way to detect Google Analytics for those websites using a different configuration? Is this related to Zombie?

Source

cangoektas

👍2

Most helpful comment

The https://github.com/AliasIO/Wappalyzer/pull/2601 allows using the following code snippet to analyze better with Puppeteer.

const Wappalyzer = require('wappalyzer');

/**
 * @var {Puppeteer.Page} page
 * @var {Puppeteer.Response} response
 */
const { apps, origPageUrl, wappalyzer } = new Wappalyzer('https://google.com');
const headers = response.headers();
const cookies = await page.cookies();
const data = await page.evaluate((processJs, patterns) => ({
  js: Function(`return ${processJs}`)()(window, patterns),
  html: document.documentElement.innerHTML,
  scripts: Array.prototype.slice
    .apply(document.scripts)
    .filter(script => script.src)
    .map(script => script.src)
    .filter(script => script.indexOf('data:text/javascript;') !== 0),
}), Wappalyzer.processJs.toString(), wappalyzer.jsPatterns);

for (const header in headers) {
  headers[header] = [headers[header]];
}

await wappalyzer.analyze(origPageUrl, {
  ...data,
  headers,
  cookies,
});

console.log(apps);

BR0kEN- on 18 Jan 2019

👍4

All 16 comments

I tried to debug this and while inserting a bunch of console.log's I noticed that sometimes, the npm package does detect the Google Analytics snippet. Sometimes it does, sometimes it doesn't. I guess this is then also related to #2375.

I noticed that the integration status here is detected through the presence of global variable names. Sometimes we'll find the GoogleAnalyticsObject for those websites in the window object, sometimes we won't. Are we looking into that too early maybe?

cangoektas on 17 Aug 2018

I am also seeing an inconsistency between the Chrome extension and npm package when passing scripts to wappalyzer's analyze() method. Specifically:

Using, 5.5.1 in a local app, I grab script from a page:

import Wappalyzer from 'wappalyzer';

const href = 'https://www.nba.com/cavaliers/'; // The page to scrape

// Load href in Puppeteer (code omitted for simplicity)...
// Grab scripts like the extension: https://github.com/AliasIO/Wappalyzer/blob/c9b0ab7aa2db7ca720014a5afb096334febebcb0/src/drivers/webextension/js/content.js#L17
const scripts = Array.prototype.slice.apply(document.scripts).filter(script => script.src).map(script => script.src).filter(script => script.indexOf('data:text/javascript;') !== 0);
const html = document.documentElement.innerHTML;

// Back in node, pass the data from Puppeteer to wappalyzer...

const wappalyzer = new Wappalyzer(href);

const { applications } = await wappalyzer.analyze(href, {
  html,
  headers: response.headers(), // response is a Puppeteer response object
  cookies: page.cookies(), //  page is a Puppeteer page object
  scripts,
});

Using https://www.nba.com/cavaliers/ as an example, I have confirmed that scripts contains a URL matching the DoubleClick for Publishers script. Scripts looks something like this:

[
  'https://www.googleadservices.com/pagead/conversion_async.js',
  'https://www.googletagservices.com/tag/js/gpt.js', // Should match https://github.com/AliasIO/Wappalyzer/blob/master/src/apps.json#L2629
  'https://www.googletagmanager.com/gtm.js?id=GTM-P8VS57',
  etc...
]

However, applications does not contain DoubleClick for Publishers.

So, excluding any incompatibility using headers and cookies from Puppeteer may be introducing, scripts seems to be failing to correctly match strings (or do so before applications is returned).

Other applications missing from the npm analysis of https://www.nba.com/cavaliers/ application are:

Google Analytics
Parse.ly
SiteCatalyst
Handlebars
Akamai
Google Tag Manager
Font Awesome
Google Font API
Ensighten
Modernizr
jQuery
jQuery UI

These are all detected by the Chrome extension.

Applications successfully detected by the npm package are:

Bootstrap
Drupal
Nginx
PHP

sir-dunxalot on 5 Oct 2018

@AliasIO Any tips for a workaround to the inconsistencies noted here?

sir-dunxalot on 10 Oct 2018

I have developed a workaround that avoids using zombie to crawl the page and, instead, uses page data scraped from Puppeteer:

import Wappalyzer from 'wappalyzer';

const href = 'https://www.nba.com/cavaliers/'; // The page to scrape

// Load href in Puppeteer (code omitted for simplicity)...
// Grab scripts like the extension: https://github.com/AliasIO/Wappalyzer/blob/c9b0ab7aa2db7ca720014a5afb096334febebcb0/src/drivers/webextension/js/content.js#L17
const scripts = Array.prototype.slice.apply(document.scripts).filter(script => script.src).map(script => script.src).filter(script => script.indexOf('data:text/javascript;') !== 0);
const html = document.documentElement.innerHTML;

// Back in node, pass the data from Puppeteer to wappalyzer...

const cookies = await page.cookies(); // page is a Puppeteer page object
const wappalyzerDriver = new Wappalyzer(href, {
  userAgent: 'Whatever UA you want to use',
});

const headers = response.headers(); // response is a Puppeteer response object

/* Replicate https://github.com/AliasIO/Wappalyzer/blob/master/src/drivers/npm/driver.js#L31 */

for (let header in headers) {
  headers[header] = [headers[header]];
}

/* To avoid crawling the site again, which Wappalyzer does with Zombie,
we don't use Wappalyzer's default usage instructions. Instead, we work
directly with the Wappalyzer driver's `wappalyzer` object to anlayze
pre-crawled page content. In some cases, we have to prep the page data
using the Wappalyzer driver's helper methods (e.g. getJs()) */

const { wappalyzer } = wappalyzerDriver;

await wappalyzer.analyze(href, {
  html,
  headers,
  cookies,
  js: wappalyzerDriver.getJs({ // If you don't have access to window, omit js option
    window,
   }),
  scripts,
});

const { apps } = wappalyzerDriver;

return apps;

With this setup, where Wappalyzer is not used for crawling and fetching, the following apps are detected and returned:

Bootstrap
DoubleClick for Publishers
Drupal
Facebook
Font Awesome
Google AdSense
Google Analytics
Google Font API
Google Plus
Google Tag Manager
Handlebars
Nginx
PHP

That's more comprehensive than the list returned by the NPM driver:

Bootstrap
Drupal
Nginx
PHP

Part of the problem for me was understanding the difference between the NPM driver and the Wappalyzer source.

In either case, the NPM driver was not returning a full list of technology detections vs the Chrome extension.

sir-dunxalot on 10 Oct 2018

👍4

Hi @sir-dunxalot

Are you able to provide a PR that supports the new architecture @AliasIO has provided? Fx. see this: https://github.com/AliasIO/Wappalyzer/pull/2560#issuecomment-445430097

anotherdevs on 17 Jan 2019

The https://github.com/AliasIO/Wappalyzer/pull/2601 allows using the following code snippet to analyze better with Puppeteer.

const Wappalyzer = require('wappalyzer');

/**
 * @var {Puppeteer.Page} page
 * @var {Puppeteer.Response} response
 */
const { apps, origPageUrl, wappalyzer } = new Wappalyzer('https://google.com');
const headers = response.headers();
const cookies = await page.cookies();
const data = await page.evaluate((processJs, patterns) => ({
  js: Function(`return ${processJs}`)()(window, patterns),
  html: document.documentElement.innerHTML,
  scripts: Array.prototype.slice
    .apply(document.scripts)
    .filter(script => script.src)
    .map(script => script.src)
    .filter(script => script.indexOf('data:text/javascript;') !== 0),
}), Wappalyzer.processJs.toString(), wappalyzer.jsPatterns);

for (const header in headers) {
  headers[header] = [headers[header]];
}

await wappalyzer.analyze(origPageUrl, {
  ...data,
  headers,
  cookies,
});

console.log(apps);

BR0kEN- on 18 Jan 2019

👍4

@BR0kEN- Can you share a working example using wappalyzer with puppeteer?

ThomasLohner on 29 Jul 2019

@ThomasLohner, do you mean the above one is no longer valid? I use Wappalyzer 5.8.2 with Puppeteer 1.15.0 and it works ok.

BR0kEN- on 29 Jul 2019

AFAICS, brand new 5.8.3 does not differ from the previous version drastically so https://github.com/AliasIO/Wappalyzer/issues/2421#issuecomment-455586546 should work.

BR0kEN- on 29 Jul 2019

@BR0kEN- I tried 5.8.3 and got:

(node:5824) UnhandledPromiseRejectionWarning: TypeError [ERR_INVALID_ARG_TYPE]: The "url" argument must be of type string. Received type object
    at validateString (internal/validators.js:107:11)
    at Url.parse (url.js:155:3)
    at Object.urlParse [as parse] (url.js:150:13)
    at new Driver (/Users/t.lohner/work/wappalyzer/node_modules/wappalyzer/driver.js:94:28)
    at new Wappalyzer (/Users/t.lohner/work/wappalyzer/node_modules/wappalyzer/index.js:9:12)
    at /Users/t.lohner/work/wappalyzer/index.js:8:43
    at Object.<anonymous> (/Users/t.lohner/work/wappalyzer/index.js:33:3)
    at Module._compile (internal/modules/cjs/loader.js:776:30)
    at Object.Module._extensions..js (internal/modules/cjs/loader.js:787:10)
    at Module.load (internal/modules/cjs/loader.js:643:32)
    at Function.Module._load (internal/modules/cjs/loader.js:556:12)
    at Function.Module.runMain (internal/modules/cjs/loader.js:839:10)
    at internal/main/run_main_module.js:17:11
(node:5824) UnhandledPromiseRejectionWarning: Unhandled promise rejection. This error originated either by throwing inside of an async function without a catch block, or by rejecting a promise which was not handled with .catch(). (rejection id: 1)
(node:5824) [DEP0018] DeprecationWarning: Unhandled promise rejections are deprecated. In the future, promise rejections that are not handled will terminate the Node.js process with a non-zero exit code.

That's because of the first param null in Wappalyzer(). I didn't get it working, but maybe i'm just too stupid ^^ Can you show me an example including the part setting Pupeteer page and response?

ThomasLohner on 29 Jul 2019

Ah, well, I see. The thing is that the first parameter of Wappalyzer constructor must be a string while you pass (probably taken from the example I posted) an object (null). Simply change the new Wappalyzer(null, 'https://google.com'); to new Wappalyzer('https://google.com');.

BR0kEN- on 29 Jul 2019

Never again call yourself stupid, mate! In my architecture, it's a bit complex to show but I'll try to simplify.

(async () => {
  const browser = await puppeteer.launch({
    ignoreHTTPSErrors: true,
    /** @link https://github.com/alixaxel/chrome-aws-lambda/blob/master/source/index.js */
    args: [
      '--incognito',
      '--no-sandbox',
      '--disable-gpu',
      '--disable-setuid-sandbox',
      '--ignore-certificate-errors',
      '--ignore-certificate-errors-spki-list',
      '--enable-features=NetworkService',
      '--disable-features=site-per-process',
      /** @link https://github.com/GoogleChrome/puppeteer/issues/2391#issuecomment-382055246 */
      '--proxy-server="direct://',
      '--proxy-bypass-list=*',
      '--disable-background-timer-throttling',
      '--disable-breakpad',
      '--disable-client-side-phishing-detection',
      '--disable-cloud-import',
      '--disable-default-apps',
      '--disable-dev-shm-usage',
      '--disable-extensions',
      '--disable-gesture-typing',
      '--disable-hang-monitor',
      '--disable-infobars',
      '--disable-notifications',
      '--disable-offer-store-unmasked-wallet-cards',
      '--disable-offer-upload-credit-cards',
      '--disable-popup-blocking',
      '--disable-print-preview',
      '--disable-prompt-on-repost',
      '--disable-setuid-sandbox',
      '--disable-speech-api',
      '--disable-sync',
      '--disable-tab-for-desktop-share',
      '--disable-translate',
      '--disable-voice-input',
      '--disable-wake-on-wifi',
      '--enable-async-dns',
      '--enable-simple-cache-backend',
      '--enable-tcp-fast-open',
      '--enable-webgl',
      '--hide-scrollbars',
      '--media-cache-size=33554432',
      '--metrics-recording-only',
      '--mute-audio',
      '--no-default-browser-check',
      '--no-first-run',
      '--no-pings',
      '--no-zygote',
      '--password-store=basic',
      '--prerender-from-omnibox=disabled',
      '--use-gl=swiftshader',
      '--use-mock-keychain',
    ],
  });

  const page = await browser.newPage();
  const response = await page.goto('https://google.com');
})();

BR0kEN- on 29 Jul 2019

I thought of leaving null out and tried this very simple approach:

const browser = await puppeteer.launch();
const page = await browser.newPage();
const response = await page.goto('https://google.com');

But now i get:

(node:6080) UnhandledPromiseRejectionWarning: TypeError: Cannot read property 'toString' of undefined
    at /Users/t.lohner/work/wappalyzer/index.js:76:26
    at processTicksAndRejections (internal/process/task_queues.js:85:5)
(node:6080) UnhandledPromiseRejectionWarning: Unhandled promise rejection. This error originated either by throwing inside of an async function without a catch block, or by rejecting a promise which was not handled with .catch(). (rejection id: 1)
(node:6080) [DEP0018] DeprecationWarning: Unhandled promise rejections are deprecated. In the future, promise rejections that are not handled will terminate the Node.js process with a non-zero exit code.

I see a response, what seems to break is: Wappalyzer.processJs.toString().

package.json:

{
  "name": "sc_wappalyzer",
  "version": "1.0.0",
  "description": "",
  "main": "index.js",
  "scripts": {
  },
  "author": "",
  "license": "ISC",
  "dependencies": {
    "puppeteer": "^1.19.0",
    "wappalyzer": "^5.8.3"
  }
}

ThomasLohner on 29 Jul 2019

👍1

@ThomasLohner
Any workaround on the Wappalyzer.processJs.toString() error ?