Skip to content

Instantly share code, notes, and snippets.

@montasim

montasim/userAgents.js

Created January 28, 2025 08:23
Show Gist options
  • Save montasim/f3d70cf9879b22e29bab3d3918ee7f53 to your computer and use it in GitHub Desktop.
Save montasim/f3d70cf9879b22e29bab3d3918ee7f53 to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. montasim created this gist Jan 28, 2025.
    118 changes: 118 additions & 0 deletions userAgents.js
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,118 @@
    /**
    * @fileoverview Defines a list of known user agent strings.
    * This list includes user agents for API testing tools, bots, crawlers, HTTP libraries, automation tools, and scrapers.
    * It can be used to identify and filter traffic from these sources.
    *
    * @module vendor/userAgents
    * @version 1.0.0
    * @license CC BY-NC-ND 4.0
    *
    * @contact Mohammad Montasim-Al-Mamun Shuvo
    * @created 2025-01-28
    * @contactEmail [email protected]
    * @contactGithub https://github.com/montasim
    */

    /**
    * A list of known user agent substrings.
    * This array contains substrings that are commonly found in the user-agent strings
    * of API testing tools, bots, crawlers, HTTP libraries, automation tools, and scrapers.
    * Use this list to identify or block unwanted traffic.
    *
    * Example usage:
    * ```javascript
    * const isBlockedUserAgent = userAgents.some(agent => userAgent.includes(agent));
    * if (isBlockedUserAgent) {
    * // Handle blocked user agents (e.g., deny access or log)
    * }
    * ```
    *
    * @type {string[]}
    */
    const userAgents = [
    /** User agents for popular API testing tools. */

    'PostmanRuntime/',
    'Insomnia',
    'RapidAPI-Client',
    'Paw/',
    'curl/',
    'HTTPie/',
    'Thunder Client',
    'AdvancedRestClient',
    'RestSharp',
    'Swagger-UI',
    'k6',
    'ApacheBench',
    'SoapUI',
    'Apache JMeter',
    'Fiddler',
    'Wget',

    /** User agents for well-known bots and crawlers. */
    'Python-urllib',
    'Java/',
    'Googlebot',
    'Bingbot',
    'Yahoo! Slurp',
    'DuckDuckBot',
    'Baiduspider',
    'YandexBot',
    'Sogou',
    'Exabot',
    'facebot',
    'ia_archiver',
    'MJ12bot',
    'AhrefsBot',
    'SemrushBot',
    'DotBot',
    'SerpstatBot',
    'Mediapartners-Google',

    /** User agents for HTTP libraries commonly used for making programmatic requests. */
    'libwww-perl',
    'python-requests',
    'http-client',
    'Go-http-client',
    'axios',
    'Node-fetch',
    'okhttp',
    'WinHttp.WinHttpRequest',
    'GuzzleHttp/',
    'HttpClient/',

    /** User agents for popular browser automation and testing tools. */
    'Selenium',
    'puppeteer',
    'Playwright',
    'PhantomJS',
    'CasperJS',
    'Cypress',
    'webdriver',

    /** User agents for miscellaneous tools and web scrapers. */
    'HTTrack',
    'GoScraper',
    'Scrapy',
    'Mechanize',
    'zgrab',
    'Harvest',
    'URL Grabber',
    'Nikto',
    'Nmap',
    'Netcraft',
    'masscan',

    /** Generic patterns commonly found in the user-agent strings of bots, crawlers, and scrapers. */
    'bot',
    'crawler',
    'spider',
    'http client',
    'scan',
    'scrape',
    'scanner',
    'fetcher',
    'probe',
    ];

    export default userAgents;