Skip to content

Instantly share code, notes, and snippets.

@TechByTom

TechByTom/Web-InterfaceCategorizer.ps1

Last active February 27, 2025 19:57
Show Gist options
  • Save TechByTom/04d3ac248b0a197048e27f44700c94f7 to your computer and use it in GitHub Desktop.
Save TechByTom/04d3ac248b0a197048e27f44700c94f7 to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. TechByTom revised this gist Feb 27, 2025. 1 changed file with 268 additions and 55 deletions.
    323 changes: 268 additions & 55 deletions Web-InterfaceCategorizer.ps1
    View file
    Original file line number Diff line number Diff line change
    @@ -9,6 +9,37 @@ param(
    [string]$OutputDirectory = ".\categorized_interfaces"
    )

    # Create Windows shortcut (.lnk) file
    function New-Shortcut {
    param(
    [Parameter(Mandatory=$true)]
    [string]$SourcePath,
    [Parameter(Mandatory=$true)]
    [string]$ShortcutPath
    )

    # Convert paths to absolute paths
    $SourcePath = [System.IO.Path]::GetFullPath($SourcePath)
    $ShortcutPath = [System.IO.Path]::GetFullPath($ShortcutPath)

    $WScriptShell = New-Object -ComObject WScript.Shell
    $Shortcut = $WScriptShell.CreateShortcut($ShortcutPath)
    $Shortcut.TargetPath = $SourcePath
    $Shortcut.WorkingDirectory = [System.IO.Path]::GetDirectoryName($SourcePath)
    $Shortcut.Save()
    }

    # Get original file path from .lnk file
    function Get-ShortcutTarget {
    param(
    [string]$ShortcutPath
    )

    $WScriptShell = New-Object -ComObject WScript.Shell
    $Shortcut = $WScriptShell.CreateShortcut($ShortcutPath)
    return $Shortcut.TargetPath
    }

    # Function to test if a string matches any pattern in an array
    function Test-Patterns {
    param(
    @@ -24,25 +55,6 @@ function Test-Patterns {
    return $false
    }

    # Function to convert PSObject to Hashtable
    function ConvertTo-Hashtable {
    param (
    [Parameter(ValueFromPipeline)]
    $InputObject
    )

    process {
    if ($null -eq $InputObject) { return $null }
    if ($InputObject -is [System.Collections.Hashtable]) { return $InputObject }

    $hash = @{}
    $InputObject.PSObject.Properties | ForEach-Object {
    $hash[$_.Name] = $_.Value
    }
    return $hash
    }
    }

    # Function to check if headers match the specified patterns
    function Test-Headers {
    param(
    @@ -72,6 +84,25 @@ function Test-Headers {
    return $true
    }

    # Function to convert PSObject to Hashtable
    function ConvertTo-Hashtable {
    param (
    [Parameter(ValueFromPipeline)]
    $InputObject
    )

    process {
    if ($null -eq $InputObject) { return $null }
    if ($InputObject -is [System.Collections.Hashtable]) { return $InputObject }

    $hash = @{}
    $InputObject.PSObject.Properties | ForEach-Object {
    $hash[$_.Name] = $_.Value
    }
    return $hash
    }
    }

    # Function to categorize a single interface based on fingerprints
    function Get-InterfaceCategory {
    param(
    @@ -81,9 +112,9 @@ function Get-InterfaceCategory {

    foreach ($category in $Fingerprints.PSObject.Properties) {
    $fingerprint = $category.Value
    $matched = $true
    $allPropertiesMatched = $true

    # Check URL patterns
    # Check URL patterns if specified
    if ($fingerprint.urlPatterns) {
    $urlFound = $false
    foreach ($port in $InterfaceData.OpenPorts) {
    @@ -100,12 +131,13 @@ function Get-InterfaceCategory {
    }
    }
    if (-not $urlFound) {
    $matched = $false
    $allPropertiesMatched = $false
    continue
    }
    }

    # Check content patterns
    if ($matched -and $fingerprint.contentPatterns) {
    # Check content patterns if specified
    if ($allPropertiesMatched -and $fingerprint.contentPatterns) {
    $contentFound = $false
    foreach ($port in $InterfaceData.OpenPorts) {
    if ($port.WebRequest) {
    @@ -121,12 +153,13 @@ function Get-InterfaceCategory {
    }
    }
    if (-not $contentFound) {
    $matched = $false
    $allPropertiesMatched = $false
    continue
    }
    }

    # Check header patterns
    if ($matched -and $fingerprint.headerPatterns) {
    # Check header patterns if specified
    if ($allPropertiesMatched -and $fingerprint.headerPatterns) {
    $headersFound = $false
    foreach ($port in $InterfaceData.OpenPorts) {
    if ($port.WebRequest) {
    @@ -142,25 +175,44 @@ function Get-InterfaceCategory {
    }
    }
    if (-not $headersFound) {
    $matched = $false
    $allPropertiesMatched = $false
    continue
    }
    }

    if ($matched) {
    # Only return the category if ALL specified properties match
    if ($allPropertiesMatched) {
    return $category.Name
    }
    }

    return "Unknown"
    }

    # Function to generate HTML report
    # Function to extract title from HTML content
    function Get-HtmlTitle {
    param(
    [string]$Content
    )

    if (-not $Content) { return $null }

    if ($Content -match '<title[^>]*>(.*?)</title>') {
    return $matches[1].Trim()
    }

    return $null
    }

    # Updated report generation function
    function New-CategoryReport {
    param(
    [string]$OutputDirectory
    [string]$OutputDirectory,
    [string]$ScanDirectory
    )

    $reportPath = Join-Path $OutputDirectory "categorization_report.html"
    $timestamp = Get-Date -Format "yyyyMMdd_HHmmss"
    $reportPath = Join-Path $OutputDirectory "categorization_report_${timestamp}.html"
    $categories = Get-ChildItem -Path $OutputDirectory -Directory

    $html = @"
    @@ -172,14 +224,39 @@ function New-CategoryReport {
    body { font-family: Arial, sans-serif; margin: 20px; }
    h1 { color: #333; }
    h2 { color: #666; margin-top: 30px; }
    h3 { color: #666; margin-top: 20px; margin-left: 20px; }
    .category { margin-bottom: 30px; }
    table { border-collapse: collapse; width: 100%; }
    table { border-collapse: collapse; width: 100%; margin-top: 10px; }
    th, td { border: 1px solid #ddd; padding: 8px; text-align: left; }
    th { background-color: #f5f5f5; }
    tr:nth-child(even) { background-color: #f9f9f9; }
    tr:hover { background-color: #f5f5f5; }
    .stats { margin: 20px 0; padding: 10px; background-color: #f8f9fa; border-radius: 4px; }
    .category-count { color: #666; font-size: 0.9em; margin-left: 10px; }
    .collapse-btn {
    background: none;
    border: none;
    color: #666;
    cursor: pointer;
    font-size: 1.2em;
    padding: 0 10px;
    vertical-align: middle;
    }
    .collapse-btn:hover { color: #333; }
    .section-header {
    display: flex;
    align-items: center;
    }
    .collapsed { display: none; }
    </style>
    <script>
    function toggleSection(btn, targetId) {
    const section = document.getElementById(targetId);
    const isCollapsed = section.classList.contains('collapsed');
    section.classList.toggle('collapsed');
    btn.textContent = isCollapsed ? '▼' : '▶';
    }
    </script>
    </head>
    <body>
    <h1>Interface Categorization Report</h1>
    @@ -189,22 +266,31 @@ function New-CategoryReport {
    </div>
    "@

    foreach ($category in $categories) {
    $files = Get-ChildItem -Path $category.FullName -Filter "*.json"
    # First process all non-Unknown categories
    foreach ($category in ($categories | Where-Object { $_.Name -ne "Unknown" })) {
    $files = Get-ChildItem -Path $category.FullName -Filter "*.lnk"
    $sectionId = "category-$($category.Name.ToLower())"

    $html += @"
    <div class="category">
    <h2>$($category.Name)</h2>
    <table>
    <tr>
    <th>IP Address</th>
    <th>Hostname</th>
    <th>Ports</th>
    <th>Links</th>
    </tr>
    <div class="section-header">
    <button class="collapse-btn" onclick="toggleSection(this, '$sectionId')">▼</button>
    <h2>$($category.Name) <span class="category-count">($($files.Count) interfaces)</span></h2>
    </div>
    <div id="$sectionId">
    <table>
    <tr>
    <th>IP Address</th>
    <th>Hostname</th>
    <th>Ports</th>
    <th>Web Links</th>
    <th>JSON File</th>
    </tr>
    "@

    foreach ($file in $files) {
    $interfaceData = Get-Content $file.FullName | ConvertFrom-Json
    $originalPath = Get-ShortcutTarget -ShortcutPath $file.FullName
    $interfaceData = Get-Content $originalPath | ConvertFrom-Json
    $ports = ($interfaceData.OpenPorts | ForEach-Object { $_.PortNumber }) -join ", "
    $links = @()

    @@ -226,19 +312,135 @@ function New-CategoryReport {
    }) -join "<br>"

    $html += @"
    <tr>
    <td>$($interfaceData.IP)</td>
    <td>$($interfaceData.Hostname)</td>
    <td>$ports</td>
    <td>$linksHtml</td>
    </tr>
    <tr>
    <td>$($interfaceData.IP)</td>
    <td>$($interfaceData.Hostname)</td>
    <td>$ports</td>
    <td>$linksHtml</td>
    <td><a href="file://$originalPath">$([System.IO.Path]::GetFileName($originalPath))</a></td>
    </tr>
    "@
    }

    $html += @"
    </table>
    </table>
    </div>
    </div>
    "@
    }

    # Process Unknown category with title grouping
    $unknownCategory = $categories | Where-Object { $_.Name -eq "Unknown" }
    if ($unknownCategory) {
    $files = Get-ChildItem -Path $unknownCategory.FullName -Filter "*.lnk"

    if ($files) {
    $html += @"
    <div class="category">
    <div class="section-header">
    <button class="collapse-btn" onclick="toggleSection(this, 'category-unknown')">▼</button>
    <h2>Unknown <span class="category-count">($($files.Count) interfaces)</span></h2>
    </div>
    <div id="category-unknown">
    "@

    # Group unknown files by title
    $titleGroups = @{}

    foreach ($file in $files) {
    $originalPath = Get-ShortcutTarget -ShortcutPath $file.FullName
    $interfaceData = Get-Content $originalPath | ConvertFrom-Json
    $title = "No Title"

    # Try to find a title in any of the web responses
    foreach ($port in $interfaceData.OpenPorts) {
    if ($port.WebRequest) {
    $requests = @($port.WebRequest)
    if ($requests[0] -isnot [array]) { $requests = @($requests) }

    foreach ($request in $requests) {
    if ($request.Content) {
    $extractedTitle = Get-HtmlTitle -Content $request.Content
    if ($extractedTitle) {
    $title = $extractedTitle
    break
    }
    }
    }
    }
    if ($title -ne "No Title") { break }
    }

    if (-not $titleGroups.ContainsKey($title)) {
    $titleGroups[$title] = @()
    }
    $titleGroups[$title] += $file
    }

    # Sort title groups by count (most to least)
    foreach ($titleGroup in $titleGroups.GetEnumerator() | Sort-Object { $_.Value.Count } -Descending) {
    $titleId = "unknown-title-$([System.Web.HttpUtility]::UrlEncode($titleGroup.Key))"
    $html += @"
    <div class="section-header">
    <button class="collapse-btn" onclick="toggleSection(this, '$titleId')">▼</button>
    <h3>Title: $($titleGroup.Key) <span class="category-count">($($titleGroup.Value.Count) interfaces)</span></h3>
    </div>
    <div id="$titleId">
    <table>
    <tr>
    <th>IP Address</th>
    <th>Hostname</th>
    <th>Ports</th>
    <th>Web Links</th>
    <th>JSON File</th>
    </tr>
    "@

    foreach ($file in $titleGroup.Value) {
    $originalPath = Get-ShortcutTarget -ShortcutPath $file.FullName
    $interfaceData = Get-Content $originalPath | ConvertFrom-Json
    $ports = ($interfaceData.OpenPorts | ForEach-Object { $_.PortNumber }) -join ", "
    $links = @()

    foreach ($port in $interfaceData.OpenPorts) {
    if ($port.WebRequest) {
    $requests = @($port.WebRequest)
    if ($requests[0] -isnot [array]) { $requests = @($requests) }

    foreach ($request in $requests) {
    if ($request.Url) {
    $links += $request.Url
    }
    }
    }
    }

    $linksHtml = ($links | Select-Object -Unique | ForEach-Object {
    "<a href='$_' target='_blank'>$_</a>"
    }) -join "<br>"

    $html += @"
    <tr>
    <td>$($interfaceData.IP)</td>
    <td>$($interfaceData.Hostname)</td>
    <td>$ports</td>
    <td>$linksHtml</td>
    <td><a href="file://$originalPath">$([System.IO.Path]::GetFileName($originalPath))</a></td>
    </tr>
    "@
    }

    $html += @"
    </table>
    </div>
    "@
    }

    $html += @"
    </div>
    </div>
    "@
    }
    }

    $html += @"
    @@ -252,7 +454,15 @@ function New-CategoryReport {

    # Main script execution
    try {
    # Create output directory if it doesn't exist
    # Convert relative paths to absolute paths
    $ScanDirectory = Convert-Path $ScanDirectory
    $FingerprintsFile = Convert-Path $FingerprintsFile

    # Create timestamped output directory using absolute path
    $timestamp = Get-Date -Format "yyyyMMdd_HHmmss"
    $OutputDirectory = Join-Path (Convert-Path .) $OutputDirectory
    $OutputDirectory = Join-Path $OutputDirectory $timestamp

    if (-not (Test-Path $OutputDirectory)) {
    New-Item -ItemType Directory -Path $OutputDirectory | Out-Null
    }
    @@ -280,8 +490,11 @@ try {
    New-Item -ItemType Directory -Path $categoryDir | Out-Null
    }

    # Move file to appropriate category directory
    Move-Item $_.FullName -Destination $categoryDir -Force
    # Create .lnk file instead of symbolic link
    $linkPath = Join-Path $categoryDir "$($_.BaseName).lnk"
    if (-not (Test-Path $linkPath)) {
    New-Shortcut -SourcePath $_.FullName -ShortcutPath $linkPath
    }

    # Track result
    if (-not $results.ContainsKey($category)) {
    @@ -297,7 +510,7 @@ try {
    }

    # Generate HTML report
    $reportPath = New-CategoryReport -OutputDirectory $OutputDirectory
    $reportPath = New-CategoryReport -OutputDirectory $OutputDirectory -ScanDirectory $ScanDirectory

    # Display summary
    Write-Host "`nCategorization Summary:"
  2. TechByTom renamed this gist Feb 25, 2025. 1 changed file with 202 additions and 214 deletions.
    416 changes: 202 additions & 214 deletions Web-InterfaceIdentifier-0.4.1-CSVified.ps1 → Web-InterfaceIdentifier-0.4.2-CSVified.ps1
    View file
    Original file line number Diff line number Diff line change
    @@ -724,13 +724,13 @@ function Fetch-Url {
    Url = $currentUrl
    RedirectDepth = $RedirectCount
    StatusCode = $null
    Error = $null
    Message = $null # Changed from "Error" to "Message"
    Headers = @{}
    Content = $null
    Favicon = $null
    Title = $null
    RedirectedRequest = $null
    Certificate = $certDetails # Certificate details we've already retrieved
    Certificate = $certDetails
    }

    # Check if we should skip HTTPS connection due to certificate failure or timeout
    @@ -744,10 +744,10 @@ function Fetch-Url {
    if ($certDetails -and $certDetails.Error -and (Test-IsTimeoutException -ExceptionMessage $certDetails.Error)) {
    $timeoutError = $true
    Write-Host " → Certificate retrieval timed out - skipping HTTPS connection attempt" -ForegroundColor Yellow
    $requestResult.Error = "HTTPS connection skipped due to certificate retrieval timeout"
    $requestResult.Message = "HTTPS connection skipped due to certificate retrieval timeout"
    } else {
    Write-Host " → Certificate retrieval failed - skipping HTTPS connection attempt" -ForegroundColor Yellow
    $requestResult.Error = "HTTPS connection skipped due to certificate retrieval failure"
    $requestResult.Message = "HTTPS connection skipped due to certificate retrieval failure"
    }

    $results += $requestResult
    @@ -771,8 +771,9 @@ function Fetch-Url {
    $webRequest = [System.Net.WebRequest]::Create($currentUrl)
    $webRequest.Method = "HEAD"
    $webRequest.Timeout = $Timeout * 1000

    # Only set certificate validation callback for HTTPS requests
    if ($IsSSL) {
    # For HTTPS requests
    [System.Net.ServicePointManager]::ServerCertificateValidationCallback = { $true }
    }

    @@ -782,15 +783,15 @@ function Fetch-Url {

    if ($contentLength -gt $MaxResponseSize) {
    Write-Host " → Warning: Response too large ($([math]::Round($contentLength/1MB, 2)) MB). Skipping." -ForegroundColor Yellow
    $requestResult.Error = "Response size ($([math]::Round($contentLength/1MB, 2)) MB) exceeds maximum allowed size ($([math]::Round($MaxResponseSize/1MB)) MB)"
    $requestResult.Message = "Response size ($([math]::Round($contentLength/1MB, 2)) MB) exceeds maximum allowed size ($([math]::Round($MaxResponseSize/1MB)) MB)"
    $results += $requestResult
    break
    }
    } catch {
    # Check if this is a timeout exception
    if (Test-IsTimeoutException -ExceptionMessage $_.Exception.Message) {
    Write-Host " → HEAD request timed out" -ForegroundColor Yellow
    $requestResult.Error = "HEAD request timed out: $($_.Exception.Message)"
    $requestResult.Message = "HEAD request timed out: $($_.Exception.Message)"
    $results += $requestResult
    break
    }
    @@ -809,7 +810,9 @@ function Fetch-Url {
    ErrorAction = 'SilentlyContinue'
    }

    # Different handling for HTTP and HTTPS
    if ($IsSSL) {
    # HTTPS handling
    if ($psVersion -ge 6) {
    # For PowerShell 6+ we can use SkipCertificateCheck
    $webRequestParams.Add('SkipCertificateCheck', $true)
    @@ -828,6 +831,7 @@ function Fetch-Url {
    }
    }
    } else {
    # HTTP handling - no SSL/TLS specific code
    $response = Invoke-WebRequest @webRequestParams
    }

    @@ -841,7 +845,7 @@ function Fetch-Url {
    if ($actualLength -gt $MaxResponseSize) {
    Write-Host " → Warning: Response too large ($([math]::Round($actualLength/1MB, 2)) MB). Truncating." -ForegroundColor Yellow
    $requestResult.Content = $response.Content.Substring(0, $MaxResponseSize)
    $requestResult.Error = "Response truncated: exceeded maximum size of $([math]::Round($MaxResponseSize/1MB)) MB"
    $requestResult.Message = "Response truncated: exceeded maximum size of $([math]::Round($MaxResponseSize/1MB)) MB"
    } else {
    $requestResult.Content = $response.Content
    Write-Host " → Received $([math]::Round($actualLength/1KB, 2)) KB"
    @@ -872,7 +876,13 @@ function Fetch-Url {
    $statusCode = [int]$_.Exception.Response.StatusCode
    $requestResult.StatusCode = $statusCode
    $requestResult.Headers = $_.Exception.Response.Headers
    $requestResult.Error = $_.Exception.Message

    # Instead of treating redirects as errors, treat them as normal status codes
    if ($statusCode -ge 300 -and $statusCode -lt 400) {
    $requestResult.Message = "Redirect: $statusCode $($_.Exception.Response.StatusDescription)"
    } else {
    $requestResult.Message = "$statusCode $($_.Exception.Response.StatusDescription)"
    }

    # Try to get response content even for error status codes
    try {
    @@ -938,7 +948,7 @@ function Fetch-Url {
    Url = $currentUrl
    RedirectDepth = $RedirectCount
    StatusCode = $null
    Error = $null
    Message = $null # Changed from "Error" to "Message"
    Headers = @{}
    Content = $null
    Favicon = $null
    @@ -955,7 +965,7 @@ function Fetch-Url {
    }
    }

    Write-Host " → Warning: HTTP $statusCode - $($_.Exception.Message)" -ForegroundColor Yellow
    Write-Host " → Status: HTTP $statusCode - $($_.Exception.Response.StatusDescription)" -ForegroundColor Yellow
    } else {
    # Check for timeout exception
    if (Test-IsTimeoutException -ExceptionMessage $_.Exception.Message) {
    @@ -964,7 +974,7 @@ function Fetch-Url {
    Write-Host " → Warning: $($_.Exception.Message)" -ForegroundColor Yellow
    }

    $requestResult.Error = $_.Exception.Message
    $requestResult.Message = $_.Exception.Message
    $results += $requestResult
    }
    break
    @@ -976,236 +986,214 @@ function Fetch-Url {
    }

    # Function to export JSON data to CSV in both wide and long formats
    function Export-ToCSV {
    function Append-HostToCSV {
    param(
    [string]$InputDirectory,
    [string]$OutputFile,
    [ValidateSet("Long", "Wide")]
    [string]$Format = "Wide"
    [string]$OutputDir,
    [string]$WideCSVPath,
    [string]$LongCSVPath,
    [PSCustomObject]$HostData
    )

    # Get all JSON files in the directory
    $jsonFiles = Get-ChildItem -Path $InputDirectory -Filter "*.json"
    Write-Host "Found $($jsonFiles.Count) JSON files to process for CSV export"

    # Create an array to hold all CSV data
    $csvData = @()
    # Helper function to extract title from HTML content
    function Get-HtmlTitle {
    param([string]$HtmlContent)

    if ([string]::IsNullOrWhiteSpace($HtmlContent)) {
    return $null
    }

    try {
    # Try to match the title tag
    if ($HtmlContent -match '<title[^>]*>(.*?)</title>') {
    return $matches[1].Trim()
    }
    } catch {
    # If there's any error in regex processing, return null
    return $null
    }

    return $null
    }

    if ($Format -eq "Long") {
    # Long format (normalized: one row per request/redirect)
    foreach ($file in $jsonFiles) {
    Write-Host "Processing $($file.Name)"
    $hostData = Get-Content -Path $file.FullName | ConvertFrom-Json
    # Create wide format row(s)
    $wideRows = @()
    foreach ($port in $HostData.OpenPorts) {
    $rowData = [ordered]@{
    IP = $HostData.IP
    Hostname = $HostData.Hostname
    DNSResolution = $HostData.DNSResolution.Resolves
    DNSMatchesIP = $HostData.DNSResolution.IPMatch
    Port = $port.PortNumber
    Protocol = $port.Protocol
    Service = $port.Service
    IsSSL = $port.IsSSL
    TotalRedirects = ($port.WebRequest | Measure-Object).Count
    }

    # Add redirect information
    for ($i = 0; $i -lt $port.WebRequest.Count; $i++) {
    $request = $port.WebRequest[$i]
    $prefix = "Redirect$($i+1)_"

    # For each open port
    foreach ($port in $hostData.OpenPorts) {
    # Base row data (host and port info)
    $baseRowData = [ordered]@{
    IP = $hostData.IP
    Hostname = $hostData.Hostname
    DNSResolution = $hostData.DNSResolution.Resolves
    DNSMatchesIP = $hostData.DNSResolution.IPMatch
    Port = $port.PortNumber
    Protocol = $port.Protocol
    Service = $port.Service
    IsSSL = $port.IsSSL
    }
    # Extract title from HTML content
    $title = Get-HtmlTitle -HtmlContent $request.Content

    # Add request details
    $rowData["${prefix}URL"] = $request.Url
    $rowData["${prefix}Message"] = $request.Message
    $rowData["${prefix}Error"] = $request.Error
    $rowData["${prefix}HasContent"] = if ($request.Content) { $true } else { $false }
    $rowData["${prefix}ContentLength"] = if ($request.Content) { $request.Content.Length } else { 0 }
    $rowData["${prefix}HasFavicon"] = if ($request.Favicon) { $true } else { $false }
    $rowData["${prefix}Content"] = $request.Content
    $rowData["${prefix}Favicon"] = $request.Favicon
    $rowData["${prefix}Title"] = $title

    # Add headers if available
    if ($request.Headers -and $request.Headers.Count -gt 0) {
    $headerString = ($request.Headers.GetEnumerator() | ForEach-Object { "$($_.Key): $($_.Value)" }) -join "; "
    $rowData["${prefix}Headers"] = $headerString
    } else {
    $rowData["${prefix}Headers"] = ""
    }

    # Certificate information (if SSL)
    if ($port.IsSSL -and $request.Certificate) {
    $cert = $request.Certificate

    # Add rows for each request in the redirect chain
    for ($i = 0; $i -lt $port.WebRequest.Count; $i++) {
    $request = $port.WebRequest[$i]

    $rowData = $baseRowData.Clone()
    $rowData += [ordered]@{
    RedirectIndex = $i
    URL = $request.Url
    StatusCode = $request.StatusCode
    Error = $request.Error
    HasContent = if ($request.Content) { $true } else { $false }
    ContentLength = if ($request.Content) { $request.Content.Length } else { 0 }
    HasFavicon = if ($request.Favicon) { $true } else { $false }
    # Add certificate status and method
    $rowData["${prefix}CertStatus"] = if ($cert.PSObject.Properties.Name -contains "Status") { $cert.Status } else { "Unknown" }
    $rowData["${prefix}CertMethod"] = if ($cert.PSObject.Properties.Name -contains "Method") { $cert.Method } else { "Unknown" }
    $rowData["${prefix}CertError"] = if ($cert.PSObject.Properties.Name -contains "Error") { $cert.Error } else { $null }

    # Add certificate details if available
    if ($cert.PSObject.Properties.Name -contains "Subject" -and $cert.Subject -ne $null) {
    $rowData["${prefix}CertSubjectCN"] = $cert.Subject.CN
    $rowData["${prefix}CertIssuerCN"] = $cert.Issuer.CN
    $rowData["${prefix}CertValidFrom"] = $cert.Validity.NotBefore
    $rowData["${prefix}CertValidTo"] = $cert.Validity.NotAfter
    $rowData["${prefix}CertDaysUntilExpiration"] = $cert.Validity.DaysUntilExpiration
    $rowData["${prefix}CertSignatureAlgorithm"] = $cert.SignatureAlgorithm
    $rowData["${prefix}CertSelfSigned"] = $cert.SelfSigned
    $rowData["${prefix}CertSupportedTLSVersions"] = if ($cert.TLSSupport -and $cert.TLSSupport.SupportedProtocols) {
    ($cert.TLSSupport.SupportedProtocols -join ", ")
    } else {
    "Unknown"
    }

    # Add certificate info for SSL connections - handle both successful and failed certificate retrievals
    if ($port.IsSSL -and $request.Certificate) {
    $cert = $request.Certificate

    # Add certificate status and method
    $rowData += [ordered]@{
    CertStatus = if ($cert.PSObject.Properties.Name -contains "Status") { $cert.Status } else { "Unknown" }
    CertMethod = if ($cert.PSObject.Properties.Name -contains "Method") { $cert.Method } else { "Unknown" }
    CertError = if ($cert.PSObject.Properties.Name -contains "Error") { $cert.Error } else { $null }
    }

    # Add certificate details if available
    if ($cert.PSObject.Properties.Name -contains "Subject" -and $cert.Subject -ne $null) {
    $rowData += [ordered]@{
    CertSubjectCN = $cert.Subject.CN
    CertIssuerCN = $cert.Issuer.CN
    CertValidFrom = $cert.Validity.NotBefore
    CertValidTo = $cert.Validity.NotAfter
    CertDaysUntilExpiration = $cert.Validity.DaysUntilExpiration
    CertSignatureAlgorithm = $cert.SignatureAlgorithm
    CertSelfSigned = $cert.SelfSigned
    CertSupportedTLSVersions = if ($cert.TLSSupport -and $cert.TLSSupport.SupportedProtocols) {
    ($cert.TLSSupport.SupportedProtocols -join ", ")
    } else {
    "Unknown"
    }
    }
    } else {
    # Add placeholder values for missing certificate details
    $rowData += [ordered]@{
    CertSubjectCN = "Unknown"
    CertIssuerCN = "Unknown"
    CertValidFrom = "Unknown"
    CertValidTo = "Unknown"
    CertDaysUntilExpiration = $null
    CertSignatureAlgorithm = "Unknown"
    CertSelfSigned = $null
    CertSupportedTLSVersions = "Unknown"
    }
    }
    } else {
    # No certificate data (not SSL or no cert)
    $rowData += [ordered]@{
    CertStatus = "NotApplicable"
    CertMethod = "NotApplicable"
    CertError = $null
    CertSubjectCN = $null
    CertIssuerCN = $null
    CertValidFrom = $null
    CertValidTo = $null
    CertDaysUntilExpiration = $null
    CertSignatureAlgorithm = $null
    CertSelfSigned = $null
    CertSupportedTLSVersions = $null
    }
    # Add thumbprints
    if ($cert.Thumbprint) {
    $rowData["${prefix}CertThumbprintSHA1"] = $cert.Thumbprint.SHA1
    $rowData["${prefix}CertThumbprintSHA256"] = $cert.Thumbprint.SHA256
    }

    # Create PSObject and add to CSV data
    $csvData += [PSCustomObject]$rowData
    # Include the serial number
    $rowData["${prefix}CertSerialNumber"] = $cert.SerialNumber
    }
    }
    }
    } else {
    # Wide format (one row per host+port, with columns for each redirect)

    # First pass: determine the maximum number of redirects for any response
    $maxRedirects = 0

    foreach ($file in $jsonFiles) {
    $hostData = Get-Content -Path $file.FullName | ConvertFrom-Json
    $wideRows += [PSCustomObject]$rowData
    }

    # Create long format row(s)
    $longRows = @()
    foreach ($port in $HostData.OpenPorts) {
    # For each request in the redirect chain
    for ($i = 0; $i -lt $port.WebRequest.Count; $i++) {
    $request = $port.WebRequest[$i]

    foreach ($port in $hostData.OpenPorts) {
    $redirectCount = ($port.WebRequest | Measure-Object).Count
    if ($redirectCount -gt $maxRedirects) {
    $maxRedirects = $redirectCount
    }
    # Extract title from HTML content
    $title = Get-HtmlTitle -HtmlContent $request.Content

    # Create a new ordered dictionary for each row
    $rowData = [ordered]@{
    IP = $HostData.IP
    Hostname = $HostData.Hostname
    DNSResolution = $HostData.DNSResolution.Resolves
    DNSMatchesIP = $HostData.DNSResolution.IPMatch
    Port = $port.PortNumber
    Protocol = $port.Protocol
    Service = $port.Service
    IsSSL = $port.IsSSL
    RedirectIndex = $i
    URL = $request.Url
    StatusCode = $request.StatusCode
    Message = $request.Message
    HasContent = if ($request.Content) { $true } else { $false }
    ContentLength = if ($request.Content) { $request.Content.Length } else { 0 }
    HasFavicon = if ($request.Favicon) { $true } else { $false }
    Content = $request.Content
    Favicon = $request.Favicon
    Title = $title
    }
    }

    Write-Host "Maximum redirect chain length: $maxRedirects"

    # Second pass: create the CSV data
    foreach ($file in $jsonFiles) {
    Write-Host "Processing $($file.Name)"
    $hostData = Get-Content -Path $file.FullName | ConvertFrom-Json

    # For each open port
    foreach ($port in $hostData.OpenPorts) {
    $rowData = [ordered]@{
    # Host information
    IP = $hostData.IP
    Hostname = $hostData.Hostname
    DNSResolution = $hostData.DNSResolution.Resolves
    DNSMatchesIP = $hostData.DNSResolution.IPMatch
    # Port information
    Port = $port.PortNumber
    Protocol = $port.Protocol
    Service = $port.Service
    IsSSL = $port.IsSSL
    TotalRedirects = ($port.WebRequest | Measure-Object).Count
    }
    # Add headers if available
    if ($request.Headers -and $request.Headers.Count -gt 0) {
    $headerString = ($request.Headers.GetEnumerator() | ForEach-Object { "$($_.Key): $($_.Value)" }) -join "; "
    $rowData["Headers"] = $headerString
    } else {
    $rowData["Headers"] = ""
    }

    # Certificate information (if SSL)
    if ($port.IsSSL -and $request.Certificate) {
    $cert = $request.Certificate

    # Add certificate status and method
    $rowData["CertStatus"] = if ($cert.PSObject.Properties.Name -contains "Status") { $cert.Status } else { "Unknown" }
    $rowData["CertMethod"] = if ($cert.PSObject.Properties.Name -contains "Method") { $cert.Method } else { "Unknown" }
    $rowData["CertError"] = if ($cert.PSObject.Properties.Name -contains "Error") { $cert.Error } else { $null }

    # Add columns for each redirect in the chain
    for ($i = 0; $i -lt $maxRedirects; $i++) {
    $request = $port.WebRequest[$i]
    # Add certificate details if available
    if ($cert.PSObject.Properties.Name -contains "Subject" -and $cert.Subject -ne $null) {
    $rowData["CertSubjectCN"] = $cert.Subject.CN
    $rowData["CertIssuerCN"] = $cert.Issuer.CN
    $rowData["CertValidFrom"] = $cert.Validity.NotBefore
    $rowData["CertValidTo"] = $cert.Validity.NotAfter
    $rowData["CertDaysUntilExpiration"] = $cert.Validity.DaysUntilExpiration
    $rowData["CertSignatureAlgorithm"] = $cert.SignatureAlgorithm
    $rowData["CertSelfSigned"] = $cert.SelfSigned
    $rowData["CertSupportedTLSVersions"] = if ($cert.TLSSupport -and $cert.TLSSupport.SupportedProtocols) {
    ($cert.TLSSupport.SupportedProtocols -join ", ")
    } else {
    "Unknown"
    }

    $prefix = "Redirect$($i+1)_"
    if ($request) {
    $rowData["${prefix}URL"] = $request.Url
    $rowData["${prefix}StatusCode"] = $request.StatusCode
    $rowData["${prefix}Error"] = $request.Error
    $rowData["${prefix}HasContent"] = if ($request.Content) { $true } else { $false }
    $rowData["${prefix}ContentLength"] = if ($request.Content) { $request.Content.Length } else { 0 }
    $rowData["${prefix}HasFavicon"] = if ($request.Favicon) { $true } else { $false }

    # Certificate information (if SSL) - handle both successful and failed certificate retrievals
    if ($port.IsSSL -and $request.Certificate) {
    $cert = $request.Certificate

    # Add certificate status and method
    $rowData["${prefix}CertStatus"] = if ($cert.PSObject.Properties.Name -contains "Status") { $cert.Status } else { "Unknown" }
    $rowData["${prefix}CertMethod"] = if ($cert.PSObject.Properties.Name -contains "Method") { $cert.Method } else { "Unknown" }
    $rowData["${prefix}CertError"] = if ($cert.PSObject.Properties.Name -contains "Error") { $cert.Error } else { $null }

    # Add certificate details if available
    if ($cert.PSObject.Properties.Name -contains "Subject" -and $cert.Subject -ne $null) {
    $rowData["${prefix}CertSubjectCN"] = $cert.Subject.CN
    $rowData["${prefix}CertIssuerCN"] = $cert.Issuer.CN
    $rowData["${prefix}CertValidFrom"] = $cert.Validity.NotBefore
    $rowData["${prefix}CertValidTo"] = $cert.Validity.NotAfter
    $rowData["${prefix}CertDaysUntilExpiration"] = $cert.Validity.DaysUntilExpiration
    $rowData["${prefix}CertSignatureAlgorithm"] = $cert.SignatureAlgorithm
    $rowData["${prefix}CertSelfSigned"] = $cert.SelfSigned
    $rowData["${prefix}CertSupportedTLSVersions"] = if ($cert.TLSSupport -and $cert.TLSSupport.SupportedProtocols) {
    ($cert.TLSSupport.SupportedProtocols -join ", ")
    } else {
    "Unknown"
    }
    } else {
    # Add placeholder values for missing certificate details
    $rowData["${prefix}CertSubjectCN"] = "Unknown"
    $rowData["${prefix}CertIssuerCN"] = "Unknown"
    $rowData["${prefix}CertValidFrom"] = "Unknown"
    $rowData["${prefix}CertValidTo"] = "Unknown"
    $rowData["${prefix}CertDaysUntilExpiration"] = $null
    $rowData["${prefix}CertSignatureAlgorithm"] = "Unknown"
    $rowData["${prefix}CertSelfSigned"] = $null
    $rowData["${prefix}CertSupportedTLSVersions"] = "Unknown"
    }
    } elseif ($port.IsSSL) {
    # SSL but no certificate information available
    $rowData["${prefix}CertStatus"] = "Missing"
    $rowData["${prefix}CertMethod"] = "NotAvailable"
    $rowData["${prefix}CertError"] = "No certificate data available"
    }
    # Add thumbprints
    if ($cert.Thumbprint) {
    $rowData["CertThumbprintSHA1"] = $cert.Thumbprint.SHA1
    $rowData["CertThumbprintSHA256"] = $cert.Thumbprint.SHA256
    }

    # Include the serial number
    $rowData["CertSerialNumber"] = $cert.SerialNumber
    }

    # Create a PSObject and add to the CSV data array
    $csvData += [PSCustomObject]$rowData
    }

    $longRows += [PSCustomObject]$rowData
    }
    }

    # Export the data to CSV
    # Append to CSV files
    try {
    $csvData | Export-Csv -Path $OutputFile -NoTypeInformation -Encoding UTF8
    Write-Host "CSV report generated: $OutputFile"
    } catch {
    Write-Host "Error exporting to CSV: $($_.Exception.Message)" -ForegroundColor Red
    # If file doesn't exist, create it with headers
    if (-not (Test-Path $WideCSVPath)) {
    $wideRows | Export-Csv -Path $WideCSVPath -NoTypeInformation -Encoding UTF8
    } else {
    $wideRows | Export-Csv -Path $WideCSVPath -NoTypeInformation -Encoding UTF8 -Append -Force
    }

    # Attempt to save to an alternative location if the original fails
    $alternativeFile = Join-Path -Path (Split-Path -Parent $OutputFile) -ChildPath "web_interfaces_alternative_$(Get-Date -Format 'yyyyMMdd_HHmmss').csv"
    try {
    $csvData | Export-Csv -Path $alternativeFile -NoTypeInformation -Encoding UTF8
    Write-Host "Alternative CSV report generated: $alternativeFile" -ForegroundColor Yellow
    } catch {
    Write-Host "Failed to save alternative CSV file: $($_.Exception.Message)" -ForegroundColor Red
    if (-not (Test-Path $LongCSVPath)) {
    $longRows | Export-Csv -Path $LongCSVPath -NoTypeInformation -Encoding UTF8
    } else {
    $longRows | Export-Csv -Path $LongCSVPath -NoTypeInformation -Encoding UTF8 -Append -Force
    }

    Write-Host "CSV data appended for host: $($HostData.IP)"
    }
    catch {
    Write-Host "Error appending CSV data for host $($HostData.IP): $_" -ForegroundColor Red
    }
    }

  3. TechByTom revised this gist Feb 25, 2025. 1 changed file with 1542 additions and 0 deletions.
    1,542 changes: 1,542 additions & 0 deletions Web-InterfaceIdentifier-0.4.1-CSVified.ps1
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,1542 @@
    param(
    [Parameter(Mandatory=$true)]
    [string]$XmlPath,
    [Parameter(Mandatory=$false)]
    [int]$MaxHosts = 0, # Will be set to total hosts if 0
    [Parameter(Mandatory=$false)]
    [int]$RequestTimeout = 10,
    [Parameter(Mandatory=$false)]
    [int]$MaxResponseSize = 20MB
    )

    # Function to count total hosts in the XML file
    function Get-TotalHosts {
    param(
    [string]$XmlPath
    )

    try {
    $settings = New-Object System.Xml.XmlReaderSettings
    $settings.DtdProcessing = [System.Xml.DtdProcessing]::Parse
    $reader = [System.Xml.XmlReader]::Create($XmlPath, $settings)
    $count = 0

    while ($reader.Read()) {
    if ($reader.NodeType -eq [System.Xml.XmlNodeType]::Element -and $reader.Name -eq "host") {
    $count++
    }
    }

    return $count
    }
    finally {
    if ($reader) {
    $reader.Close()
    }
    }
    }

    # Function to format time span
    function Format-TimeSpan {
    param (
    [TimeSpan]$TimeSpan
    )

    if ($TimeSpan.TotalHours -ge 1) {
    return "{0:0}h {1:0}m {2:0}s" -f $TimeSpan.Hours, $TimeSpan.Minutes, $TimeSpan.Seconds
    }
    elseif ($TimeSpan.TotalMinutes -ge 1) {
    return "{0:0}m {1:0}s" -f $TimeSpan.Minutes, $TimeSpan.Seconds
    }
    else {
    return "{0:0}s" -f $TimeSpan.TotalSeconds
    }
    }

    function Get-AvailableTlsProtocols {
    # Define possible protocols in order from newest to oldest
    $possibleProtocols = @(
    @{ Name = "Tls13"; Enum = "Tls13" },
    @{ Name = "Tls12"; Enum = "Tls12" },
    @{ Name = "Tls11"; Enum = "Tls11" },
    @{ Name = "Tls"; Enum = "Tls" },
    @{ Name = "Ssl3"; Enum = "Ssl3" },
    @{ Name = "Ssl2"; Enum = "Ssl2" }
    )

    $availableProtocols = @()

    # Dynamically check which protocols are defined in the current .NET version
    foreach ($protocol in $possibleProtocols) {
    try {
    # Try to access the enum value
    $enumValue = [System.Security.Authentication.SslProtocols]::($protocol.Enum)
    $availableProtocols += @{ Name = $protocol.Name; Value = $enumValue }
    Write-Host " → Protocol $($protocol.Name) is available"
    }
    catch {
    Write-Host " → Protocol $($protocol.Name) is not supported on this system" -ForegroundColor Yellow
    }
    }

    if ($availableProtocols.Count -eq 0) {
    Write-Host " → Warning: No SSL/TLS protocols are available on this system" -ForegroundColor Red
    throw "No SSL/TLS protocols available"
    }

    Write-Host " → Available protocols (from newest to oldest): $($availableProtocols.Name -join ', ')"
    return $availableProtocols
    }

    # Detect available TLS/SSL protocols once at script startup
    Write-Host "Detecting available TLS/SSL protocols..."
    $GlobalAvailableTlsProtocols = Get-AvailableTlsProtocols

    Write-Host "TLS/SSL protocol detection complete. Ready to scan hosts."

    # Function to enable all locally supported TLS/SSL versions
    function Set-MaximumTlsSupport {
    $protocols = [enum]::GetValues([System.Net.SecurityProtocolType]) |
    Where-Object { $_ -ne 'SystemDefault' }

    $supportedProtocols = $protocols | ForEach-Object {
    try {
    [System.Net.ServicePointManager]::SecurityProtocol = $_
    $_
    } catch {
    Write-Host "Protocol $_ not supported"
    $null
    }
    } | Where-Object { $_ -ne $null }

    $finalProtocol = [System.Net.SecurityProtocolType]($supportedProtocols -join ',')
    [System.Net.ServicePointManager]::SecurityProtocol = $finalProtocol
    Write-Host "Enabled protocols: $finalProtocol"
    }

    function Get-CertificateDetails {
    param(
    [string]$Hostname,
    [int]$Port,
    [int]$Timeout = 10,
    [array]$AvailableProtocols = $GlobalAvailableTlsProtocols # Use the global variable by default
    )

    Write-Host (" → Retrieving SSL certificate from ${Hostname}:${Port}")

    # Helper function to display inner exceptions
    function Get-FullExceptionDetails {
    param([Exception]$Exception)

    $details = $Exception.Message
    $currentEx = $Exception

    # Traverse inner exceptions
    while ($currentEx.InnerException) {
    $currentEx = $currentEx.InnerException
    $details += "`n → Inner Exception: $($currentEx.Message)"

    # Check for specific exception types
    if ($currentEx -is [System.Security.Authentication.AuthenticationException]) {
    $details += " (Authentication error - likely certificate or protocol mismatch)"
    }
    elseif ($currentEx -is [System.IO.IOException]) {
    $details += " (IO error - likely connection reset or timeout)"
    }
    }

    return $details
    }

    # Helper function to check if an exception is timeout-related
    function Test-IsTimeoutException {
    param([string]$ExceptionMessage)

    return ($ExceptionMessage -match "timed? out|timeout|aborted|could not establish trust|operation has timed out|connection failure|connection attempt failed")
    }

    # Method 1: Try all SSL/TLS protocols from newest to oldest
    try {
    Write-Host " → Method 1: Using TLS/SSL protocols from newest to oldest"

    # Skip protocol detection - use the provided list
    if ($AvailableProtocols.Count -eq 0) {
    Write-Host " → Warning: No SSL/TLS protocols are available" -ForegroundColor Red
    throw "No SSL/TLS protocols available"
    }

    # Try each protocol with different SNI options (empty or hostname)
    $sniOptions = @("", $Hostname)
    $timeoutEncountered = $false

    foreach ($protocol in $AvailableProtocols) {
    # If we already encountered a timeout, skip remaining protocols
    if ($timeoutEncountered) {
    Write-Host " → Skipping protocol $($protocol.Name) due to previous timeout" -ForegroundColor Yellow
    continue
    }

    foreach ($sniValue in $sniOptions) {
    $tcpClient = $null
    $sslStream = $null

    try {
    Write-Host " → Trying handshake with $($protocol.Name), SNI='$sniValue'"

    # Create a completely new TCP client for each attempt
    $tcpClient = New-Object System.Net.Sockets.TcpClient
    $tcpClient.ReceiveTimeout = $Timeout * 1000
    $tcpClient.SendTimeout = $Timeout * 1000

    # Set a connection timeout
    $connectResult = $tcpClient.BeginConnect($Hostname, $Port, $null, $null)
    $connectSuccess = $connectResult.AsyncWaitHandle.WaitOne($Timeout * 1000)

    if (-not $connectSuccess) {
    $timeoutEncountered = $true
    throw "TCP connection timed out after $Timeout seconds"
    }

    # Complete the connection
    $tcpClient.EndConnect($connectResult)

    # Create SSL stream with callback that ignores ALL certificate errors
    $sslStream = New-Object System.Net.Security.SslStream(
    $tcpClient.GetStream(),
    $false, # don't leave inner stream open
    { param($sender, $cert, $chain, $errors)
    # Log the specific errors for troubleshooting
    if ($errors -ne [System.Net.Security.SslPolicyErrors]::None) {
    # This is just for debugging, we'll still return true
    Write-Host " → Certificate errors: $errors" -ForegroundColor Yellow
    }
    return $true # Always accept the certificate regardless of errors
    }
    )

    # Try SSL/TLS handshake
    try {
    # Use a reasonable timeout for the handshake (2x the regular timeout)
    # We'll set read/write timeouts first
    $sslStream.ReadTimeout = $Timeout * 2000
    $sslStream.WriteTimeout = $Timeout * 2000

    # Then try the handshake
    $sslStream.AuthenticateAsClient($sniValue, $null, $protocol.Value, $false)
    }
    catch [Exception] {
    # Get detailed exception info
    $exDetails = Get-FullExceptionDetails -Exception $_.Exception

    # Check if this is a timeout-related exception
    if (Test-IsTimeoutException -ExceptionMessage $exDetails) {
    $timeoutEncountered = $true
    Write-Host " → Timeout encountered during handshake - will skip remaining protocol attempts" -ForegroundColor Yellow
    }

    throw $exDetails
    }

    Write-Host " → SSL/TLS handshake successful with $($protocol.Name)!" -ForegroundColor Green

    # Get the certificate
    $remoteCertificate = $sslStream.RemoteCertificate
    if ($remoteCertificate -eq $null) {
    Write-Host " → No certificate found despite successful handshake" -ForegroundColor Yellow
    continue
    }

    # Convert to X509Certificate2 for more details
    $x509cert = New-Object System.Security.Cryptography.X509Certificates.X509Certificate2($remoteCertificate)

    # Display basic certificate info
    Write-Host " → Certificate details retrieved:" -ForegroundColor Green
    Write-Host " Subject: $($x509cert.Subject)"
    Write-Host " Issuer: $($x509cert.Issuer)"
    Write-Host " Valid from: $($x509cert.NotBefore) to $($x509cert.NotAfter)"

    # Extract Subject and Issuer components
    $subjectParts = $x509cert.Subject -split ', '
    $issuerParts = $x509cert.Issuer -split ', '

    $subjectCN = ($subjectParts | Where-Object { $_ -match '^CN=' } | ForEach-Object { $_ -replace '^CN=' })[0]
    $issuerCN = ($issuerParts | Where-Object { $_ -match '^CN=' } | ForEach-Object { $_ -replace '^CN=' })[0]

    # Check if self-signed
    $isSelfSigned = ($x509cert.Subject -eq $x509cert.Issuer)
    if ($isSelfSigned) {
    Write-Host " Self-signed: Yes" -ForegroundColor Yellow
    } else {
    Write-Host " Self-signed: No"
    }

    # Calculate days until expiration
    $daysUntilExpiration = [math]::Round(($x509cert.NotAfter - (Get-Date)).TotalDays, 1)

    # Return certificate details
    return @{
    Status = "Success"
    Method = "DirectTLS"
    Protocol = $protocol.Name
    SNI = $sniValue
    Subject = @{
    CN = $subjectCN
    O = ($subjectParts | Where-Object { $_ -match '^O=' } | ForEach-Object { $_ -replace '^O=' })
    OU = ($subjectParts | Where-Object { $_ -match '^OU=' } | ForEach-Object { $_ -replace '^OU=' })
    C = ($subjectParts | Where-Object { $_ -match '^C=' } | ForEach-Object { $_ -replace '^C=' })
    S = ($subjectParts | Where-Object { $_ -match '^S=' -or $_ -match '^ST=' } | ForEach-Object { $_ -replace '^S=|^ST=' })
    }
    Issuer = @{
    CN = $issuerCN
    O = ($issuerParts | Where-Object { $_ -match '^O=' } | ForEach-Object { $_ -replace '^O=' })
    OU = ($issuerParts | Where-Object { $_ -match '^OU=' } | ForEach-Object { $_ -replace '^OU=' })
    C = ($issuerParts | Where-Object { $_ -match '^C=' } | ForEach-Object { $_ -replace '^C=' })
    S = ($issuerParts | Where-Object { $_ -match '^S=' -or $_ -match '^ST=' } | ForEach-Object { $_ -replace '^S=|^ST=' })
    }
    Validity = @{
    NotBefore = $x509cert.NotBefore.ToString('yyyy-MM-dd HH:mm:ss')
    NotAfter = $x509cert.NotAfter.ToString('yyyy-MM-dd HH:mm:ss')
    DaysUntilExpiration = $daysUntilExpiration
    }
    SerialNumber = $x509cert.SerialNumber
    Thumbprint = @{
    SHA256 = $x509cert.GetCertHashString('SHA256')
    SHA1 = $x509cert.GetCertHashString('SHA1')
    }
    SignatureAlgorithm = $x509cert.SignatureAlgorithm.FriendlyName
    Version = $x509cert.Version
    SelfSigned = $isSelfSigned
    TLSSupport = @{
    SupportedProtocols = @($protocol.Name)
    }
    }
    }
    catch {
    $exMessage = $_
    Write-Host " → Attempt failed: $exMessage" -ForegroundColor Yellow

    # Check if this is a timeout-related exception
    if (Test-IsTimeoutException -ExceptionMessage $exMessage) {
    $timeoutEncountered = $true
    Write-Host " → Timeout detected - will skip remaining protocol attempts" -ForegroundColor Yellow
    break # Exit this SNI loop
    }

    # Clean up resources properly
    if ($sslStream) {
    try { $sslStream.Dispose() } catch {}
    }
    if ($tcpClient) {
    try { $tcpClient.Dispose() } catch {}
    }
    }
    }

    # If we encountered a timeout in the inner loop, break out of the outer loop too
    if ($timeoutEncountered) {
    break
    }
    }

    if ($timeoutEncountered) {
    Write-Host " → Skipping remaining protocol attempts due to timeout" -ForegroundColor Yellow
    } else {
    Write-Host " → All SSL/TLS protocol and SNI combinations failed" -ForegroundColor Yellow
    }
    }
    catch {
    $exDetails = Get-FullExceptionDetails -Exception $_.Exception
    Write-Host " → Method 1 failed: $exDetails" -ForegroundColor Yellow
    }

    # Method 2: WebRequest with custom TLS setting
    # Only try Method 2 if we didn't encounter a timeout in Method 1
    $skipMethod2 = $false
    if (Test-Path variable:timeoutEncountered) {
    $skipMethod2 = $timeoutEncountered
    }

    if ($skipMethod2) {
    Write-Host " → Skipping Method 2 due to timeout in Method 1" -ForegroundColor Yellow
    } else {
    try {
    Write-Host " → Method 2: Using WebRequest with custom TLS settings"

    # Save original settings
    $originalProtocol = [System.Net.ServicePointManager]::SecurityProtocol
    $originalCallback = [System.Net.ServicePointManager]::ServerCertificateValidationCallback

    try {
    # Try to use all possible TLS/SSL protocols
    # We'll use reflection to get all possible values for SecurityProtocolType
    $allValues = 0

    # Get all protocols via reflection from SecurityProtocolType enum
    $protocolType = [System.Net.SecurityProtocolType]
    $protocolFields = $protocolType.GetFields() | Where-Object { $_.IsStatic -and $_.IsLiteral }

    foreach ($field in $protocolFields) {
    try {
    $value = $field.GetValue($null)
    Write-Host " → Adding protocol $($field.Name) ($value)"
    $allValues = $allValues -bor $value
    } catch {
    Write-Host " → Couldn't add protocol $($field.Name): $($_.Exception.Message)" -ForegroundColor Yellow
    }
    }

    # If we couldn't get any protocols, fall back to hardcoded combination
    if ($allValues -eq 0) {
    Write-Host " → Using fallback protocol combination" -ForegroundColor Yellow
    try {
    $allValues = [System.Net.SecurityProtocolType]::Tls12 -bor
    [System.Net.SecurityProtocolType]::Tls11 -bor
    [System.Net.SecurityProtocolType]::Tls -bor
    [System.Net.SecurityProtocolType]::Ssl3
    }
    catch {
    # If even that fails, try just TLS 1.2
    $allValues = [System.Net.SecurityProtocolType]::Tls12
    }
    }

    # Apply our protocol selection
    [System.Net.ServicePointManager]::SecurityProtocol = $allValues

    # Accept all certificates regardless of errors
    [System.Net.ServicePointManager]::ServerCertificateValidationCallback = {
    param($sender, $cert, $chain, $errors)
    # Log the specific certificate errors for diagnostics
    if ($errors -ne [System.Net.Security.SslPolicyErrors]::None) {
    Write-Host " → Certificate validation errors: $errors" -ForegroundColor Yellow
    }
    return $true
    }

    # Create HTTP request
    $url = "https://${Hostname}:${Port}"
    $webRequest = [System.Net.WebRequest]::Create($url)
    $webRequest.Method = "HEAD"
    $webRequest.Timeout = $Timeout * 1000
    $webRequest.AllowAutoRedirect = $false

    # Attempt to get response (may fail but we might still get the certificate)
    try {
    $response = $webRequest.GetResponse()
    $response.Close()
    }
    catch [System.Net.WebException] {
    # Extract detailed information from the WebException
    $ex = $_.Exception
    $statusCode = -1
    $statusDesc = "Unknown"

    if ($ex.Response -ne $null) {
    $statusCode = [int]$ex.Response.StatusCode
    $statusDesc = $ex.Response.StatusDescription
    }

    Write-Host " → WebException: Status Code=$statusCode, Description=$statusDesc" -ForegroundColor Yellow

    # Check if we have inner exception details
    if ($ex.InnerException) {
    $innerExMsg = Get-FullExceptionDetails -Exception $ex.InnerException
    Write-Host " → Inner exception details: $innerExMsg" -ForegroundColor Yellow

    # Check if this is a timeout related exception
    if (Test-IsTimeoutException -ExceptionMessage $innerExMsg) {
    throw "Timeout encountered in WebRequest method"
    }
    }

    # Check if this is a timeout-related exception directly
    if (Test-IsTimeoutException -ExceptionMessage $ex.Message) {
    throw "Timeout encountered in WebRequest method"
    }

    Write-Host " → Still checking for certificate..." -ForegroundColor Yellow
    }

    # Try to get certificate from the ServicePoint
    $cert = $webRequest.ServicePoint.Certificate
    if ($cert -eq $null) {
    throw "No certificate found in ServicePoint"
    }

    # Convert to X509Certificate2
    $x509cert = New-Object System.Security.Cryptography.X509Certificates.X509Certificate2($cert)

    # Display basic certificate info
    Write-Host " → Certificate details retrieved via WebRequest:" -ForegroundColor Green
    Write-Host " Subject: $($x509cert.Subject)"
    Write-Host " Issuer: $($x509cert.Issuer)"
    Write-Host " Valid from: $($x509cert.NotBefore) to $($x509cert.NotAfter)"

    # Extract Subject and Issuer components
    $subjectParts = $x509cert.Subject -split ', '
    $issuerParts = $x509cert.Issuer -split ', '

    $subjectCN = ($subjectParts | Where-Object { $_ -match '^CN=' } | ForEach-Object { $_ -replace '^CN=' })[0]
    $issuerCN = ($issuerParts | Where-Object { $_ -match '^CN=' } | ForEach-Object { $_ -replace '^CN=' })[0]

    # Calculate days until expiration
    $daysUntilExpiration = [math]::Round(($x509cert.NotAfter - (Get-Date)).TotalDays, 1)

    # Return certificate details
    return @{
    Status = "Success"
    Method = "WebRequest"
    Protocol = "Unknown (WebRequest)"
    Subject = @{
    CN = $subjectCN
    O = ($subjectParts | Where-Object { $_ -match '^O=' } | ForEach-Object { $_ -replace '^O=' })
    OU = ($subjectParts | Where-Object { $_ -match '^OU=' } | ForEach-Object { $_ -replace '^OU=' })
    C = ($subjectParts | Where-Object { $_ -match '^C=' } | ForEach-Object { $_ -replace '^C=' })
    S = ($subjectParts | Where-Object { $_ -match '^S=' -or $_ -match '^ST=' } | ForEach-Object { $_ -replace '^S=|^ST=' })
    }
    Issuer = @{
    CN = $issuerCN
    O = ($issuerParts | Where-Object { $_ -match '^O=' } | ForEach-Object { $_ -replace '^O=' })
    OU = ($issuerParts | Where-Object { $_ -match '^OU=' } | ForEach-Object { $_ -replace '^OU=' })
    C = ($issuerParts | Where-Object { $_ -match '^C=' } | ForEach-Object { $_ -replace '^C=' })
    S = ($issuerParts | Where-Object { $_ -match '^S=' -or $_ -match '^ST=' } | ForEach-Object { $_ -replace '^S=|^ST=' })
    }
    Validity = @{
    NotBefore = $x509cert.NotBefore.ToString('yyyy-MM-dd HH:mm:ss')
    NotAfter = $x509cert.NotAfter.ToString('yyyy-MM-dd HH:mm:ss')
    DaysUntilExpiration = $daysUntilExpiration
    }
    SerialNumber = $x509cert.SerialNumber
    Thumbprint = @{
    SHA256 = $x509cert.GetCertHashString('SHA256')
    SHA1 = $x509cert.GetCertHashString('SHA1')
    }
    SignatureAlgorithm = $x509cert.SignatureAlgorithm.FriendlyName
    Version = $x509cert.Version
    SelfSigned = ($x509cert.Subject -eq $x509cert.Issuer)
    TLSSupport = @{
    SupportedProtocols = @("Unknown - WebRequest method")
    }
    }
    }
    finally {
    # Restore original settings
    [System.Net.ServicePointManager]::SecurityProtocol = $originalProtocol
    [System.Net.ServicePointManager]::ServerCertificateValidationCallback = $originalCallback
    }
    }
    catch {
    $exDetails = Get-FullExceptionDetails -Exception $_.Exception

    # Check if this was a timeout
    if (Test-IsTimeoutException -ExceptionMessage $exDetails) {
    Write-Host " → Method 2 timed out: $exDetails" -ForegroundColor Yellow
    } else {
    Write-Host " → Method 2 failed: $exDetails" -ForegroundColor Yellow
    }
    }
    }

    # If all methods fail, return a consistent failure object
    Write-Host " → Certificate retrieval failed - all methods exhausted" -ForegroundColor Yellow

    # Return a failure object
    return @{
    Status = "Failed"
    Method = "AllMethodsFailed"
    Error = "Could not retrieve certificate from ${Hostname}:${Port} after trying all methods"
    Subject = @{ CN = "Unknown"; O = @(); OU = @() }
    Issuer = @{ CN = "Unknown"; O = @(); OU = @() }
    Validity = @{
    NotBefore = "Unknown"
    NotAfter = "Unknown"
    DaysUntilExpiration = $null
    }
    SerialNumber = "Unknown"
    SelfSigned = $null
    TLSSupport = @{
    SupportedProtocols = @()
    }
    }
    }

    function Test-DNSResolution {
    param(
    [string]$Hostname,
    [string]$IP
    )

    try {
    $dnsResults = [System.Net.Dns]::GetHostEntry($Hostname)
    $ipAddresses = $dnsResults.AddressList | ForEach-Object { $_.IPAddressToString }

    # Check if the IP we have matches any of the DNS results
    $ipMatch = $ipAddresses -contains $IP

    return @{
    Resolves = $true
    IPAddresses = $ipAddresses
    IPMatch = $ipMatch
    Error = $null
    }
    }
    catch {
    return @{
    Resolves = $false
    IPAddresses = @()
    IPMatch = $false
    Error = $_.Exception.Message
    }
    }
    }

    # Enhanced URL sanitization function
    function Sanitize-Url {
    param(
    [string]$Url
    )

    if ([string]::IsNullOrWhiteSpace($Url)) {
    return $Url
    }

    # Remove any backslashes from the URL, but preserve double forward slashes
    $Url = $Url.Replace('\', '/')

    # Fix any double forward slashes that aren't part of the protocol
    if ($Url -match '^(?:https?:\/\/)') {
    $protocol = $matches[0]
    $rest = $Url.Substring($protocol.Length)
    $rest = $rest -replace '//+', '/'
    $Url = $protocol + $rest
    }

    # Ensure no trailing slashes in hostname portion
    if ($Url -match '^(https?:\/\/[^\/]+)(.*)$') {
    $hostname = $matches[1].TrimEnd('/')
    $path = $matches[2]
    $Url = $hostname + $path
    }

    return $Url.Trim()
    }

    function Get-TargetHostname {
    param(
    [string]$Hostname,
    [string]$IP
    )

    if (-not $Hostname) {
    Write-Host " No hostname provided, using IP: $IP"
    return $IP
    }

    $dnsCheck = Test-DNSResolution -Hostname $Hostname -IP $IP

    if (-not $dnsCheck.Resolves) {
    Write-Host " Hostname '$Hostname' does not resolve in DNS, using IP: $IP"
    return $IP
    }

    if (-not $dnsCheck.IPMatch) {
    Write-Host " Warning: Hostname '$Hostname' resolves to different IPs: $($dnsCheck.IPAddresses -join ', ')"
    Write-Host " Target IP '$IP' not in DNS results, using IP instead of hostname"
    return $IP
    }

    Write-Host " Hostname '$Hostname' resolves correctly to IP: $IP"
    return $Hostname
    }

    function Get-Favicon {
    param(
    [string]$BaseUrl,
    [bool]$IsSSL,
    [int]$Timeout = 10
    )

    try {
    # First try standard /favicon.ico
    $faviconUrl = "{0}/favicon.ico" -f $BaseUrl

    Write-Host " → Attempting: $faviconUrl"

    if ($IsSSL) {
    $response = Invoke-WebRequest -Uri $faviconUrl -MaximumRedirection 0 -SkipCertificateCheck -TimeoutSec $Timeout -ErrorAction SilentlyContinue
    } else {
    $response = Invoke-WebRequest -Uri $faviconUrl -MaximumRedirection 0 -TimeoutSec $Timeout -ErrorAction SilentlyContinue
    }

    if ($response.StatusCode -eq 200 -and $response.RawContentLength -gt 0) {
    return [Convert]::ToBase64String($response.Content)
    }
    } catch {
    # Don't output anything - the error is expected for sites without favicon.ico
    }

    return $null
    }

    # Enable all supported TLS/SSL versions
    Set-MaximumTlsSupport

    function Sanitize-Hostname {
    param(
    [string]$Hostname
    )
    return $Hostname.Replace('\', '').Trim()
    }

    function Fetch-Url {
    param(
    [string]$TargetHost,
    [int]$Port,
    [bool]$IsSSL,
    [int]$Timeout = 10
    )

    $MaxRedirects = 10
    $RedirectCount = 0
    $results = @()

    $protocol = if ($IsSSL) { "https" } else { "http" }
    $currentUrl = "${protocol}://${TargetHost}:${Port}"
    $originalHost = $TargetHost
    $originalPort = $Port

    # Helper function to check if an exception is timeout-related
    function Test-IsTimeoutException {
    param([string]$ExceptionMessage)

    return ($ExceptionMessage -match "timed? out|timeout|aborted|could not establish trust|operation has timed out|connection failure|connection attempt failed")
    }

    # Get certificate details early for SSL connections
    $certDetails = $null
    if ($IsSSL) {
    Write-Host " → Getting certificate details for ${TargetHost}:${Port}"
    $certDetails = Get-CertificateDetails -Hostname $TargetHost -Port $Port -Timeout $Timeout -AvailableProtocols $GlobalAvailableTlsProtocols
    }

    # Create the initial result object with certificate info
    $requestResult = [ordered]@{
    Url = $currentUrl
    RedirectDepth = $RedirectCount
    StatusCode = $null
    Error = $null
    Headers = @{}
    Content = $null
    Favicon = $null
    Title = $null
    RedirectedRequest = $null
    Certificate = $certDetails # Certificate details we've already retrieved
    }

    # Check if we should skip HTTPS connection due to certificate failure or timeout
    $skipHttpsRequest = $false

    if ($IsSSL) {
    # Check if certificate retrieval failed or timed out
    if (-not $certDetails -or $certDetails.Status -eq "Failed") {
    # Check if the error indicates a timeout
    $timeoutError = $false
    if ($certDetails -and $certDetails.Error -and (Test-IsTimeoutException -ExceptionMessage $certDetails.Error)) {
    $timeoutError = $true
    Write-Host " → Certificate retrieval timed out - skipping HTTPS connection attempt" -ForegroundColor Yellow
    $requestResult.Error = "HTTPS connection skipped due to certificate retrieval timeout"
    } else {
    Write-Host " → Certificate retrieval failed - skipping HTTPS connection attempt" -ForegroundColor Yellow
    $requestResult.Error = "HTTPS connection skipped due to certificate retrieval failure"
    }

    $results += $requestResult
    $skipHttpsRequest = $true
    }
    }

    # Only attempt HTTP/HTTPS connections if not skipped
    if (-not $skipHttpsRequest) {
    while ($true) {
    try {
    # Sanitize the current URL before making the request
    $currentUrl = Sanitize-Url -Url $currentUrl

    # Update URL in the result object
    $requestResult.Url = $currentUrl

    Write-Host " → Attempting: $currentUrl"

    # Create WebRequest to check content length first
    $webRequest = [System.Net.WebRequest]::Create($currentUrl)
    $webRequest.Method = "HEAD"
    $webRequest.Timeout = $Timeout * 1000
    if ($IsSSL) {
    # For HTTPS requests
    [System.Net.ServicePointManager]::ServerCertificateValidationCallback = { $true }
    }

    try {
    $headResponse = $webRequest.GetResponse()
    $contentLength = $headResponse.ContentLength

    if ($contentLength -gt $MaxResponseSize) {
    Write-Host " → Warning: Response too large ($([math]::Round($contentLength/1MB, 2)) MB). Skipping." -ForegroundColor Yellow
    $requestResult.Error = "Response size ($([math]::Round($contentLength/1MB, 2)) MB) exceeds maximum allowed size ($([math]::Round($MaxResponseSize/1MB)) MB)"
    $results += $requestResult
    break
    }
    } catch {
    # Check if this is a timeout exception
    if (Test-IsTimeoutException -ExceptionMessage $_.Exception.Message) {
    Write-Host " → HEAD request timed out" -ForegroundColor Yellow
    $requestResult.Error = "HEAD request timed out: $($_.Exception.Message)"
    $results += $requestResult
    break
    }

    # If HEAD request fails with non-timeout error, proceed with normal GET but limit the read size
    Write-Host " → HEAD request failed: $($_.Exception.Message). Trying GET instead." -ForegroundColor Yellow
    }

    # Use different Invoke-WebRequest parameters based on PowerShell version
    $psVersion = $PSVersionTable.PSVersion.Major
    $webRequestParams = @{
    Uri = $currentUrl
    MaximumRedirection = 0
    TimeoutSec = $Timeout
    MaximumRetryCount = 0
    ErrorAction = 'SilentlyContinue'
    }

    if ($IsSSL) {
    if ($psVersion -ge 6) {
    # For PowerShell 6+ we can use SkipCertificateCheck
    $webRequestParams.Add('SkipCertificateCheck', $true)
    $response = Invoke-WebRequest @webRequestParams
    } else {
    # For older PowerShell versions
    try {
    # Temporarily set certificate callback
    $originalCallback = [System.Net.ServicePointManager]::ServerCertificateValidationCallback
    [System.Net.ServicePointManager]::ServerCertificateValidationCallback = { $true }

    $response = Invoke-WebRequest @webRequestParams
    } finally {
    # Restore original callback
    [System.Net.ServicePointManager]::ServerCertificateValidationCallback = $originalCallback
    }
    }
    } else {
    $response = Invoke-WebRequest @webRequestParams
    }

    # Check actual content length after download
    $actualLength = if ($response.RawContentLength -gt 0) {
    $response.RawContentLength
    } else {
    [System.Text.Encoding]::UTF8.GetByteCount($response.Content)
    }

    if ($actualLength -gt $MaxResponseSize) {
    Write-Host " → Warning: Response too large ($([math]::Round($actualLength/1MB, 2)) MB). Truncating." -ForegroundColor Yellow
    $requestResult.Content = $response.Content.Substring(0, $MaxResponseSize)
    $requestResult.Error = "Response truncated: exceeded maximum size of $([math]::Round($MaxResponseSize/1MB)) MB"
    } else {
    $requestResult.Content = $response.Content
    Write-Host " → Received $([math]::Round($actualLength/1KB, 2)) KB"
    }

    # Extract title from HTML content if available
    if ($requestResult.Content) {
    if ($requestResult.Content -match '<title[^>]*>(.*?)</title>') {
    $requestResult.Title = $matches[1].Trim()
    Write-Host " → Page title: $($requestResult.Title)"
    } else {
    $requestResult.Title = $null
    Write-Host " → No title found in page"
    }
    }

    # Try to get favicon
    $favicon = Get-Favicon -BaseUrl $currentUrl -IsSSL $IsSSL -Timeout $Timeout
    if ($favicon) {
    $requestResult.Favicon = $favicon
    }

    $results += $requestResult
    break

    } catch {
    if ($_.Exception.Response) {
    $statusCode = [int]$_.Exception.Response.StatusCode
    $requestResult.StatusCode = $statusCode
    $requestResult.Headers = $_.Exception.Response.Headers
    $requestResult.Error = $_.Exception.Message

    # Try to get response content even for error status codes
    try {
    $stream = $_.Exception.Response.GetResponseStream()
    $reader = New-Object System.IO.StreamReader($stream)
    $responseContent = $reader.ReadToEnd()
    $contentLength = [System.Text.Encoding]::UTF8.GetByteCount($responseContent)
    $requestResult.Content = $responseContent
    Write-Host " → Received $contentLength bytes"

    # Extract title from HTML content if available
    if ($responseContent -match '<title[^>]*>(.*?)</title>') {
    $requestResult.Title = $matches[1].Trim()
    Write-Host " → Page title: $($requestResult.Title)"
    }
    } catch {
    # If we can't read the response content, continue with error handling
    }

    $results += $requestResult

    if ($statusCode -ge 300 -and $statusCode -lt 400) {
    if ($RedirectCount -ge $MaxRedirects) {
    Write-Host " → Warning: Maximum redirects reached" -ForegroundColor Yellow
    break
    }

    $location = $_.Exception.Response.Headers.GetValues("Location")[0]
    $location = Sanitize-Url -Url $location

    # Check if redirect goes from HTTP to HTTPS
    $isHttpsRedirect = -not $IsSSL -and ($location.StartsWith("https://") -or $location.StartsWith("//"))
    if ($isHttpsRedirect) {
    Write-Host " → Not following redirect to HTTPS: $location" -ForegroundColor Yellow
    break
    }

    try {
    if ($location.StartsWith("http://") -or $location.StartsWith("https://")) {
    $uri = [System.Uri]::new($location)
    if ($uri.Host -ne $originalHost -or $uri.Port -ne $originalPort) {
    Write-Host " → Not following redirect to different host/port: $location" -ForegroundColor Yellow
    break
    }
    $currentUrl = $location
    }
    elseif ($location.StartsWith("//")) {
    $currentUrl = "${protocol}:${location}"
    }
    elseif ($location.StartsWith("/")) {
    $currentUrl = "${protocol}://${originalHost}:${originalPort}${location}"
    }
    else {
    $baseUri = [System.Uri]$currentUrl
    $currentUrl = [System.Uri]::new($baseUri, $location).ToString()
    }

    $RedirectCount++
    Write-Host " → Following redirect to: $currentUrl"

    # Create a new request result for this redirect
    $requestResult = [ordered]@{
    Url = $currentUrl
    RedirectDepth = $RedirectCount
    StatusCode = $null
    Error = $null
    Headers = @{}
    Content = $null
    Favicon = $null
    Title = $null
    RedirectedRequest = $null
    Certificate = $certDetails
    }

    continue
    }
    catch {
    Write-Host " → Warning: Invalid redirect URL: $location" -ForegroundColor Yellow
    break
    }
    }

    Write-Host " → Warning: HTTP $statusCode - $($_.Exception.Message)" -ForegroundColor Yellow
    } else {
    # Check for timeout exception
    if (Test-IsTimeoutException -ExceptionMessage $_.Exception.Message) {
    Write-Host " → Request timed out: $($_.Exception.Message)" -ForegroundColor Yellow
    } else {
    Write-Host " → Warning: $($_.Exception.Message)" -ForegroundColor Yellow
    }

    $requestResult.Error = $_.Exception.Message
    $results += $requestResult
    }
    break
    }
    }
    }

    return $results
    }

    # Function to export JSON data to CSV in both wide and long formats
    function Export-ToCSV {
    param(
    [string]$InputDirectory,
    [string]$OutputFile,
    [ValidateSet("Long", "Wide")]
    [string]$Format = "Wide"
    )

    # Get all JSON files in the directory
    $jsonFiles = Get-ChildItem -Path $InputDirectory -Filter "*.json"
    Write-Host "Found $($jsonFiles.Count) JSON files to process for CSV export"

    # Create an array to hold all CSV data
    $csvData = @()

    if ($Format -eq "Long") {
    # Long format (normalized: one row per request/redirect)
    foreach ($file in $jsonFiles) {
    Write-Host "Processing $($file.Name)"
    $hostData = Get-Content -Path $file.FullName | ConvertFrom-Json

    # For each open port
    foreach ($port in $hostData.OpenPorts) {
    # Base row data (host and port info)
    $baseRowData = [ordered]@{
    IP = $hostData.IP
    Hostname = $hostData.Hostname
    DNSResolution = $hostData.DNSResolution.Resolves
    DNSMatchesIP = $hostData.DNSResolution.IPMatch
    Port = $port.PortNumber
    Protocol = $port.Protocol
    Service = $port.Service
    IsSSL = $port.IsSSL
    }

    # Add rows for each request in the redirect chain
    for ($i = 0; $i -lt $port.WebRequest.Count; $i++) {
    $request = $port.WebRequest[$i]

    $rowData = $baseRowData.Clone()
    $rowData += [ordered]@{
    RedirectIndex = $i
    URL = $request.Url
    StatusCode = $request.StatusCode
    Error = $request.Error
    HasContent = if ($request.Content) { $true } else { $false }
    ContentLength = if ($request.Content) { $request.Content.Length } else { 0 }
    HasFavicon = if ($request.Favicon) { $true } else { $false }
    }

    # Add certificate info for SSL connections - handle both successful and failed certificate retrievals
    if ($port.IsSSL -and $request.Certificate) {
    $cert = $request.Certificate

    # Add certificate status and method
    $rowData += [ordered]@{
    CertStatus = if ($cert.PSObject.Properties.Name -contains "Status") { $cert.Status } else { "Unknown" }
    CertMethod = if ($cert.PSObject.Properties.Name -contains "Method") { $cert.Method } else { "Unknown" }
    CertError = if ($cert.PSObject.Properties.Name -contains "Error") { $cert.Error } else { $null }
    }

    # Add certificate details if available
    if ($cert.PSObject.Properties.Name -contains "Subject" -and $cert.Subject -ne $null) {
    $rowData += [ordered]@{
    CertSubjectCN = $cert.Subject.CN
    CertIssuerCN = $cert.Issuer.CN
    CertValidFrom = $cert.Validity.NotBefore
    CertValidTo = $cert.Validity.NotAfter
    CertDaysUntilExpiration = $cert.Validity.DaysUntilExpiration
    CertSignatureAlgorithm = $cert.SignatureAlgorithm
    CertSelfSigned = $cert.SelfSigned
    CertSupportedTLSVersions = if ($cert.TLSSupport -and $cert.TLSSupport.SupportedProtocols) {
    ($cert.TLSSupport.SupportedProtocols -join ", ")
    } else {
    "Unknown"
    }
    }
    } else {
    # Add placeholder values for missing certificate details
    $rowData += [ordered]@{
    CertSubjectCN = "Unknown"
    CertIssuerCN = "Unknown"
    CertValidFrom = "Unknown"
    CertValidTo = "Unknown"
    CertDaysUntilExpiration = $null
    CertSignatureAlgorithm = "Unknown"
    CertSelfSigned = $null
    CertSupportedTLSVersions = "Unknown"
    }
    }
    } else {
    # No certificate data (not SSL or no cert)
    $rowData += [ordered]@{
    CertStatus = "NotApplicable"
    CertMethod = "NotApplicable"
    CertError = $null
    CertSubjectCN = $null
    CertIssuerCN = $null
    CertValidFrom = $null
    CertValidTo = $null
    CertDaysUntilExpiration = $null
    CertSignatureAlgorithm = $null
    CertSelfSigned = $null
    CertSupportedTLSVersions = $null
    }
    }

    # Create PSObject and add to CSV data
    $csvData += [PSCustomObject]$rowData
    }
    }
    }
    } else {
    # Wide format (one row per host+port, with columns for each redirect)

    # First pass: determine the maximum number of redirects for any response
    $maxRedirects = 0

    foreach ($file in $jsonFiles) {
    $hostData = Get-Content -Path $file.FullName | ConvertFrom-Json

    foreach ($port in $hostData.OpenPorts) {
    $redirectCount = ($port.WebRequest | Measure-Object).Count
    if ($redirectCount -gt $maxRedirects) {
    $maxRedirects = $redirectCount
    }
    }
    }

    Write-Host "Maximum redirect chain length: $maxRedirects"

    # Second pass: create the CSV data
    foreach ($file in $jsonFiles) {
    Write-Host "Processing $($file.Name)"
    $hostData = Get-Content -Path $file.FullName | ConvertFrom-Json

    # For each open port
    foreach ($port in $hostData.OpenPorts) {
    $rowData = [ordered]@{
    # Host information
    IP = $hostData.IP
    Hostname = $hostData.Hostname
    DNSResolution = $hostData.DNSResolution.Resolves
    DNSMatchesIP = $hostData.DNSResolution.IPMatch

    # Port information
    Port = $port.PortNumber
    Protocol = $port.Protocol
    Service = $port.Service
    IsSSL = $port.IsSSL
    TotalRedirects = ($port.WebRequest | Measure-Object).Count
    }

    # Add columns for each redirect in the chain
    for ($i = 0; $i -lt $maxRedirects; $i++) {
    $request = $port.WebRequest[$i]

    $prefix = "Redirect$($i+1)_"
    if ($request) {
    $rowData["${prefix}URL"] = $request.Url
    $rowData["${prefix}StatusCode"] = $request.StatusCode
    $rowData["${prefix}Error"] = $request.Error
    $rowData["${prefix}HasContent"] = if ($request.Content) { $true } else { $false }
    $rowData["${prefix}ContentLength"] = if ($request.Content) { $request.Content.Length } else { 0 }
    $rowData["${prefix}HasFavicon"] = if ($request.Favicon) { $true } else { $false }

    # Certificate information (if SSL) - handle both successful and failed certificate retrievals
    if ($port.IsSSL -and $request.Certificate) {
    $cert = $request.Certificate

    # Add certificate status and method
    $rowData["${prefix}CertStatus"] = if ($cert.PSObject.Properties.Name -contains "Status") { $cert.Status } else { "Unknown" }
    $rowData["${prefix}CertMethod"] = if ($cert.PSObject.Properties.Name -contains "Method") { $cert.Method } else { "Unknown" }
    $rowData["${prefix}CertError"] = if ($cert.PSObject.Properties.Name -contains "Error") { $cert.Error } else { $null }

    # Add certificate details if available
    if ($cert.PSObject.Properties.Name -contains "Subject" -and $cert.Subject -ne $null) {
    $rowData["${prefix}CertSubjectCN"] = $cert.Subject.CN
    $rowData["${prefix}CertIssuerCN"] = $cert.Issuer.CN
    $rowData["${prefix}CertValidFrom"] = $cert.Validity.NotBefore
    $rowData["${prefix}CertValidTo"] = $cert.Validity.NotAfter
    $rowData["${prefix}CertDaysUntilExpiration"] = $cert.Validity.DaysUntilExpiration
    $rowData["${prefix}CertSignatureAlgorithm"] = $cert.SignatureAlgorithm
    $rowData["${prefix}CertSelfSigned"] = $cert.SelfSigned
    $rowData["${prefix}CertSupportedTLSVersions"] = if ($cert.TLSSupport -and $cert.TLSSupport.SupportedProtocols) {
    ($cert.TLSSupport.SupportedProtocols -join ", ")
    } else {
    "Unknown"
    }
    } else {
    # Add placeholder values for missing certificate details
    $rowData["${prefix}CertSubjectCN"] = "Unknown"
    $rowData["${prefix}CertIssuerCN"] = "Unknown"
    $rowData["${prefix}CertValidFrom"] = "Unknown"
    $rowData["${prefix}CertValidTo"] = "Unknown"
    $rowData["${prefix}CertDaysUntilExpiration"] = $null
    $rowData["${prefix}CertSignatureAlgorithm"] = "Unknown"
    $rowData["${prefix}CertSelfSigned"] = $null
    $rowData["${prefix}CertSupportedTLSVersions"] = "Unknown"
    }
    } elseif ($port.IsSSL) {
    # SSL but no certificate information available
    $rowData["${prefix}CertStatus"] = "Missing"
    $rowData["${prefix}CertMethod"] = "NotAvailable"
    $rowData["${prefix}CertError"] = "No certificate data available"
    }
    }
    }

    # Create a PSObject and add to the CSV data array
    $csvData += [PSCustomObject]$rowData
    }
    }
    }

    # Export the data to CSV
    try {
    $csvData | Export-Csv -Path $OutputFile -NoTypeInformation -Encoding UTF8
    Write-Host "CSV report generated: $OutputFile"
    } catch {
    Write-Host "Error exporting to CSV: $($_.Exception.Message)" -ForegroundColor Red

    # Attempt to save to an alternative location if the original fails
    $alternativeFile = Join-Path -Path (Split-Path -Parent $OutputFile) -ChildPath "web_interfaces_alternative_$(Get-Date -Format 'yyyyMMdd_HHmmss').csv"
    try {
    $csvData | Export-Csv -Path $alternativeFile -NoTypeInformation -Encoding UTF8
    Write-Host "Alternative CSV report generated: $alternativeFile" -ForegroundColor Yellow
    } catch {
    Write-Host "Failed to save alternative CSV file: $($_.Exception.Message)" -ForegroundColor Red
    }
    }
    }

    # Add this function to the script
    function Append-HostToCSV {
    param(
    [string]$OutputDir,
    [string]$WideCSVPath,
    [string]$LongCSVPath,
    [PSCustomObject]$HostData
    )

    # Helper function to extract title from HTML content
    function Get-HtmlTitle {
    param([string]$HtmlContent)

    if ([string]::IsNullOrWhiteSpace($HtmlContent)) {
    return $null
    }

    try {
    # Try to match the title tag
    if ($HtmlContent -match '<title[^>]*>(.*?)</title>') {
    return $matches[1].Trim()
    }
    } catch {
    # If there's any error in regex processing, return null
    return $null
    }

    return $null
    }

    # Create wide format row(s)
    $wideRows = @()
    foreach ($port in $HostData.OpenPorts) {
    $rowData = [ordered]@{
    IP = $HostData.IP
    Hostname = $HostData.Hostname
    DNSResolution = $HostData.DNSResolution.Resolves
    DNSMatchesIP = $HostData.DNSResolution.IPMatch
    Port = $port.PortNumber
    Protocol = $port.Protocol
    Service = $port.Service
    IsSSL = $port.IsSSL
    TotalRedirects = ($port.WebRequest | Measure-Object).Count
    }

    # Add redirect information
    for ($i = 0; $i -lt $port.WebRequest.Count; $i++) {
    $request = $port.WebRequest[$i]
    $prefix = "Redirect$($i+1)_"

    # Extract title from HTML content
    $title = Get-HtmlTitle -HtmlContent $request.Content

    # Add request details
    $rowData["${prefix}URL"] = $request.Url
    $rowData["${prefix}StatusCode"] = $request.StatusCode
    $rowData["${prefix}Error"] = $request.Error
    $rowData["${prefix}HasContent"] = if ($request.Content) { $true } else { $false }
    $rowData["${prefix}ContentLength"] = if ($request.Content) { $request.Content.Length } else { 0 }
    $rowData["${prefix}HasFavicon"] = if ($request.Favicon) { $true } else { $false }
    $rowData["${prefix}Content"] = $request.Content
    $rowData["${prefix}Favicon"] = $request.Favicon
    $rowData["${prefix}Title"] = $title

    # Add headers if available
    if ($request.Headers -and $request.Headers.Count -gt 0) {
    $headerString = ($request.Headers.GetEnumerator() | ForEach-Object { "$($_.Key): $($_.Value)" }) -join "; "
    $rowData["${prefix}Headers"] = $headerString
    } else {
    $rowData["${prefix}Headers"] = ""
    }

    # Certificate information (if SSL)
    if ($port.IsSSL -and $request.Certificate) {
    $cert = $request.Certificate

    # Add certificate status and method
    $rowData["${prefix}CertStatus"] = if ($cert.PSObject.Properties.Name -contains "Status") { $cert.Status } else { "Unknown" }
    $rowData["${prefix}CertMethod"] = if ($cert.PSObject.Properties.Name -contains "Method") { $cert.Method } else { "Unknown" }
    $rowData["${prefix}CertError"] = if ($cert.PSObject.Properties.Name -contains "Error") { $cert.Error } else { $null }

    # Add certificate details if available
    if ($cert.PSObject.Properties.Name -contains "Subject" -and $cert.Subject -ne $null) {
    $rowData["${prefix}CertSubjectCN"] = $cert.Subject.CN
    $rowData["${prefix}CertIssuerCN"] = $cert.Issuer.CN
    $rowData["${prefix}CertValidFrom"] = $cert.Validity.NotBefore
    $rowData["${prefix}CertValidTo"] = $cert.Validity.NotAfter
    $rowData["${prefix}CertDaysUntilExpiration"] = $cert.Validity.DaysUntilExpiration
    $rowData["${prefix}CertSignatureAlgorithm"] = $cert.SignatureAlgorithm
    $rowData["${prefix}CertSelfSigned"] = $cert.SelfSigned
    $rowData["${prefix}CertSupportedTLSVersions"] = if ($cert.TLSSupport -and $cert.TLSSupport.SupportedProtocols) {
    ($cert.TLSSupport.SupportedProtocols -join ", ")
    } else {
    "Unknown"
    }

    # Add thumbprints
    if ($cert.Thumbprint) {
    $rowData["${prefix}CertThumbprintSHA1"] = $cert.Thumbprint.SHA1
    $rowData["${prefix}CertThumbprintSHA256"] = $cert.Thumbprint.SHA256
    }

    # Include the serial number
    $rowData["${prefix}CertSerialNumber"] = $cert.SerialNumber
    }
    }
    }

    $wideRows += [PSCustomObject]$rowData
    }

    # Create long format row(s)
    $longRows = @()
    foreach ($port in $HostData.OpenPorts) {
    # For each request in the redirect chain
    for ($i = 0; $i -lt $port.WebRequest.Count; $i++) {
    $request = $port.WebRequest[$i]

    # Extract title from HTML content
    $title = Get-HtmlTitle -HtmlContent $request.Content

    # Create a new ordered dictionary for each row
    $rowData = [ordered]@{
    IP = $HostData.IP
    Hostname = $HostData.Hostname
    DNSResolution = $HostData.DNSResolution.Resolves
    DNSMatchesIP = $HostData.DNSResolution.IPMatch
    Port = $port.PortNumber
    Protocol = $port.Protocol
    Service = $port.Service
    IsSSL = $port.IsSSL
    RedirectIndex = $i
    URL = $request.Url
    StatusCode = $request.StatusCode
    Error = $request.Error
    HasContent = if ($request.Content) { $true } else { $false }
    ContentLength = if ($request.Content) { $request.Content.Length } else { 0 }
    HasFavicon = if ($request.Favicon) { $true } else { $false }
    Content = $request.Content
    Favicon = $request.Favicon
    Title = $title
    }

    # Add headers if available
    if ($request.Headers -and $request.Headers.Count -gt 0) {
    $headerString = ($request.Headers.GetEnumerator() | ForEach-Object { "$($_.Key): $($_.Value)" }) -join "; "
    $rowData["Headers"] = $headerString
    } else {
    $rowData["Headers"] = ""
    }

    # Certificate information (if SSL)
    if ($port.IsSSL -and $request.Certificate) {
    $cert = $request.Certificate

    # Add certificate status and method
    $rowData["CertStatus"] = if ($cert.PSObject.Properties.Name -contains "Status") { $cert.Status } else { "Unknown" }
    $rowData["CertMethod"] = if ($cert.PSObject.Properties.Name -contains "Method") { $cert.Method } else { "Unknown" }
    $rowData["CertError"] = if ($cert.PSObject.Properties.Name -contains "Error") { $cert.Error } else { $null }

    # Add certificate details if available
    if ($cert.PSObject.Properties.Name -contains "Subject" -and $cert.Subject -ne $null) {
    $rowData["CertSubjectCN"] = $cert.Subject.CN
    $rowData["CertIssuerCN"] = $cert.Issuer.CN
    $rowData["CertValidFrom"] = $cert.Validity.NotBefore
    $rowData["CertValidTo"] = $cert.Validity.NotAfter
    $rowData["CertDaysUntilExpiration"] = $cert.Validity.DaysUntilExpiration
    $rowData["CertSignatureAlgorithm"] = $cert.SignatureAlgorithm
    $rowData["CertSelfSigned"] = $cert.SelfSigned
    $rowData["CertSupportedTLSVersions"] = if ($cert.TLSSupport -and $cert.TLSSupport.SupportedProtocols) {
    ($cert.TLSSupport.SupportedProtocols -join ", ")
    } else {
    "Unknown"
    }

    # Add thumbprints
    if ($cert.Thumbprint) {
    $rowData["CertThumbprintSHA1"] = $cert.Thumbprint.SHA1
    $rowData["CertThumbprintSHA256"] = $cert.Thumbprint.SHA256
    }

    # Include the serial number
    $rowData["CertSerialNumber"] = $cert.SerialNumber
    }
    }

    $longRows += [PSCustomObject]$rowData
    }
    }

    # Append to CSV files
    try {
    # If file doesn't exist, create it with headers
    if (-not (Test-Path $WideCSVPath)) {
    $wideRows | Export-Csv -Path $WideCSVPath -NoTypeInformation -Encoding UTF8
    } else {
    $wideRows | Export-Csv -Path $WideCSVPath -NoTypeInformation -Encoding UTF8 -Append -Force
    }

    if (-not (Test-Path $LongCSVPath)) {
    $longRows | Export-Csv -Path $LongCSVPath -NoTypeInformation -Encoding UTF8
    } else {
    $longRows | Export-Csv -Path $LongCSVPath -NoTypeInformation -Encoding UTF8 -Append -Force
    }

    Write-Host "CSV data appended for host: $($HostData.IP)"
    }
    catch {
    Write-Host "Error appending CSV data for host $($HostData.IP): $_" -ForegroundColor Red
    }
    }

    # Convert relative path to absolute
    $XmlPath = if ([System.IO.Path]::IsPathRooted($XmlPath)) {
    $XmlPath
    } else {
    Join-Path -Path (Get-Location) -ChildPath $XmlPath
    }

    if (-not (Test-Path $XmlPath)) {
    Write-Error "XML file not found at path: $XmlPath"
    exit 1
    }

    # Count total hosts and set MaxHosts if not specified
    $totalHosts = Get-TotalHosts -XmlPath $XmlPath
    if ($MaxHosts -eq 0) {
    $MaxHosts = $totalHosts
    Write-Host "Total hosts in file: $totalHosts"
    } else {
    Write-Host "Processing $MaxHosts of $totalHosts hosts"
    }

    $timestamp = Get-Date -Format "yyyy-MM-dd_HH-mm-ss"
    $outputDir = Join-Path -Path (Get-Location) -ChildPath "webinterface_capture_$timestamp"
    New-Item -ItemType Directory -Path $outputDir -Force | Out-Null

    $startTime = Get-Date
    $processedHosts = 0

    try {
    $settings = New-Object System.Xml.XmlReaderSettings
    $settings.DtdProcessing = [System.Xml.DtdProcessing]::Parse
    $reader = [System.Xml.XmlReader]::Create($XmlPath, $settings)

    while ($reader.Read() -and ($processedHosts -lt $MaxHosts)) {
    if ($reader.NodeType -eq [System.Xml.XmlNodeType]::Element -and $reader.Name -eq "host") {
    $hostXml = [xml]$reader.ReadOuterXml()
    $targetHost = $hostXml.host
    $hostIP = $targetHost.address | Where-Object { $_.addrtype -eq "ipv4" } | Select-Object -ExpandProperty addr
    $hostname = $targetHost.hostnames.hostname.name

    Write-Host "`nScanning host: $hostIP $(if ($hostname) { "($hostname)" })"

    $hostStartTime = Get-Date

    # Determine the appropriate hostname/IP to use based on DNS resolution
    $targetHostname = Get-TargetHostname -Hostname $hostname -IP $hostIP

    # Get all open ports from the Nmap XML
    $openPorts = @($targetHost.ports.port | Where-Object { $_.state.state -eq "open" })

    if ($openPorts.Count -eq 0) {
    Write-Host " No open ports found in Nmap results"
    $ports = @()
    } else {
    $ports = $openPorts | ForEach-Object {
    $isSSL = $false
    if ($_.service.tunnel -eq "ssl" -or $_.service.name -match "ssl|tls|https" -or $_.service.script.output -match "ssl|tls") {
    $isSSL = $true
    }

    Write-Host "Port $($_.portid) ($($_.service.name)$(if ($isSSL) { "/ssl" })):"

    $webRequests = Fetch-Url -TargetHost $targetHostname -Port $_.portid -IsSSL $isSSL -Timeout $RequestTimeout

    [PSCustomObject]@{
    PortNumber = $_.portid
    Protocol = $_.protocol
    Service = $_.service.name
    IsSSL = $isSSL
    WebRequest = $webRequests
    }
    }
    }

    $dnsCheck = Test-DNSResolution -Hostname $hostname -IP $hostIP
    $hostResult = [PSCustomObject]@{
    IP = $hostIP
    Hostname = $hostname
    DNSResolution = $dnsCheck
    OpenPorts = $ports
    ScanTime = $targetHost.starttime
    }

    $outputFile = Join-Path -Path $outputDir -ChildPath "$hostIP.json"
    $hostResult | ConvertTo-Json -Depth 10 | Out-File $outputFile

    # Generate the CSV File content for this host
    $csvWideFile = Join-Path -Path $outputDir -ChildPath "web_interfaces_wide.csv"
    $csvLongFile = Join-Path -Path $outputDir -ChildPath "web_interfaces_long.csv"
    Append-HostToCSV -OutputDir $outputDir -WideCSVPath $csvWideFile -LongCSVPath $csvLongFile -HostData $hostResult

    $processedHosts++
    $percentComplete = [math]::Round(($processedHosts / $MaxHosts) * 100, 1)

    # Calculate time estimates
    $elapsedTime = (Get-Date) - $startTime
    $averageTimePerHost = $elapsedTime.TotalSeconds / $processedHosts
    $remainingHosts = $MaxHosts - $processedHosts
    $estimatedRemainingSeconds = $averageTimePerHost * $remainingHosts
    $estimatedRemaining = [TimeSpan]::FromSeconds($estimatedRemainingSeconds)

    # Format the remaining time
    $remainingStr = Format-TimeSpan -TimeSpan $estimatedRemaining

    Write-Host ("Completed host: {0} ({1} of {2} - {3}% complete, est. {4} remaining)`n" -f
    $hostIP, $processedHosts, $MaxHosts, $percentComplete, $remainingStr)
    }
    }
    } catch {
    Write-Error "Error processing NMAP XML: $_"
    exit 1
    } finally {
    if ($reader) {
    $reader.Close()
    }
    }

    $totalTime = Format-TimeSpan -TimeSpan ((Get-Date) - $startTime)
    Write-Host "Processing complete in $totalTime. Results saved in: $outputDir"
  4. TechByTom created this gist Feb 10, 2025.
    312 changes: 312 additions & 0 deletions Web-InterfaceCategorizer.ps1
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,312 @@
    param(
    [Parameter(Mandatory=$true)]
    [string]$ScanDirectory,

    [Parameter(Mandatory=$true)]
    [string]$FingerprintsFile,

    [Parameter(Mandatory=$false)]
    [string]$OutputDirectory = ".\categorized_interfaces"
    )

    # Function to test if a string matches any pattern in an array
    function Test-Patterns {
    param(
    [string]$Text,
    [array]$Patterns
    )

    foreach ($pattern in $Patterns) {
    if ($Text -match [regex]::Escape($pattern)) {
    return $true
    }
    }
    return $false
    }

    # Function to convert PSObject to Hashtable
    function ConvertTo-Hashtable {
    param (
    [Parameter(ValueFromPipeline)]
    $InputObject
    )

    process {
    if ($null -eq $InputObject) { return $null }
    if ($InputObject -is [System.Collections.Hashtable]) { return $InputObject }

    $hash = @{}
    $InputObject.PSObject.Properties | ForEach-Object {
    $hash[$_.Name] = $_.Value
    }
    return $hash
    }
    }

    # Function to check if headers match the specified patterns
    function Test-Headers {
    param(
    [array]$Headers,
    $HeaderPatterns
    )

    # Convert PSObject to Hashtable if needed
    $headerPatternsHash = $HeaderPatterns | ConvertTo-Hashtable

    foreach ($headerPattern in $headerPatternsHash.GetEnumerator()) {
    $headerFound = $false
    foreach ($header in $Headers) {
    if ($header.Key -eq $headerPattern.Key) {
    foreach ($value in $header.Value) {
    if (Test-Patterns -Text $value -Patterns $headerPattern.Value) {
    $headerFound = $true
    break
    }
    }
    }
    }
    if (-not $headerFound) {
    return $false
    }
    }
    return $true
    }

    # Function to categorize a single interface based on fingerprints
    function Get-InterfaceCategory {
    param(
    [PSCustomObject]$InterfaceData,
    [PSCustomObject]$Fingerprints
    )

    foreach ($category in $Fingerprints.PSObject.Properties) {
    $fingerprint = $category.Value
    $matched = $true

    # Check URL patterns
    if ($fingerprint.urlPatterns) {
    $urlFound = $false
    foreach ($port in $InterfaceData.OpenPorts) {
    if ($port.WebRequest) {
    $requests = @($port.WebRequest)
    if ($requests[0] -isnot [array]) { $requests = @($requests) }

    foreach ($request in $requests) {
    if (Test-Patterns -Text $request.Url -Patterns $fingerprint.urlPatterns) {
    $urlFound = $true
    break
    }
    }
    }
    }
    if (-not $urlFound) {
    $matched = $false
    }
    }

    # Check content patterns
    if ($matched -and $fingerprint.contentPatterns) {
    $contentFound = $false
    foreach ($port in $InterfaceData.OpenPorts) {
    if ($port.WebRequest) {
    $requests = @($port.WebRequest)
    if ($requests[0] -isnot [array]) { $requests = @($requests) }

    foreach ($request in $requests) {
    if ($request.Content -and (Test-Patterns -Text $request.Content -Patterns $fingerprint.contentPatterns)) {
    $contentFound = $true
    break
    }
    }
    }
    }
    if (-not $contentFound) {
    $matched = $false
    }
    }

    # Check header patterns
    if ($matched -and $fingerprint.headerPatterns) {
    $headersFound = $false
    foreach ($port in $InterfaceData.OpenPorts) {
    if ($port.WebRequest) {
    $requests = @($port.WebRequest)
    if ($requests[0] -isnot [array]) { $requests = @($requests) }

    foreach ($request in $requests) {
    if ($request.Headers -and (Test-Headers -Headers $request.Headers -HeaderPatterns $fingerprint.headerPatterns)) {
    $headersFound = $true
    break
    }
    }
    }
    }
    if (-not $headersFound) {
    $matched = $false
    }
    }

    if ($matched) {
    return $category.Name
    }
    }

    return "Unknown"
    }

    # Function to generate HTML report
    function New-CategoryReport {
    param(
    [string]$OutputDirectory
    )

    $reportPath = Join-Path $OutputDirectory "categorization_report.html"
    $categories = Get-ChildItem -Path $OutputDirectory -Directory

    $html = @"
    <!DOCTYPE html>
    <html>
    <head>
    <title>Interface Categorization Report</title>
    <style>
    body { font-family: Arial, sans-serif; margin: 20px; }
    h1 { color: #333; }
    h2 { color: #666; margin-top: 30px; }
    .category { margin-bottom: 30px; }
    table { border-collapse: collapse; width: 100%; }
    th, td { border: 1px solid #ddd; padding: 8px; text-align: left; }
    th { background-color: #f5f5f5; }
    tr:nth-child(even) { background-color: #f9f9f9; }
    tr:hover { background-color: #f5f5f5; }
    .stats { margin: 20px 0; padding: 10px; background-color: #f8f9fa; border-radius: 4px; }
    </style>
    </head>
    <body>
    <h1>Interface Categorization Report</h1>
    <div class="stats">
    <p>Total Categories: $($categories.Count)</p>
    <p>Generated: $(Get-Date)</p>
    </div>
    "@

    foreach ($category in $categories) {
    $files = Get-ChildItem -Path $category.FullName -Filter "*.json"
    $html += @"
    <div class="category">
    <h2>$($category.Name)</h2>
    <table>
    <tr>
    <th>IP Address</th>
    <th>Hostname</th>
    <th>Ports</th>
    <th>Links</th>
    </tr>
    "@

    foreach ($file in $files) {
    $interfaceData = Get-Content $file.FullName | ConvertFrom-Json
    $ports = ($interfaceData.OpenPorts | ForEach-Object { $_.PortNumber }) -join ", "
    $links = @()

    foreach ($port in $interfaceData.OpenPorts) {
    if ($port.WebRequest) {
    $requests = @($port.WebRequest)
    if ($requests[0] -isnot [array]) { $requests = @($requests) }

    foreach ($request in $requests) {
    if ($request.Url) {
    $links += $request.Url
    }
    }
    }
    }

    $linksHtml = ($links | Select-Object -Unique | ForEach-Object {
    "<a href='$_' target='_blank'>$_</a>"
    }) -join "<br>"

    $html += @"
    <tr>
    <td>$($interfaceData.IP)</td>
    <td>$($interfaceData.Hostname)</td>
    <td>$ports</td>
    <td>$linksHtml</td>
    </tr>
    "@
    }

    $html += @"
    </table>
    </div>
    "@
    }

    $html += @"
    </body>
    </html>
    "@

    $html | Out-File -FilePath $reportPath -Encoding UTF8
    return $reportPath
    }

    # Main script execution
    try {
    # Create output directory if it doesn't exist
    if (-not (Test-Path $OutputDirectory)) {
    New-Item -ItemType Directory -Path $OutputDirectory | Out-Null
    }

    # Load fingerprints file
    $fingerprints = Get-Content $FingerprintsFile | ConvertFrom-Json

    # Track categorization results
    $results = @{}

    # Process each JSON file in the scan directory
    Get-ChildItem -Path $ScanDirectory -Filter "*.json" | ForEach-Object {
    Write-Host "Processing $($_.Name)..."

    try {
    # Read and parse JSON file
    $interfaceData = Get-Content $_.FullName | ConvertFrom-Json

    # Get category
    $category = Get-InterfaceCategory -InterfaceData $interfaceData -Fingerprints $fingerprints

    # Create category directory if it doesn't exist
    $categoryDir = Join-Path $OutputDirectory $category
    if (-not (Test-Path $categoryDir)) {
    New-Item -ItemType Directory -Path $categoryDir | Out-Null
    }

    # Move file to appropriate category directory
    Move-Item $_.FullName -Destination $categoryDir -Force

    # Track result
    if (-not $results.ContainsKey($category)) {
    $results[$category] = 0
    }
    $results[$category]++

    Write-Host "Categorized as: $category"
    }
    catch {
    Write-Warning "Error processing $($_.Name): $_"
    }
    }

    # Generate HTML report
    $reportPath = New-CategoryReport -OutputDirectory $OutputDirectory

    # Display summary
    Write-Host "`nCategorization Summary:"
    $results.GetEnumerator() | Sort-Object Name | ForEach-Object {
    Write-Host "$($_.Key): $($_.Value) interfaces"
    }

    Write-Host "`nReport generated at: $reportPath"
    }
    catch {
    Write-Error "Script execution failed: $_"
    }
    156 changes: 156 additions & 0 deletions Web-InterfaceFingerprintsExample.jsom
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,156 @@
    {
    "iDRAC": {
    "urlPatterns": [
    "/restgui/start.html",
    "/login.html",
    "/data/login"
    ],
    "contentPatterns": [
    "ng-app='loginapp'",
    "idrac-start-screen",
    "images/clarityicons"
    ],
    "headerPatterns": {
    "Server": ["Apache"],
    "Content-Security-Policy": ["default-src 'self'; connect-src *"]
    }
    },
    "CiscoIOS": {
    "urlPatterns": [
    "/level/15/exec/-/",
    "/login.html",
    "/cgi-bin/login"
    ],
    "contentPatterns": [
    "Cisco Systems",
    "routername",
    "class=\"cisco\""
    ],
    "headerPatterns": {
    "Server": ["cisco-IOS"]
    }
    },
    "HPiLO": {
    "urlPatterns": [
    "/json/login_session",
    "/html/login.html",
    "/redfish/v1"
    ],
    "contentPatterns": [
    "iLO",
    "hp-navbar",
    "integrity_footer"
    ],
    "headerPatterns": {
    "Server": ["HP-iLO-Server", "HPE-iLO-Server"],
    "X-Frame-Options": ["SAMEORIGIN"]
    }
    },
    "Synology": {
    "urlPatterns": [
    "/webman/",
    "/webapi/",
    "/DSM.php"
    ],
    "contentPatterns": [
    "SYNO.SDS",
    "SynologyDiskStation",
    "webman"
    ],
    "headerPatterns": {
    "Server": ["Synology"],
    "X-Powered-By": ["PHP"]
    }
    },
    "QNAP": {
    "urlPatterns": [
    "/cgi-bin/authLogin.cgi",
    "/cgi-bin/login.html"
    ],
    "contentPatterns": [
    "QTS Gateway",
    "QNAP",
    "qnap-loading-page"
    ],
    "headerPatterns": {
    "Server": ["http server 1.0"]
    }
    },
    "UniFiController": {
    "urlPatterns": [
    "/manage",
    "/login",
    "/api/login"
    ],
    "contentPatterns": [
    "UniFi Network",
    "ubnt-UniFi",
    "ng-controller=\"LoginController\""
    ],
    "headerPatterns": {
    "Server": ["UniFi"],
    "X-Frame-Options": ["SAMEORIGIN"]
    }
    },
    "ESXi": {
    "urlPatterns": [
    "/ui/",
    "/folder",
    "/sdk"
    ],
    "contentPatterns": [
    "VMware ESXi",
    "vsphere-client",
    "loginButton"
    ],
    "headerPatterns": {
    "Server": ["VMware"],
    "X-Frame-Options": ["DENY"]
    }
    },
    "Proxmox": {
    "urlPatterns": [
    "/pve-docs",
    "/api2/json/access/ticket",
    "/pve2/mobile.html#v"
    ],
    "contentPatterns": [
    "Proxmox",
    "pve-lang-name",
    "proxmoxlib.js"
    ],
    "headerPatterns": {
    "Server": ["pve-api-daemon", "proxmox"]
    }
    },
    "PfSense": {
    "urlPatterns": [
    "/index.php",
    "/css/pfSense.css"
    ],
    "contentPatterns": [
    "pfSense",
    "login-page",
    "fadeOut"
    ],
    "headerPatterns": {
    "X-Powered-By": ["PHP"],
    "X-Frame-Options": ["DENY"]
    }
    },
    "TrueNAS": {
    "urlPatterns": [
    "/ui/sessions/signin",
    "/api/v2.0"
    ],
    "contentPatterns": [
    "TrueNAS",
    "ix-auto",
    "freenas-login"
    ],
    "headerPatterns": {
    "Server": ["nginx"],
    "X-Frame-Options": ["SAMEORIGIN"]
    }
    }
    }
    352 changes: 352 additions & 0 deletions Web-InterfaceIdentifier-0.3.2.ps1
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,352 @@
    param(
    [Parameter(Mandatory=$true)]
    [string]$XmlPath,
    [Parameter(Mandatory=$false)]
    [int]$MaxHosts = 15,
    [Parameter(Mandatory=$false)]
    [int]$RequestTimeout = 10,
    [Parameter(Mandatory=$false)]
    [int]$MaxResponseSize = 20MB
    )

    # Function to enable all locally supported TLS/SSL versions
    function Set-MaximumTlsSupport {
    $protocols = [enum]::GetValues([System.Net.SecurityProtocolType]) |
    Where-Object { $_ -ne 'SystemDefault' }

    $supportedProtocols = $protocols | ForEach-Object {
    try {
    [System.Net.ServicePointManager]::SecurityProtocol = $_
    $_
    } catch {
    Write-Host "Protocol $_ not supported"
    $null
    }
    } | Where-Object { $_ -ne $null }

    $finalProtocol = [System.Net.SecurityProtocolType]($supportedProtocols -join ',')
    [System.Net.ServicePointManager]::SecurityProtocol = $finalProtocol
    Write-Host "Enabled protocols: $finalProtocol"
    }

    # Function definitions must come before usage
    function Test-DNSResolution {
    param(
    [string]$Hostname,
    [string]$IP
    )

    try {
    $dnsResults = [System.Net.Dns]::GetHostEntry($Hostname)
    $ipAddresses = $dnsResults.AddressList | ForEach-Object { $_.IPAddressToString }

    # Check if the IP we have matches any of the DNS results
    $ipMatch = $ipAddresses -contains $IP

    return @{
    Resolves = $true
    IPAddresses = $ipAddresses
    IPMatch = $ipMatch
    Error = $null
    }
    }
    catch {
    return @{
    Resolves = $false
    IPAddresses = @()
    IPMatch = $false
    Error = $_.Exception.Message
    }
    }
    }

    function Get-TargetHostname {
    param(
    [string]$Hostname,
    [string]$IP
    )

    if (-not $Hostname) {
    Write-Host " No hostname provided, using IP: $IP"
    return $IP
    }

    $dnsCheck = Test-DNSResolution -Hostname $Hostname -IP $IP

    if (-not $dnsCheck.Resolves) {
    Write-Host " Hostname '$Hostname' does not resolve in DNS, using IP: $IP"
    return $IP
    }

    if (-not $dnsCheck.IPMatch) {
    Write-Host " Warning: Hostname '$Hostname' resolves to different IPs: $($dnsCheck.IPAddresses -join ', ')"
    Write-Host " Target IP '$IP' not in DNS results, using IP instead of hostname"
    return $IP
    }

    Write-Host " Hostname '$Hostname' resolves correctly to IP: $IP"
    return $Hostname
    }

    function Get-Favicon {
    param(
    [string]$BaseUrl,
    [bool]$IsSSL,
    [int]$Timeout = 10
    )

    try {
    # First try standard /favicon.ico
    $faviconUrl = "{0}/favicon.ico" -f $BaseUrl

    Write-Host " → Attempting: $faviconUrl"

    if ($IsSSL) {
    $response = Invoke-WebRequest -Uri $faviconUrl -MaximumRedirection 0 -SkipCertificateCheck -TimeoutSec $Timeout -ErrorAction SilentlyContinue
    } else {
    $response = Invoke-WebRequest -Uri $faviconUrl -MaximumRedirection 0 -TimeoutSec $Timeout -ErrorAction SilentlyContinue
    }

    if ($response.StatusCode -eq 200 -and $response.RawContentLength -gt 0) {
    return [Convert]::ToBase64String($response.Content)
    }
    } catch {
    # Don't output anything - the error is expected for sites without favicon.ico
    }

    return $null
    }

    function Fetch-Url {
    param(
    [string]$TargetHost,
    [int]$Port,
    [bool]$IsSSL,
    [int]$Timeout = 10
    )

    $MaxRedirects = 10
    $RedirectCount = 0
    $results = @()

    $protocol = if ($IsSSL) { "https" } else { "http" }
    $currentUrl = "{0}://{1}:{2}" -f $protocol, $TargetHost, $Port

    while ($true) {
    try {
    $requestResult = [ordered]@{
    Url = $currentUrl
    RedirectDepth = $RedirectCount
    StatusCode = $null
    Error = $null
    Headers = @{}
    Content = $null
    Favicon = $null
    RedirectedRequest = $null
    }

    Write-Host " → Attempting: $currentUrl"

    # Create WebRequest to check content length first
    $webRequest = [System.Net.WebRequest]::Create($currentUrl)
    $webRequest.Method = "HEAD"
    $webRequest.Timeout = $Timeout * 1000
    if ($IsSSL) {
    # For HTTPS requests
    [System.Net.ServicePointManager]::ServerCertificateValidationCallback = { $true }
    }

    try {
    $headResponse = $webRequest.GetResponse()
    $contentLength = $headResponse.ContentLength

    if ($contentLength -gt $MaxResponseSize) {
    Write-Host " → Warning: Response too large ($([math]::Round($contentLength/1MB, 2)) MB). Skipping." -ForegroundColor Yellow
    $requestResult.Error = "Response size ($([math]::Round($contentLength/1MB, 2)) MB) exceeds maximum allowed size ($([math]::Round($MaxResponseSize/1MB)) MB)"
    $results += $requestResult
    break
    }
    } catch {
    # If HEAD request fails, proceed with normal GET but limit the read size
    }

    if ($IsSSL) {
    $response = Invoke-WebRequest -Uri $currentUrl -MaximumRedirection 0 -SkipCertificateCheck -TimeoutSec $Timeout -MaximumRetryCount 0 -ErrorAction SilentlyContinue
    } else {
    $response = Invoke-WebRequest -Uri $currentUrl -MaximumRedirection 0 -TimeoutSec $Timeout -MaximumRetryCount 0 -ErrorAction SilentlyContinue
    }

    # Check actual content length after download
    $actualLength = if ($response.RawContentLength -gt 0) {
    $response.RawContentLength
    } else {
    [System.Text.Encoding]::UTF8.GetByteCount($response.Content)
    }

    if ($actualLength -gt $MaxResponseSize) {
    Write-Host " → Warning: Response too large ($([math]::Round($actualLength/1MB, 2)) MB). Truncating." -ForegroundColor Yellow
    $requestResult.Content = $response.Content.Substring(0, $MaxResponseSize)
    $requestResult.Error = "Response truncated: exceeded maximum size of $([math]::Round($MaxResponseSize/1MB)) MB"
    } else {
    $requestResult.Content = $response.Content
    Write-Host " → Received $([math]::Round($actualLength/1KB, 2)) KB"
    }

    # Try to get favicon
    $favicon = Get-Favicon -BaseUrl $currentUrl -IsSSL $IsSSL -Timeout $Timeout
    if ($favicon) {
    $requestResult.Favicon = $favicon
    }

    $results += $requestResult
    break

    } catch {
    if ($_.Exception.Response) {
    $statusCode = [int]$_.Exception.Response.StatusCode
    $requestResult.StatusCode = $statusCode
    $requestResult.Headers = $_.Exception.Response.Headers
    $requestResult.Error = $_.Exception.Message

    # Try to get response content even for error status codes
    try {
    $stream = $_.Exception.Response.GetResponseStream()
    $reader = New-Object System.IO.StreamReader($stream)
    $responseContent = $reader.ReadToEnd()
    $contentLength = [System.Text.Encoding]::UTF8.GetByteCount($responseContent)
    $requestResult.Content = $responseContent
    Write-Host " → Received $contentLength bytes"
    } catch {
    # If we can't read the response content, continue with error handling
    }

    $results += $requestResult

    if ($statusCode -ge 300 -and $statusCode -lt 400) {
    if ($RedirectCount -ge $MaxRedirects) {
    Write-Host " → Warning: Maximum redirects reached" -ForegroundColor Yellow
    break
    }

    $location = $_.Exception.Response.Headers.GetValues("Location")[0]

    # Check if redirect goes from HTTP to HTTPS
    $isHttpsRedirect = -not $IsSSL -and ($location.StartsWith("https://") -or $location.StartsWith("//"))
    if ($isHttpsRedirect) {
    Write-Host " → Not following redirect to HTTPS: $location" -ForegroundColor Yellow
    break
    }

    if (-not $location.StartsWith("http")) {
    $baseUri = [System.Uri]::new($currentUrl)
    $location = "{0}://{1}:{2}{3}" -f $baseUri.Scheme, $baseUri.Host, $baseUri.Port, $location
    }

    $RedirectCount++
    Write-Host " → Following redirect to: $location"
    $currentUrl = $location
    continue
    }

    Write-Host " → Warning: HTTP $statusCode - $($_.Exception.Message)" -ForegroundColor Yellow
    } else {
    $requestResult.Error = $_.Exception.Message
    $results += $requestResult
    Write-Host " → Warning: $($_.Exception.Message)" -ForegroundColor Yellow
    }
    break
    }
    }
    return $results
    }

    # Enable all supported TLS/SSL versions
    Set-MaximumTlsSupport

    # Convert relative path to absolute
    $XmlPath = if ([System.IO.Path]::IsPathRooted($XmlPath)) {
    $XmlPath
    } else {
    Join-Path -Path (Get-Location) -ChildPath $XmlPath
    }

    if (-not (Test-Path $XmlPath)) {
    Write-Error "XML file not found at path: $XmlPath"
    exit 1
    }

    $timestamp = Get-Date -Format "yyyy-MM-dd_HH-mm-ss"
    $outputDir = Join-Path -Path (Get-Location) -ChildPath "nmap_results_$timestamp"
    New-Item -ItemType Directory -Path $outputDir -Force | Out-Null

    try {
    $settings = New-Object System.Xml.XmlReaderSettings
    $settings.DtdProcessing = [System.Xml.DtdProcessing]::Parse
    $reader = [System.Xml.XmlReader]::Create($XmlPath, $settings)
    $processedHosts = 0

    while ($reader.Read() -and ($processedHosts -lt $MaxHosts)) {
    if ($reader.NodeType -eq [System.Xml.XmlNodeType]::Element -and $reader.Name -eq "host") {
    $hostXml = [xml]$reader.ReadOuterXml()
    $targetHost = $hostXml.host
    $hostIP = $targetHost.address | Where-Object { $_.addrtype -eq "ipv4" } | Select-Object -ExpandProperty addr
    $hostname = $targetHost.hostnames.hostname.name

    Write-Host "`nScanning host: $hostIP $(if ($hostname) { "($hostname)" })"

    # Determine the appropriate hostname/IP to use based on DNS resolution
    $targetHostname = Get-TargetHostname -Hostname $hostname -IP $hostIP

    # Get all open ports from the Nmap XML
    $openPorts = @($targetHost.ports.port | Where-Object { $_.state.state -eq "open" })

    if ($openPorts.Count -eq 0) {
    Write-Host " No open ports found in Nmap results"
    $ports = @()
    } else {
    $ports = $openPorts | ForEach-Object {
    $isSSL = $false
    if ($_.service.tunnel -eq "ssl" -or $_.service.name -match "ssl|tls|https" -or $_.service.script.output -match "ssl|tls") {
    $isSSL = $true
    }

    Write-Host "Port $($_.portid) ($($_.service.name)$(if ($isSSL) { "/ssl" })):"

    $webRequests = Fetch-Url -TargetHost $targetHostname -Port $_.portid -IsSSL $isSSL -Timeout $RequestTimeout

    [PSCustomObject]@{
    PortNumber = $_.portid
    Protocol = $_.protocol
    Service = $_.service.name
    IsSSL = $isSSL
    WebRequest = $webRequests
    }
    }
    }

    $dnsCheck = Test-DNSResolution -Hostname $hostname -IP $hostIP
    $hostResult = [PSCustomObject]@{
    IP = $hostIP
    Hostname = $hostname
    DNSResolution = $dnsCheck
    OpenPorts = $ports
    ScanTime = $targetHost.starttime
    }

    $outputFile = Join-Path -Path $outputDir -ChildPath "$hostIP.json"
    $hostResult | ConvertTo-Json -Depth 10 | Out-File $outputFile

    $processedHosts++
    Write-Host "Completed host: $hostIP ($processedHosts of $MaxHosts)`n"
    }
    }
    } catch {
    Write-Error "Error processing NMAP XML: $_"
    exit 1
    } finally {
    if ($reader) {
    $reader.Close()
    }
    }

    Write-Host "Processing complete. Results saved in: $outputDir"