Work around Microsoft's new ISO download countermeasures

* The Microsoft servers now use session Id whitelisting, so add querying
  of https://vlscppe.microsoft.com/tags with the session Id.
* Closes #52.
* Also harmonize/improve -replace calls
* Also make sure we use POST for the getskuinformationbyproductedition
  query (in case Microsoft add some more countermeasures).
* Also drop 'cmd /c' invocation in sign.sh since the cygwin people can't
  seem to get their act together there...

Fido.ps1

@@ -1,6 +1,6 @@
 #
-# Fido v1.34 - Feature ISO Downloader, for retail Windows images and UEFI Shell
-# Copyright © 2019-2022 Pete Batard <pete@akeo.ie>
+# Fido v1.39 - Feature ISO Downloader, for retail Windows images and UEFI Shell
+# Copyright © 2019-2023 Pete Batard <pete@akeo.ie>
 # Command line support: Copyright © 2021 flx5
 # ConvertTo-ImageSource: Copyright © 2016 Chris Carter
 #
@@ -45,7 +45,9 @@ param(
 	# (Optional) Only display the download URL [Toggles commandline mode]
 	[switch]$GetUrl = $False,
 	# (Optional) Increase verbosity
-	[switch]$Verbose = $False
+	[switch]$Verbose = $False,
+	# (Optional) Disable the progress bar
+	[switch]$DisableProgress = $False
 )
 #endregion
 
@@ -99,9 +101,9 @@ $WindowsVersions = @(
 	@(
 		@("Windows 11", "windows11"),
 		@(
-			"22H2 (Build 22621.382 - 2022.09)",
-			@("Windows 11 Home/Pro/Edu", 2360),
-			@("Windows 11 Home China ", ($zh + 2361))
+			"22H2 v1 (Build 22621.525 - 2022.10)",
+			@("Windows 11 Home/Pro/Edu", 2370),
+			@("Windows 11 Home China ", ($zh + 2371))
 		),
 		@(
 			"21H2 v1 (Build 22000.318 - 2021.11)",
@@ -116,6 +118,11 @@ $WindowsVersions = @(
 	),
 	@(
 		@("Windows 10", "Windows10ISO"),
+		@(
+			"22H2 (Build 19045.2006 - 2022.10)",
+			@("Windows 10 Home/Pro/Edu", 2377),
+			@("Windows 10 Home China ", ($zh + 2378))
+		),
 		@(
 			"21H2 (Build 19044.1288 - 2021.11)",
 			@("Windows 10 Home/Pro/Edu", 2084),
@@ -279,6 +286,11 @@ $WindowsVersions = @(
 	),
 	@(
 		@("UEFI Shell 2.2", "UEFI_SHELL 2.2"),
+		@(
+			"22H2 (edk2-stable202211)",
+			@("Release", 0),
+			@("Debug", 1)
+		),
 		@(
 			"22H1 (edk2-stable202205)",
 			@("Release", 0),
@@ -487,7 +499,7 @@ function ConvertTo-ImageSource
 
 function Throw-Error([object]$Req, [string]$Alt)
 {
-	$Err = $(GetElementById -Request $r -Id "errorModalMessage").innerText
+	$Err = $(GetElementById -Request $Req -Id "errorModalMessage").innerText -replace "<[^>]+>" -replace "\s+", " "
 	if (-not $Err) {
 		$Err = $Alt
 	} else {
@@ -590,7 +602,7 @@ $RequestData["GetLangs"] = @("a8f8f489-4c7f-463a-9ca6-5cff94d8d041", "getskuinfo
 # This GUID applies to visitors of the en-US download page. Other locales may get a different GUID.
 $RequestData["GetLinks"] = @("6e2a1789-ef16-4f27-a296-74ef7ef5d96b", "GetProductDownloadLinksBySku" )
 # Create a semi-random Linux User-Agent string
-$FirefoxVersion = Get-Random -Minimum 50 -Maximum 90
+$FirefoxVersion = Get-Random -Minimum 90 -Maximum 110
 $FirefoxDate = Get-RandomDate
 $UserAgent = "Mozilla/5.0 (X11; Linux i586; rv:$FirefoxVersion.0) Gecko/$FirefoxDate Firefox/$FirefoxVersion.0"
 $Verbosity = 2
@@ -698,6 +710,9 @@ function Get-Windows-Languages([int]$SelectedVersion, [int]$SelectedEdition)
 	} elseif ($WindowsVersions[$SelectedVersion][0][1].StartsWith("UEFI_SHELL")) {
 		$languages += @(New-Object PsObject -Property @{ DisplayLanguage = "English (US)"; Language = "en-us"; Id = 0 })
 	} else {
+		# Microsoft download protection now requires the sessionId to be whitelisted through vlscppe.microsoft.com/tags
+		$url = "https://vlscppe.microsoft.com/tags?org_id=y6jn8c31&session_id=" + $SessionId
+		Invoke-WebRequest -UseBasicParsing -MaximumRedirection 0 -UserAgent $UserAgent $url | Out-Null
 		$url = "https://www.microsoft.com/" + $QueryLocale + "/api/controls/contentinclude/html"
 		$url += "?pageId=" + $RequestData["GetLangs"][0]
 		$url += "&host=www.microsoft.com"
@@ -712,7 +727,7 @@ function Get-Windows-Languages([int]$SelectedVersion, [int]$SelectedEdition)
 
 		$script:SelectedIndex = 0
 		try {
-			$r = Invoke-WebRequest -UseBasicParsing -UserAgent $UserAgent -SessionVariable "Session" $url
+			$r = Invoke-WebRequest -Method Post -UseBasicParsing -UserAgent $UserAgent -SessionVariable "Session" $url
 			if ($r -match "errorModalMessage") {
 				Throw-Error -Req $r -Alt "Could not retrieve languages from server"
 			}
@@ -812,7 +827,15 @@ function Get-Windows-Download-Links([int]$SelectedVersion, [int]$SelectedRelease
 			$sr = New-Object System.IO.StreamReader($wr.GetResponse().GetResponseStream())
 			$r = $sr.ReadToEnd()
 			if ($r -match "errorModalMessage") {
-				Throw-Error -Req $r -Alt "Could not retrieve architectures from server"
+				$regex = New-Object Text.RegularExpressions.Regex '<p id="errorModalMessage">(.+?)<\/p>'
+				$m = $regex.Match($r)
+				# Make the typical error message returned by Microsoft's servers more presentable
+				$Alt = $m.Groups[1] -replace "<[^>]+>" -replace "\s+", " "
+				$Alt += " " + $SessionId
+				if (-not $Alt) {
+					$Alt = "Could not retrieve architectures from server"
+				}
+				Throw-Error -Req $r -Alt $Alt
 			}
 			$pattern = '(?s)(<input.*?></input>)'
 			ForEach-Object { [regex]::Matches($r, $pattern) } | ForEach-Object { $html += $_.Groups[1].value }
@@ -860,6 +883,9 @@ function Process-Download-Link([string]$Url)
 				$tmp_size = [uint64]::Parse($str_size)
 				$Size = Size-To-Human-Readable $tmp_size
 				Write-Host "Downloading '$File' ($Size)..."
+				if ($DisableProgress) {
+					$ProgressPreference = 'SilentlyContinue'
+				}
 				Invoke-WebRequest -UseBasicParsing -Uri $Url -OutFile $File
 			} else {
 				Write-Host Download Link: $Url
@@ -941,7 +967,7 @@ if ($Cmd) {
 			if (!$Ed -and $Verbosity -ge 1) {
 				Write-Host "No edition specified (-Ed). Defaulting to '$($edition.Edition)'."
 			}
-			$Selected += "," + $edition.Edition -replace "Windows [0-9\.]*", ""
+			$Selected += "," + $edition.Edition -replace "Windows [0-9\.]*"
 			$winEditionId = $edition.Id
 			break;
 		}
@@ -1177,8 +1203,8 @@ exit $ExitCode
 # SIG # Begin signature block
 # MIIkWQYJKoZIhvcNAQcCoIIkSjCCJEYCAQExDzANBglghkgBZQMEAgEFADB5Bgor
 # BgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLG
-# KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCALq9WUjK/n7osl
-# kwRBr6duVZgEzaRe3mDKv+dLomlaQaCCElkwggVvMIIEV6ADAgECAhBI/JO0YFWU
+# KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCB4DCsid0Hcotrs
+# wgTJd2lY28Qu5omAVl5T1mk2BP1C1qCCElkwggVvMIIEV6ADAgECAhBI/JO0YFWU
 # jTanyYqJ1pQWMA0GCSqGSIb3DQEBDAUAMHsxCzAJBgNVBAYTAkdCMRswGQYDVQQI
 # DBJHcmVhdGVyIE1hbmNoZXN0ZXIxEDAOBgNVBAcMB1NhbGZvcmQxGjAYBgNVBAoM
 # EUNvbW9kbyBDQSBMaW1pdGVkMSEwHwYDVQQDDBhBQUEgQ2VydGlmaWNhdGUgU2Vy
@@ -1281,23 +1307,23 @@ exit $ExitCode
 # aWMgQ29kZSBTaWduaW5nIENBIEVWIFIzNgIRAL+xUAG79ZLUlip3l+pzb6MwDQYJ
 # YIZIAWUDBAIBBQCgfDAQBgorBgEEAYI3AgEMMQIwADAZBgkqhkiG9w0BCQMxDAYK
 # KwYBBAGCNwIBBDAcBgorBgEEAYI3AgELMQ4wDAYKKwYBBAGCNwIBFTAvBgkqhkiG
-# 9w0BCQQxIgQgFTNCPPaPOvuFXjtSmSj6kmL7aelK+dtEo03k3KPZTlQwDQYJKoZI
-# hvcNAQEBBQAEggIAeNJXRkURPIcJyxDqW8X4frFhcWLt0akkWfV2btU74/OpewpH
-# TPDADh93//9j01mOrbbFovL+5y3G/WE3EeLuzqvrfZC6g+McInSpK9wYeOEzjkfH
-# 4ACKOImDO5WNXILKtTQyfVwRuTgmWPb6SCC6HJIipciGlfuvrNcDMBjMyAbWblLe
-# OsU/N4lf1Wjd5oaM5b9kaZ7jwHLy1LderrvE0TZW7SqttfsPuqYI+QBq7ZrIxPcW
-# JKExxsWXLAAjxaSTHSTvYBE8uSA0ptEWna46mTywe6mkOHSDpnJBldwlINpNtfEl
-# wDxZ5x3phesQ71RjaE5YYupc5lt2gmpN0xz+l5Hj31wh9at8wusIYEiERWg1lBGi
-# oZZDLoNz2xjpWnPdOWaJnXKXJ7+iz28+BgjbcvMik3T0EtB41fvImq/y6UGk4vSE
-# yYQ6NFadcuCRS7yNiiP1aNw+SuZeq7rqBHlURgMP8Bck4sJIUfM3N4ONlKShBq1K
-# lq9W0IjKYVtLPSihQAbHx63DkmsJsgXlraQ9oVbhrOI5uEr03EDw1GYWrsJRiBRL
-# YQReZP6clxEoTSHqseDFm+HWmi4hv2YuVtSVobQFax5ljn5nkXRGbjIr39lE+tX6
-# loXHW4+c9R5cgyCGUCWg4HVNMSXZbNjvSVOIRxK1cDg+62XYCpAUDVCf2f2hgg49
+# 9w0BCQQxIgQgZJjDDP2YWPPmGrEuHBNI0duArydubM/gmnG4BcpOG8cwDQYJKoZI
+# hvcNAQEBBQAEggIAeMyrcbmpsFeC8p7Hw21CgIxWszJ1tkiFmVcEuf5TU5NSUwYA
+# NsT/TaKGDc0mn313EiA3OM2yBwWdJFQsF0O87carfF3FXoKNU90f/IbyzhiOuGXo
+# sT2LeKGy57hdmi+j0pcf1DorfKKQ3XQ45RI1USCedKBxaC0ZVGl1PCHogtxxwvyG
+# tV5J90NS0ztTtuL2M2JCBobE6v7p6AWxkffzb97JygXY8yKc5E7XWuUEtU7Lkv6t
+# bARG9idcAi151yNkReF+iZh1ZrS7+awhnWqmzxJ1XTf2lvZObl7YfK7CekLtjxzO
+# M1Q1iLRzbIeMX8B9Z0S6TUtKDPAzWiIKUUWBaad/ojG3uZl7fA6WEgB2mt0CQNOG
+# UAre5uYpqTpvNXCksAYcjmYmQcd2NenXSq4gnIbiPQdFXl/AMpFE127Cd7R7H/vP
+# 6a4mbsfuPuQD9sHK79hGVl86iiCq00Jur3asfbV1RGKCJZTiQXzmXRaHD8Ce8a0h
+# 2PmukEnTDNJv1t3gKMojBmT7KDXS8v6idRXVnovGrf0+0WXOCau3nBWfBAZB8A8r
+# GUS0k9JyqilE6aAE8no59wqvXIDMKLLP6Br8czBcTIAaAoMAjvfJeYCsNJZH5ICz
+# wEPxpQyGLzL/Z2LzdNUFs+j9m1zQ7a4d2ln5G5vDa/Qbs5IwGm4PRix2tqyhgg49
 # MIIOOQYKKwYBBAGCNwMDATGCDikwgg4lBgkqhkiG9w0BBwKggg4WMIIOEgIBAzEN
 # MAsGCWCGSAFlAwQCATCCAQ8GCyqGSIb3DQEJEAEEoIH/BIH8MIH5AgEBBgtghkgB
-# hvhFAQcXAzAxMA0GCWCGSAFlAwQCAQUABCD5Ric03I9pVDsoAfQBikG2Nk8QtF+C
-# jRKwiA3WiZoyAAIVAKlIW+qovWh/vTmKAgLa7ii9nmKzGA8yMDIyMDkyMzE2MDEy
-# OVowAwIBHqCBhqSBgzCBgDELMAkGA1UEBhMCVVMxHTAbBgNVBAoTFFN5bWFudGVj
+# hvhFAQcXAzAxMA0GCWCGSAFlAwQCAQUABCDJlO4Qu/KVPQXhqzZUwkQuxnuXzr+y
+# ALuw9uhTw5x50QIVAI5lXv0dlojOQ5uU1re1l1WZ0wZjGA8yMDIzMDEwODIyMDQ0
+# NFowAwIBHqCBhqSBgzCBgDELMAkGA1UEBhMCVVMxHTAbBgNVBAoTFFN5bWFudGVj
 # IENvcnBvcmF0aW9uMR8wHQYDVQQLExZTeW1hbnRlYyBUcnVzdCBOZXR3b3JrMTEw
 # LwYDVQQDEyhTeW1hbnRlYyBTSEEyNTYgVGltZVN0YW1waW5nIFNpZ25lciAtIEcz
 # oIIKizCCBTgwggQgoAMCAQICEHsFsdRJaFFE98mJ0pwZnRIwDQYJKoZIhvcNAQEL
@@ -1360,13 +1386,13 @@ exit $ExitCode
 # BgNVBAoTFFN5bWFudGVjIENvcnBvcmF0aW9uMR8wHQYDVQQLExZTeW1hbnRlYyBU
 # cnVzdCBOZXR3b3JrMSgwJgYDVQQDEx9TeW1hbnRlYyBTSEEyNTYgVGltZVN0YW1w
 # aW5nIENBAhB71OWvuswHP6EBIwQiQU0SMAsGCWCGSAFlAwQCAaCBpDAaBgkqhkiG
-# 9w0BCQMxDQYLKoZIhvcNAQkQAQQwHAYJKoZIhvcNAQkFMQ8XDTIyMDkyMzE2MDEy
-# OVowLwYJKoZIhvcNAQkEMSIEIByRkrigMw50nFG5C1eKvrGlS8ZnhBTXMhAhFl6x
-# GwufMDcGCyqGSIb3DQEJEAIvMSgwJjAkMCIEIMR0znYAfQI5Tg2l5N58FMaA+eKC
-# ATz+9lPvXbcf32H4MAsGCSqGSIb3DQEBAQSCAQBrJmQv2jU+K0w+IZbPxhLWTWse
-# LZUtQ31RIaZsNPo1Tqdx16bGqoA+JvBXffyDnnMQxVVNBrKqnPBEfff09WPeVWyn
-# SeLQJYmZPtt9ww3/9aJFmSkgeXPTJi/qBn+iymIEvYxgr1DbAY2STY05c/dwDF53
-# 760Bu3acQsT7cDMqgPxEMoIPYJRTdXXL8uBKUEZyWgHWYASpboljFSXBb0v6Hekp
-# 7hf4GPm80eOntPWpACN161wPLj9K5e7eu7/ZR9oyMWI5C/ckFnsxQgHXjEfmRK3C
-# P9iecy4hm1QE4UmDuY/DpOcodqaKT3nZ8aAdkIkHV1ayew3lrNgT8n5gfCCp
+# 9w0BCQMxDQYLKoZIhvcNAQkQAQQwHAYJKoZIhvcNAQkFMQ8XDTIzMDEwODIyMDQ0
+# NFowLwYJKoZIhvcNAQkEMSIEIJCXdWNCvCOkl1MVXUM64ID5Hk3rlqG10XOEU4X6
+# lgIdMDcGCyqGSIb3DQEJEAIvMSgwJjAkMCIEIMR0znYAfQI5Tg2l5N58FMaA+eKC
+# ATz+9lPvXbcf32H4MAsGCSqGSIb3DQEBAQSCAQCIgN/3Y1woMyQD3RfjYoePc1VS
+# M3RbvhneaXXr6LSxRh/MJd04waviGo0j/8hdoMzmyDBTVJdGuY7K2jZGFg82y5H6
+# qQAcgkTT0KyXTkT6MwjBor1nRG5IY/20NXtXIOfN3KmRQnZy+lLH3rXguxEEFObw
+# DNHTB3sAAxaJdJQhKKqw1cuExubwLEzKbU+HFvNavyd3iTf3Free1eYQYmO5LliK
+# SYWJ4CbawK2qu0U/e5xznALGZ5NdJ3tZAFK3w0ZF0W9m6CfNW0TmjG6ae8RF4J+4
+# SWGCh2KrY4PVhyKxKiB3z5rCHYGkCrNJJdlOkoaUVBa/MQ+SildBrHvnhZZ1
 # SIG # End signature block

README.md

@@ -1,5 +1,5 @@
-Fido: A PowerShell download script for Windows ISOs and UEFI Shell
-==================================================================
+Fido: A PowerShell download script for Microsoft Windows and UEFI Shell ISOs
+============================================================================
 
 [![Licence](https://img.shields.io/badge/license-GPLv3-blue.svg?style=flat-square)](https://www.gnu.org/licenses/gpl-3.0.en.html)
 [![Github stats](https://img.shields.io/github/downloads/pbatard/Fido/total.svg?style=flat-square)](https://github.com/pbatard/Fido/releases)
@@ -9,12 +9,12 @@ Description
 
 Fido is a PowerShell script that is primarily designed to be used in [Rufus](https://github.com/pbatard/rufus), but that
 can also be used in standalone fashion, and whose purpose is to automate access to the official Microsoft Windows retail
-ISO download links as well as provide convenient access to bootable UEFI Shell images.
+ISO download links as well as provide convenient access to [bootable UEFI Shell images](https://github.com/pbatard/UEFI-Shell).
 
 This script exists because, while Microsoft does make retail ISO download links freely and publicly available (at least
-for Windows 8 and Windows 10), it only does so after actively forcing users to jump through a lot of unwarranted hoops,
-that create an exceedingly counterproductive, if not downright unfriendly, consumer experience and that greatly detract
-from what people really want (direct access to ISO downloads).
+for Windows 8 through Windows 11), up until recent releases, most of these links were only available after forcing users
+to jump through a lot of unwarranted hoops that created an exceedingly counterproductive, if not downright unfriendly,
+consumer experience, that greatly detracted from what people really want (direct access to ISO downloads).
 
 As to the reason one might want to download Windows __retail__ ISOs, as opposed to the ISOs that are generated by
 Microsoft's own Media Creation Tool (MCT), this is because using official retail ISOs is currently the only way to
@@ -42,16 +42,16 @@ License
 How it works
 ------------
 
-The script basically performs the same operation as one might perform when visiting either of the following URLs (that
-is, provided that you have also changed your `User-Agent` browser string, since, when they detect that you are using a
-version of Windows that is the same as the one you are trying to download, the Microsoft web servers at these addresses
-redirect you __away__ from the pages that allow you to download retail ISOs):
+The script basically performs the same operation as one might perform when visiting the following URL (that is, in the
+case of Windows 10, provided that you have also changed your `User-Agent` browser string, since, the Microsoft web
+servers detect that you are using a version of Windows that is the same as the one you are trying to download, they
+may redirect you __away__ from the page that allows you to obtain a direct ISO download link):
 
 https://www.microsoft.com/en-us/software-download
 
-After checking access to these URLs, to confirm that they are accessible, the script first queries the web API from the
+After checking basic access to the Microsoft software downloads website the script first queries the web API from the
 Microsoft servers, to request the language selection available for the version of Windows selected, and then requests
-the actual download links for all the architectures available for that language + version.
+the actual download links, for all the architectures available for that language + version.
 
 Requirements
 ------------
@@ -87,6 +87,7 @@ The options are:
    architecture as the one from the current system.
 - `GetUrl`: By default, the script attempts to automatically launch the download. But when using the `-GetUrl` switch,
    the script only displays the download URL, which can then be piped into another command or into a file.
+- `DisableProgress`: Disable progress report. This may speed up downloads when using the command line.
 
 Examples of a commandline download:
 

sign.sh

@@ -21,7 +21,7 @@ sign_file() {
 }
 
 # Update the Authenticode signature
-cmd.exe /c '"C:\Program Files (x86)\Windows Kits\10\bin\10.0.22000.0\x64\signtool" sign /v /sha1 3dbc3a2a0e9ce8803b422cfdbc60acd33164965d /fd SHA256 /tr http://sha256timestamp.ws.symantec.com/sha256/timestamp /td SHA256 Fido.ps1'
+MSYS2_ARG_CONV_EXCL='*' "C:\Program Files (x86)\Windows Kits\10\bin\10.0.22000.0\x64\signtool" sign /v /sha1 3dbc3a2a0e9ce8803b422cfdbc60acd33164965d /fd SHA256 /tr http://sha256timestamp.ws.symantec.com/sha256/timestamp /td SHA256 Fido.ps1
 read -s -p "Enter pass phrase for `realpath $PRIVATE_KEY`: " PASSWORD
 echo
 # Confirm that the pass phrase is valid by trying to sign a dummy file