Update for Unicode 16 / TR46 rev 33

In addition to the dependency update on tr46, this includes a series of recent URL Standard changes: * whatwg/url@cd8f1d6 * whatwg/url@a6e4492 * whatwg/url@c3d173f * whatwg/url@7f3e3b6 Closes #291.

Author: domenic

README.md

@@ -4,7 +4,7 @@ whatwg-url is a full implementation of the WHATWG [URL Standard](https://url.spe
 
 ## Specification conformance
 
-whatwg-url is currently up to date with the URL spec up to commit [da212c9](https://github.com/whatwg/url/commit/da212c98f0bba9c35aec3728bbcc13f8f3a7eb52).
+whatwg-url is currently up to date with the URL spec up to commit [cfae7c4](https://github.com/whatwg/url/commit/cfae7c4ed1851886390e132fc336b653fada9123).
 
 For `file:` URLs, whose [origin is left unspecified](https://url.spec.whatwg.org/#concept-url-origin), whatwg-url chooses to use a new opaque origin (which serializes to `"null"`).
 

lib/url-state-machine.js

@@ -344,10 +344,6 @@ function parseHost(input, isOpaque = false) {
  return failure;
  }
 
- if (containsForbiddenDomainCodePoint(asciiDomain)) {
- return failure;
- }
-
  if (endsInANumber(asciiDomain)) {
  return parseIPv4(asciiDomain);
  }
@@ -429,15 +425,26 @@ function serializeHost(host) {
 
 function domainToASCII(domain, beStrict = false) {
  const result = tr46.toASCII(domain, {
+ checkHyphens: beStrict,
  checkBidi: true,
- checkHyphens: false,
  checkJoiners: true,
  useSTD3ASCIIRules: beStrict,
- verifyDNSLength: beStrict
+ transitionalProcessing: false,
+ verifyDNSLength: beStrict,
+ ignoreInvalidPunycode: false
  });
- if (result === null || result === "") {
+ if (result === null) {
  return failure;
  }
+
+ if (!beStrict) {
+ if (result === "") {
+ return failure;
+ }
+ if (containsForbiddenDomainCodePoint(result)) {
+ return failure;
+ }
+ }
  return result;
 }
 

package-lock.json

@@ -12,7 +12,7 @@
  "license": "MIT",
  "repository": "jsdom/whatwg-url",
  "dependencies": {
- "tr46": "^5.0.0",
+ "tr46": "^5.1.0",
  "webidl-conversions": "^7.0.0"
  },
  "devDependencies": {

package.json

@@ -13,7 +13,7 @@ const path = require("path");
 // 1. Go to https://github.com/web-platform-tests/wpt/tree/master/url
 // 2. Press "y" on your keyboard to get a permalink
 // 3. Copy the commit hash
-const commitHash = "a9fe2e77b64b0ce9aefb2ed4331c52bfa44fced4";
+const commitHash = "7c9c41aedbcd49bf96fb9e35a133293b3977b83d";
 
 const urlPrefix = `https://raw.githubusercontent.com/web-platform-tests/wpt/${commitHash}/url/`;
 const targetDir = path.resolve(__dirname, "..", "test", "web-platform-tests");
@@ -25,7 +25,8 @@ const resources = [
  "resources/toascii.json",
  "resources/urltestdata.json",
  "resources/urltestdata-javascript-only.json",
- "resources/IdnaTestV2.json"
+ "resources/IdnaTestV2.json",
+ "resources/IdnaTestV2-removed.json"
 ];
 
 // These tests we can download and run directly in /test/web-platform.js.
@@ -54,6 +55,7 @@ exports.directlyRunnableTests = [
 // can distinguish.
 exports.resourceDependentTests = [
  "IdnaTestV2.window.js",
+ "IdnaTestV2-removed.window.js",
  "url-constructor.any.js",
  "url-origin.any.js",
  "url-setters.any.js"

scripts/get-latest-platform-tests.js

@@ -11,6 +11,7 @@ const { utf8PercentEncodeString, isSpecialQueryPercentEncode } = require("../lib
 const { URL, URLSearchParams } = require("..");
 
 const idnaTestV2Data = require("./web-platform-tests/resources/IdnaTestV2.json");
+const idnaTestV2RemovedData = require("./web-platform-tests/resources/IdnaTestV2-removed.json");
 const urlTestData = require("./web-platform-tests/resources/urltestdata.json");
 const urlTestDataJavaScriptOnly = require("./web-platform-tests/resources/urltestdata-javascript-only.json");
 const settersData = require("./web-platform-tests/resources/setters_tests.json");
@@ -29,6 +30,7 @@ describe("Data file-based web platform tests", () => {
  resourceDependentTests,
  [
  "IdnaTestV2.window.js",
+ "IdnaTestV2-removed.window.js",
  "url-constructor.any.js",
  "url-origin.any.js",
  "url-setters.any.js"
@@ -41,6 +43,10 @@ describe("Data file-based web platform tests", () => {
  sandbox.runTests(idnaTestV2Data);
  });
 
+ runWPT("IdnaTestV2-removed.window.js", sandbox => {
+ sandbox.runTests(idnaTestV2RemovedData);
+ });
+
  runWPT("url-constructor.any.js", sandbox => {
  sandbox.runURLTests(urlTestData);
  sandbox.runURLTests(urlTestDataJavaScriptOnly);