From 7a630a9792c313d81db112419352b92909eade6e Mon Sep 17 00:00:00 2001 From: Jason Dorweiller Date: Fri, 26 Jul 2024 14:53:55 -0400 Subject: [PATCH 1/4] fix psl parsing --- src/trackers/helpers/url.js | 15 ++-- test/custom-psl-paring.test.js | 140 +++++++++++++++++++++++++++++++++ 2 files changed, 150 insertions(+), 5 deletions(-) create mode 100644 test/custom-psl-paring.test.js diff --git a/src/trackers/helpers/url.js b/src/trackers/helpers/url.js index 899e8c8..b93b470 100644 --- a/src/trackers/helpers/url.js +++ b/src/trackers/helpers/url.js @@ -15,7 +15,7 @@ class ParsedURL extends URL { } get domainInfo() { - // extend domainInfo to use PSL + // extend domainInfo to use PSL if (!this._domainInfo) { this._domainInfo = parse(this.hostname, { extractHostname: false, @@ -25,7 +25,12 @@ class ParsedURL extends URL { if (!this._domainInfo.isPrivate && pslExtras && pslExtras.privatePSL) { // get list of possible suffix matches, we can have multiple matches for a single request // a.example.com and b.a.example.com for a request from 123.b.a.example.com - const suffixMatches = pslExtras.privatePSL.filter(suffix => this._domainInfo.hostname.match(suffix)) + // check that suffix is preceded by a dot or is at the beginning of the hostname + const suffixMatches = pslExtras.privatePSL.filter(suffix => { + const escapedSuffix = suffix.replace('.', '\\.'); + const regex = new RegExp(`(^|\\.)${escapedSuffix}$`); + return regex.test(this._domainInfo.hostname); + }); // reformat domainInfo to make this request look like a private domain if (suffixMatches && suffixMatches.length) { @@ -34,13 +39,13 @@ class ParsedURL extends URL { const suffix = suffixMatches.reduce((l,s) => { return l.length >= s.length ? l : s }) - + // Array of subdomain after removing suffix from hostname - const splitSubdomain = this._domainInfo.hostname.replace(suffix, '').replace(/\.$/,'').split('.') + const splitSubdomain = this._domainInfo.hostname.replace(new RegExp(`\\.?${suffix}$`), '').split('.'); const domainWithoutSuffix = splitSubdomain.pop() this._domainInfo.publicSuffix = suffix - this._domainInfo.domain = `${domainWithoutSuffix}.${suffix}` + this._domainInfo.domain = domainWithoutSuffix ? `${domainWithoutSuffix}.${suffix}` : suffix; this._domainInfo.domainWithoutSuffix = domainWithoutSuffix this._domainInfo.subdomain = splitSubdomain.join('.') this._domainInfo.isPrivate = true diff --git a/test/custom-psl-paring.test.js b/test/custom-psl-paring.test.js new file mode 100644 index 0000000..e2f3afd --- /dev/null +++ b/test/custom-psl-paring.test.js @@ -0,0 +1,140 @@ +const assert = require('assert'); +const URL = require('../src/trackers/helpers/url.js'); + +const testCases = [ + { + input: "media-rockstargames-com.akamaized.net", + expectedOutput: { + publicSuffix: "akamaized.net", + domain: "media-rockstargames-com.akamaized.net", + domainWithoutSuffix: "media-rockstargames-com", + subdomain: "", + isPrivate: true + } + }, + { + input: "sub.media-rockstargames-com.akamaized.net", + expectedOutput: { + publicSuffix: "akamaized.net", + domain: "media-rockstargames-com.akamaized.net", + domainWithoutSuffix: "media-rockstargames-com", + subdomain: "sub", + isPrivate: true + } + }, + { + input: "example.akamaihd.net", + expectedOutput: { + publicSuffix: "akamaihd.net", + domain: "example.akamaihd.net", + domainWithoutSuffix: "example", + subdomain: "", + isPrivate: true + } + }, + { + input: "a.b.c.d.e.example.akamaihd.net", + expectedOutput: { + publicSuffix: "akamaihd.net", + domain: "example.akamaihd.net", + domainWithoutSuffix: "example", + subdomain: "a.b.c.d.e", + isPrivate: true + } + }, + { + input: "example.com", + expectedOutput: { + publicSuffix: "com", + domain: "example.com", + domainWithoutSuffix: "example", + subdomain: "", + isPrivate: false + } + }, + { + input: "sub.test.edgekey-staging.net", + expectedOutput: { + publicSuffix: "test.edgekey-staging.net", + domain: "sub.test.edgekey-staging.net", + domainWithoutSuffix: "sub", + subdomain: "", + isPrivate: true + } + }, + { + input: "bucket.s3.us-west-2.amazonaws.com", + expectedOutput: { + publicSuffix: "s3.us-west-2.amazonaws.com", + domain: "bucket.s3.us-west-2.amazonaws.com", + domainWithoutSuffix: "bucket", + subdomain: "", + isPrivate: true + } + }, + { + input: "example.ssl.global.fastly.net", + expectedOutput: { + publicSuffix: "ssl.global.fastly.net", + domain: "example.ssl.global.fastly.net", + domainWithoutSuffix: "example", + subdomain: "", + isPrivate: true + } + }, + { + input: "example.x.incapdns.net", + expectedOutput: { + publicSuffix: "x.incapdns.net", + domain: "example.x.incapdns.net", + domainWithoutSuffix: "example", + subdomain: "", + isPrivate: true + } + }, + { + input: "example.trafficmanager.net", + expectedOutput: { + publicSuffix: "trafficmanager.net", + domain: "example.trafficmanager.net", + domainWithoutSuffix: "example", + subdomain: "", + isPrivate: true + } + }, + { + input: "akamaized.net", + expectedOutput: { + publicSuffix: "akamaized.net", + domain: "akamaized.net", + domainWithoutSuffix: "", + subdomain: "", + isPrivate: true + } + }, + { + input: "example.com.akamaized.net", + expectedOutput: { + publicSuffix: "com.akamaized.net", + domain: "example.com.akamaized.net", + domainWithoutSuffix: "example", + subdomain: "", + isPrivate: true + } + } +]; + +describe('PSL domain parsing', () => { + describe('parse domain with custom PSL', () => { + testCases.forEach(({ input, expectedOutput }) => { + it(`should correctly parse ${input}`, () => { + const result = new URL('https://' + input).domainInfo + assert.strictEqual(result.publicSuffix, expectedOutput.publicSuffix) + assert.strictEqual(result.domain, expectedOutput.domain) + assert.strictEqual(result.domainWithoutSuffix, expectedOutput.domainWithoutSuffix) + assert.strictEqual(result.subdomain, expectedOutput.subdomain) + assert.strictEqual(result.isPrivate, expectedOutput.isPrivate) + }); + }); + }); +}); From e13b8458014b2504cdefa99fc16dc55eaea58d7c Mon Sep 17 00:00:00 2001 From: Jason Dorweiller Date: Fri, 26 Jul 2024 14:59:54 -0400 Subject: [PATCH 2/4] linter --- src/trackers/helpers/url.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/trackers/helpers/url.js b/src/trackers/helpers/url.js index b93b470..0b9f7c5 100644 --- a/src/trackers/helpers/url.js +++ b/src/trackers/helpers/url.js @@ -27,9 +27,9 @@ class ParsedURL extends URL { // a.example.com and b.a.example.com for a request from 123.b.a.example.com // check that suffix is preceded by a dot or is at the beginning of the hostname const suffixMatches = pslExtras.privatePSL.filter(suffix => { - const escapedSuffix = suffix.replace('.', '\\.'); - const regex = new RegExp(`(^|\\.)${escapedSuffix}$`); - return regex.test(this._domainInfo.hostname); + const escapedSuffix = suffix.replace('.', '\\.') + const regex = new RegExp(`(^|\\.)${escapedSuffix}$`) + return regex.test(this._domainInfo.hostname) }); // reformat domainInfo to make this request look like a private domain @@ -41,11 +41,11 @@ class ParsedURL extends URL { }) // Array of subdomain after removing suffix from hostname - const splitSubdomain = this._domainInfo.hostname.replace(new RegExp(`\\.?${suffix}$`), '').split('.'); + const splitSubdomain = this._domainInfo.hostname.replace(new RegExp(`\\.?${suffix}$`), '').split('.') const domainWithoutSuffix = splitSubdomain.pop() this._domainInfo.publicSuffix = suffix - this._domainInfo.domain = domainWithoutSuffix ? `${domainWithoutSuffix}.${suffix}` : suffix; + this._domainInfo.domain = domainWithoutSuffix ? `${domainWithoutSuffix}.${suffix}` : suffix this._domainInfo.domainWithoutSuffix = domainWithoutSuffix this._domainInfo.subdomain = splitSubdomain.join('.') this._domainInfo.isPrivate = true From 1d957fd94fda409dee25bc1c47ce215f4ec497de Mon Sep 17 00:00:00 2001 From: Jason Dorweiller Date: Fri, 26 Jul 2024 15:01:01 -0400 Subject: [PATCH 3/4] linter --- src/trackers/helpers/url.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/trackers/helpers/url.js b/src/trackers/helpers/url.js index 0b9f7c5..777b564 100644 --- a/src/trackers/helpers/url.js +++ b/src/trackers/helpers/url.js @@ -30,7 +30,7 @@ class ParsedURL extends URL { const escapedSuffix = suffix.replace('.', '\\.') const regex = new RegExp(`(^|\\.)${escapedSuffix}$`) return regex.test(this._domainInfo.hostname) - }); + }) // reformat domainInfo to make this request look like a private domain if (suffixMatches && suffixMatches.length) { From c56b5eb9f934b88c737739ad0cf3a81571f3e000 Mon Sep 17 00:00:00 2001 From: Jason Dorweiller Date: Mon, 29 Jul 2024 10:24:30 -0400 Subject: [PATCH 4/4] optional test --- test/custom-psl-paring.test.js | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/test/custom-psl-paring.test.js b/test/custom-psl-paring.test.js index e2f3afd..aac9e17 100644 --- a/test/custom-psl-paring.test.js +++ b/test/custom-psl-paring.test.js @@ -1,5 +1,6 @@ const assert = require('assert'); const URL = require('../src/trackers/helpers/url.js'); +const sharedData = require('../src/trackers/helpers/sharedData') const testCases = [ { @@ -128,12 +129,16 @@ describe('PSL domain parsing', () => { describe('parse domain with custom PSL', () => { testCases.forEach(({ input, expectedOutput }) => { it(`should correctly parse ${input}`, () => { - const result = new URL('https://' + input).domainInfo - assert.strictEqual(result.publicSuffix, expectedOutput.publicSuffix) - assert.strictEqual(result.domain, expectedOutput.domain) - assert.strictEqual(result.domainWithoutSuffix, expectedOutput.domainWithoutSuffix) - assert.strictEqual(result.subdomain, expectedOutput.subdomain) - assert.strictEqual(result.isPrivate, expectedOutput.isPrivate) + if (!sharedData.config.pslExtras) { + it.skip('No custom PSL data provided') + } else { + const result = new URL('https://' + input).domainInfo + assert.strictEqual(result.publicSuffix, expectedOutput.publicSuffix) + assert.strictEqual(result.domain, expectedOutput.domain) + assert.strictEqual(result.domainWithoutSuffix, expectedOutput.domainWithoutSuffix) + assert.strictEqual(result.subdomain, expectedOutput.subdomain) + assert.strictEqual(result.isPrivate, expectedOutput.isPrivate) + } }); }); });