Merge pull request #1705 from open-domains/Kyro3400-patch-1

fix: Fix everything
This commit is contained in:
andrewstech authored and GitHub committed 2024-11-23 12:03:38 +00:00
commit 41f4d32067
57 files changed
+2986 -29

No files matched your search

+36 -12
View File
@@ -1,17 +1,29 @@
name: Validation
on:
pull_request_target:
pull_request:
push:
branches: [main]
paths:
- "domains/*"
- "tests/*"
- "utils/*"
- ".github/workflows/validate.yml"
- "dnsconfig.js"
workflow_dispatch:
concurrency:
group: ${{ github.ref }}-validation
cancel-in-progress: true
jobs:
dns:
name: DNS
runs-on: ubuntu-latest
steps:
name: DNS
if: "!contains(github.event.head_commit.message, '[skip-ci]')"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
ref: ${{ github.event.pull_request.head.sha }}
@@ -24,17 +36,17 @@ jobs:
- name: Check DNS
id: dns_check
uses: koenrh/dnscontrol-action@v3
uses: is-a-dev/dnscontrol-action@main
with:
args: check
config_file: "dnsconfig.js"
args: check
config_file: "dnsconfig.js"
- name: Add Validated DNS Label
if: ${{ steps.dns_check.outcome == 'success' }}
uses: actions-ecosystem/action-add-labels@v1
with:
labels: "Validated DNS"
github_token: ${{ steps.create_token.outputs.token }}
labels: "Validated DNS"
github_token: ${{ steps.create_token.outputs.token }}
- name: Remove Invalid DNS Label
if: ${{ steps.dns_check.outcome == 'success' }}
uses: actions-ecosystem/action-remove-labels@v1
@@ -48,12 +60,13 @@ jobs:
with:
labels: "Invalid DNS"
github_token: ${{ steps.create_token.outputs.token }}
- name: Remove Validated DNS Label
if: ${{ failure() }}
uses: actions-ecosystem/action-remove-labels@v1
with:
labels: "Validated DNS"
github_token: ${{ steps.create_token.outputs.token }}
labels: "Validated DNS"
github_token: ${{ steps.create_token.outputs.token }}
json:
name: JSON
@@ -102,3 +115,14 @@ jobs:
with:
labels: "Validated JSON"
github_token: ${{ steps.create_token.outputs.token }}
tests:
name: Tests
if: "!contains(github.event.head_commit.message, '[skip-ci]')"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- run: npm install
- run: npm test
+1 -1
View File
@@ -10,5 +10,5 @@
"TXT": ["zoho-verification=zb65903823.zmverify.zoho.in"]
},
"proxied": true
"proxied": false
}
File renamed without changes.
File renamed without changes.
@@ -10,5 +10,5 @@
"TXT": ["c974df92b9224148ebb38b910ec2ad"]
},
"proxied": true
"proxied": false
}
@@ -12,5 +12,5 @@
"TXT": ["76e43257be0fa58011fb122c38f31a"]
},
"proxied": true
"proxied": false
}
File renamed without changes.
+1 -1
View File
@@ -12,4 +12,4 @@
},
"proxied": true
}
}
+1 -1
View File
@@ -10,4 +10,4 @@
"CNAME": "ss.avnmsoft.xyz"
},
"proxied": true
}
}
+1 -1
View File
@@ -15,5 +15,5 @@
]
},
"proxied": true
"proxied": false
}
+1 -1
View File
@@ -8,7 +8,7 @@
},
"record": {
"URL": "https://discog.opensourceforce.net"
"CNAME": "discog.opensourceforce.net"
},
"proxied": false
+1 -1
View File
@@ -13,5 +13,5 @@
"NS": ["arushi.ns.cloudflare.com", "duke.ns.cloudflare.com"]
},
"proxied": true
"proxied": false
}
+1 -1
View File
@@ -10,5 +10,5 @@
"TXT": ["76e43257be0fa58011fb122c38f31a"]
},
"proxied": true
"proxied": false
}
+1 -1
View File
@@ -10,7 +10,7 @@
},
"record": {
"CNAME": ["site.namedhosting.com"]
"CNAME": "site.namedhosting.com"
},
"proxied": false
+1 -1
View File
@@ -10,5 +10,5 @@
"TXT": ["google-site-verification=qjismw0ulu2edzv896w55m7cuybjku9_mfscdobkary"]
},
"proxied": true
"proxied": false
}
+1 -1
View File
@@ -9,7 +9,7 @@
},
"record": {
"CNAME": ["013c5987-dd17-459e-9cbc-d94ae0540f0e.id.repl.co"]
"CNAME": "013c5987-dd17-459e-9cbc-d94ae0540f0e.id.repl.co"
},
"proxied": false
File renamed without changes.
+1 -1
View File
@@ -14,5 +14,5 @@
"TXT": ["v=spf1 include:spf.improvmx.com ~all"]
},
"proxied": true
"proxied": false
}
+1 -1
View File
@@ -14,5 +14,5 @@
"TXT": ["v=spf1 include:spf.improvmx.com ~all"]
},
"proxied": true
"proxied": false
}
+1 -1
View File
@@ -13,7 +13,7 @@
"NS": ["ns01.000webhost.com", "ns02.000webhost.com"]
},
"proxied": true
"proxied": false
}
@@ -8,7 +8,7 @@
},
"record": {
"CNAME": ["sig1.dkim.sure.is-cool.dev.at.icloudmailadmin.com"]
"CNAME": "sig1.dkim.sure.is-cool.dev.at.icloudmailadmin.com"
},
"proxied": false
+1 -1
View File
@@ -9,7 +9,7 @@
},
"record": {
"URL": "https://zemerik.is-a.dev"
"CNAME": "zemerik.is-a.dev"
},
"proxied": false
+2437
View File
File diff suppressed because it is too large. Load diff
+9
View File
@@ -0,0 +1,9 @@
{
"devDependencies": {
"ava": "^6.2.0",
"fs-extra": "^11.2.0"
},
"scripts": {
"test": "npx ava tests/*.test.js"
}
}
+60
View File
@@ -0,0 +1,60 @@
const t = require("ava");
const fs = require("fs-extra");
const path = require("path");
const domainsPath = path.resolve("domains");
const files = fs.readdirSync(domainsPath);
t("Nested subdomains should not exist without a parent subdomain", (t) => {
files.forEach((file) => {
// Skip directories and process only .json files
const filePath = path.join(domainsPath, file);
if (fs.lstatSync(filePath).isDirectory()) {
return; // Skip directories
}
const subdomain = file.replace(".json", "");
if (subdomain.split(".").length > 1) {
// Get parent domain by removing the last part (subdomain) from the full subdomain
const parentSubdomain = subdomain.split(".").slice(1).join(".");
// Ensure the parent subdomain exists
t.true(
files.includes(`${parentSubdomain}.json`),
`${file}: Parent subdomain ${parentSubdomain}.json does not exist`
);
}
});
t.pass();
});
t("Nested subdomains should not exist if the parent subdomain has NS records", (t) => {
files.forEach((file) => {
// Skip directories and process only .json files
const filePath = path.join(domainsPath, file);
if (fs.lstatSync(filePath).isDirectory()) {
return; // Skip directories
}
const subdomain = file.replace(".json", "");
if (subdomain.split(".").length > 1) {
// Get parent domain by removing the last part (subdomain) from the full subdomain
const parentSubdomain = subdomain.split(".").slice(1).join(".");
const parentFilePath = path.join(domainsPath, `${parentSubdomain}.json`);
// Check if the parent file exists before attempting to read it
if (fs.existsSync(parentFilePath)) {
const parentDomain = fs.readJsonSync(parentFilePath);
t.is(parentDomain.record.NS, undefined, `${file}: Parent subdomain has NS records`);
} else {
t.fail(`${parentSubdomain}.json file does not exist`);
}
}
});
t.pass();
});
+101
View File
@@ -0,0 +1,101 @@
const t = require("ava");
const fs = require("fs-extra");
const path = require("path");
const requiredFields = {
//owner: "object",
record: "object"
};
const optionalOwnerFields = {
email: "string"
};
const emailRegex = /^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$/;
const hostnameRegex = /^(?=.{1,253}$)(?:(?:[_a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)\.)+[a-zA-Z]{2,63}$/;
const domainsPath = path.resolve("domains");
const reservedDomainsPath = path.resolve("reserved");
const rootDomainFiles = ["is-a-fullstack.dev.json", "is-cool.dev.json", "is-local.org.json", "is-not-a.dev.json", "localplayer.dev.json"];
const files = fs.readdirSync(domainsPath);
const reservedFiles = fs.existsSync(reservedDomainsPath) ? fs.readdirSync(reservedDomainsPath) : [];
const Domains = [".is-a-fullstack.dev", ".is-cool.dev", ".is-local.org", ".is-not-a.dev", ".localplayer.dev"];
function validateRequiredFields(t, obj, requiredFields, file) {
Object.keys(requiredFields).forEach((key) => {
t.true(obj.hasOwnProperty(key), `${file}: Missing required field: ${key}`);
t.is(typeof obj[key], requiredFields[key], `${file}: Field ${key} should be of type ${requiredFields[key]}`);
});
}
function validateOptionalFields(t, obj, optionalFields, file) {
Object.keys(optionalFields).forEach((key) => {
if (obj.hasOwnProperty(key)) {
t.is(
typeof obj[key],
optionalFields[key],
`${file}: Field ${key} should be of type ${optionalFields[key]}`
);
}
});
}
t("All files should be valid JSON", (t) => {
files.forEach((file) => {
t.notThrows(() => fs.readJsonSync(path.join(domainsPath, file)), `${file}: Invalid JSON file`);
});
});
t("All files should have valid file names", (t) => {
files.forEach((file) => {
t.true(file.endsWith(".json"), `${file}: File does not have .json extension`);
// Check for any unwanted domain in file names
t.false(Domains.some(domain => file.endsWith(domain + ".json")), `${file}: File name should not contain restricted domain extensions`);
t.true(file === file.toLowerCase(), `${file}: File name should be lowercase`);
// Ignore root domain
if (!rootDomainFiles.includes(file)) {
t.regex(
file.replace(/\.json$/, ""),
hostnameRegex,
`${file}: FQDN must be 1-253 characters, use letters, numbers, dots, or hyphens, and not start or end with a hyphen.`
);
}
});
});
t("All files should have the required fields", (t) => {
const files = fs.readdirSync(domainsPath).filter(file => {
const filePath = path.join(domainsPath, file);
return fs.lstatSync(filePath).isFile(); // Ensure only files are included
});
files.forEach((file) => {
const data = fs.readJsonSync(path.join(domainsPath, file));
validateRequiredFields(t, data, requiredFields, file);
//validateRequiredFields(t, data.owner, requiredFields, file);
if (!data.reserved) {
t.true(Object.keys(data.record).length > 0, `${file}: No record types found`);
}
});
});
t("All files should have valid optional owner fields", (t) => {
const files = fs.readdirSync(domainsPath).filter(file => {
const filePath = path.join(domainsPath, file);
return fs.lstatSync(filePath).isFile(); // Only include files
});
files.forEach((file) => {
const data = fs.readJsonSync(path.join(domainsPath, file));
validateOptionalFields(t, data, optionalOwnerFields, file);
if (data.owner.email) {
t.regex(data.owner.email, emailRegex, `${file}: Owner email should be a valid email address`);
}
});
});
+30
View File
@@ -0,0 +1,30 @@
const t = require("ava");
const fs = require("fs-extra");
const path = require("path");
const requiredRecordsToProxy = ["A", "AAAA", "CNAME"];
function validateProxiedRecords(t, data, file) {
if (data.proxied) {
const hasProxiedRecord = Object.keys(data.record || {}).some((key) => requiredRecordsToProxy.includes(key));
t.true(hasProxiedRecord, `${file}: Proxied is true but there are no records that can be proxied`);
}
}
const domainsPath = path.resolve("domains");
// Filter to only include files (exclude directories)
const files = fs.readdirSync(domainsPath).filter((file) => {
const filePath = path.join(domainsPath, file);
return fs.lstatSync(filePath).isFile();
});
t("Domains with proxy enabled should have at least one record that can be proxied", (t) => {
files.forEach((file) => {
const filePath = path.join(domainsPath, file);
const domain = fs.readJsonSync(filePath);
validateProxiedRecords(t, domain, file);
});
});
+296
View File
@@ -0,0 +1,296 @@
const t = require("ava");
const fs = require("fs-extra");
const path = require("path");
const validRecordTypes = ["A", "AAAA", "CAA", "CNAME", "MX", "NS", "SPF", "SRV", "TXT"];
const hostnameRegex = /^(?=.{1,253}$)(?:(?:[_a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)\.)+[a-zA-Z]{2,63}$/;
const ipv4Regex = /^(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])(\.(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])){3}$/;
const ipv6Regex =
/^(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}$|^::(?:[0-9a-fA-F]{1,4}:){0,6}[0-9a-fA-F]{1,4}$|^(?:[0-9a-fA-F]{1,4}:){1,7}:$|^(?:[0-9a-fA-F]{1,4}:){0,6}::(?:[0-9a-fA-F]{1,4}:){0,5}[0-9a-fA-F]{1,4}$/;
const domainsPath = path.resolve("domains");
const files = fs.readdirSync(domainsPath);
function expandIPv6(ip) {
// Split into segments by ":"
let segments = ip.split(":");
// Count the number of segments that are empty due to "::" shorthand
const emptyIndex = segments.indexOf("");
if (emptyIndex !== -1) {
// Calculate how many "0000" segments are missing
const nonEmptySegments = segments.filter((seg) => seg !== "");
const missingSegments = 8 - nonEmptySegments.length;
// Insert the missing "0000" segments into the position of the empty segment
segments = [
...nonEmptySegments.slice(0, emptyIndex),
...Array(missingSegments).fill("0000"),
...nonEmptySegments.slice(emptyIndex)
];
}
// Expand each segment to 4 characters, padding with leading zeros
const expandedSegments = segments.map((segment) => segment.padStart(4, "0"));
// Join the segments back together
return expandedSegments.join(":");
}
function isPublicIPv4(ip, proxied) {
const parts = ip.split(".").map(Number);
// Validate IPv4 address format
if (parts.length !== 4 || parts.some((part) => isNaN(part) || part < 0 || part > 255)) {
return false;
}
// Exception for 192.0.2.1, assuming the domain is proxied
if (ip === "192.0.2.1" && proxied) {
return true;
}
// Check for private and reserved IPv4 ranges
return !(
// Private ranges
(
parts[0] === 10 ||
(parts[0] === 172 && parts[1] >= 16 && parts[1] <= 31) ||
(parts[0] === 192 && parts[1] === 168) ||
// Reserved or special-use ranges
(parts[0] === 100 && parts[1] >= 64 && parts[1] <= 127) || // Carrier-grade NAT
(parts[0] === 169 && parts[1] === 254) || // Link-local
(parts[0] === 192 && parts[1] === 0 && parts[2] === 0) || // IETF Protocol Assignments
(parts[0] === 192 && parts[1] === 0 && parts[2] === 2) || // Documentation (TEST-NET-1)
(parts[0] === 198 && parts[1] === 18) || // Network Interconnect Devices
(parts[0] === 198 && parts[1] === 51 && parts[2] === 100) || // Documentation (TEST-NET-2)
(parts[0] === 203 && parts[1] === 0 && parts[2] === 113) || // Documentation (TEST-NET-3)
parts[0] >= 224
) // Multicast and reserved ranges
);
}
function isPublicIPv6(ip) {
const normalizedIP = ip.toLowerCase();
// Check for private or special-use IPv6 ranges
return !(
(
normalizedIP.startsWith("fc") || // Unique Local Address (ULA)
normalizedIP.startsWith("fd") || // Unique Local Address (ULA)
normalizedIP.startsWith("fe80") || // Link-local
normalizedIP.startsWith("::1") || // Loopback address (::1)
normalizedIP.startsWith("2001:db8")
) // Documentation range
);
}
function isDirectory(filePath) {
return fs.lstatSync(filePath).isDirectory();
}
t("All files should have valid record types", (t) => {
files.forEach((file) => {
const filePath = path.join(domainsPath, file);
// Skip directories
if (isDirectory(filePath)) {
return;
}
const data = fs.readJsonSync(filePath);
const recordKeys = Object.keys(data.record);
recordKeys.forEach((key) => {
t.true(validRecordTypes.includes(key), `${file}: Invalid record type: ${key}`);
});
// CNAME records cannot be combined with any other record type
if (recordKeys.includes("CNAME")) {
t.is(recordKeys.length, 1, `${file}: CNAME records cannot be combined with other records`);
}
// NS records cannot be combined with any other record type, except for DS records
if (recordKeys.includes("NS")) {
t.true(
recordKeys.length === 1 || (recordKeys.length === 2 && recordKeys.includes("DS")),
`${file}: NS records cannot be combined with other records, except for DS records`
);
}
});
});
function isDirectory(filePath) {
return fs.lstatSync(filePath).isDirectory();
}
t("All files should not have duplicate record keys", (t) => {
files.forEach((file) => {
const filePath = path.join(domainsPath, file);
// Skip directories
if (isDirectory(filePath)) {
return;
}
const data = fs.readJsonSync(filePath);
const recordKeys = Object.keys(data.record);
const uniqueRecordKeys = new Set(recordKeys);
t.is(recordKeys.length, uniqueRecordKeys.size, `${file}: Duplicate record keys found`);
});
});
t("All files should have valid record values", (t) => {
files.forEach((file) => {
const filePath = path.join(domainsPath, file);
// Skip directories
if (isDirectory(filePath)) {
return;
}
const data = fs.readJsonSync(filePath);
Object.keys(data.record).forEach((key) => {
const value = data.record[key];
// *: string[]
if (["A", "AAAA", "MX", "NS"].includes(key)) {
t.true(Array.isArray(value), `${file}: Record value should be an array for ${key}`);
value.forEach((record) => {
t.true(typeof record === "string", `${file}: Record value should be a string for ${key}`);
});
// A: string[]
if (key === "A") {
value.forEach((record) => {
t.regex(
record,
ipv4Regex,
`${file}: Record value should be a valid IPv4 address for ${key} at index ${value.indexOf(record)}`
);
t.true(
isPublicIPv4(record, data.proxied),
`${file}: Record value should be a public IPv4 address for ${key} at index ${value.indexOf(record)}`
);
});
}
// AAAA: string[]
if (key === "AAAA") {
value.forEach((record) => {
t.regex(
expandIPv6(record),
ipv6Regex,
`${file}: Record value should be a valid IPv6 address for ${key} at index ${value.indexOf(record)}`
);
t.true(
isPublicIPv6(record),
`${file}: Record value should be a public IPv6 address for ${key} at index ${value.indexOf(record)}`
);
});
}
// *: string[]
if (["MX", "NS"].includes(key)) {
value.forEach((record) => {
t.regex(
record,
hostnameRegex,
`${file}: Record value should be a valid hostname for ${key} at index ${value.indexOf(record)}`
);
});
}
}
// CNAME: string
if (key === "CNAME") {
t.true(typeof value === "string", `${file}: Record value should be a string for ${key}`);
t.regex(value, hostnameRegex, `${file}: Record value should be a valid hostname for ${key}`);
}
// *: {}[]
if (["CAA", "SRV"].includes(key)) {
t.true(Array.isArray(value), `${file}: Record value should be an array for ${key}`);
value.forEach((record) => {
t.true(
typeof record === "object",
`${file}: Record value should be an object for ${key} at index ${value.indexOf(record)}`
);
});
// CAA: { flags: number, tag: string, value: string }[]
if (key === "CAA") {
value.forEach((record) => {
t.true(
typeof record.flags === "number",
`${file}: CAA record value should have a number for flags at index ${value.indexOf(record)}`
);
t.true(
typeof record.tag === "string",
`${file}: CAA record value should have a string for tag at index ${value.indexOf(record)}`
);
t.true(
typeof record.value === "string",
`${file}: CAA record value should have a string for value at index ${value.indexOf(record)}`
);
});
}
// SRV: { priority: number, weight: number, port: number, target: string }[]
if (key === "SRV") {
value.forEach((record) => {
t.true(
typeof record.priority === "number",
`${file}: SRV record value should have a number for priority at index ${value.indexOf(record)}`
);
t.true(
typeof record.weight === "number",
`${file}: SRV record value should have a number for weight at index ${value.indexOf(record)}`
);
t.true(
typeof record.port === "number",
`${file}: SRV record value should have a number for port at index ${value.indexOf(record)}`
);
t.true(
typeof record.target === "string",
`${file}: SRV record value should have a string for target at index ${value.indexOf(record)}`
);
t.regex(
value.target,
hostnameRegex,
`${file}: SRV record value should be a valid hostname for target at index ${value.indexOf(record)}`
);
});
}
}
// TXT|SPF: string | string[]
if (["SPF", "TXT"].includes(key)) {
if (Array.isArray(value)) {
value.forEach((record) => {
t.true(
typeof record === "string",
`${file}: Record value should be a string for ${key} at index ${value.indexOf(record)}`
);
});
} else {
t.true(typeof value === "string", `${file}: Record value should be a string for ${key}`);
}
}
});
});
});