feat: replace axios with got-scraping

package.json

@@ -58,19 +58,18 @@
 		"@types/cheerio": "0.22.35",
 		"@types/cookie": "0.6.0",
 		"@types/retry": "0.12.5",
-		"axios": "1.7.2",
 		"cheerio": "1.0.0-rc.12",
 		"cookie": "0.6.0",
 		"deepmerge": "4.3.1",
-		"hpagent": "1.2.0",
-		"puppeteer-core": "22.12.1",
+		"got-scraping": "4.0.6",
+		"puppeteer-core": "22.13.1",
 		"retry": "0.13.1"
 	},
 	"devDependencies": {
 		"@tuplo/shell": "1.2.2",
 		"@types/mime": "3.0.4",
-		"@types/node": "20.14.10",
-		"@vitest/coverage-v8": "1.6.0",
+		"@types/node": "20.14.11",
+		"@vitest/coverage-v8": "2.0.3",
 		"esbuild": "0.23.0",
 		"eslint": "8.57.0",
 		"eslint-config-prettier": "9.1.0",
@@ -80,10 +79,10 @@
 		"mime": "4.0.4",
 		"npm-check-updates": "16.14.20",
 		"nyc": "17.0.0",
-		"prettier": "3.3.2",
+		"prettier": "3.3.3",
 		"tsx": "4.16.2",
 		"typescript": "5.5.3",
-		"typescript-eslint": "7.15.0",
-		"vitest": "1.6.0"
+		"typescript-eslint": "7.16.1",
+		"vitest": "2.0.3"
 	}
 }

sh/build.ts

@@ -6,13 +6,7 @@ async function main() {
 	await $`rm -rf dist`;
 	await $`tsc --project tsconfig.build.json`;
 
-	const flags = [
-		"--bundle",
-		"--platform=node",
-		"--external:puppeteer-core",
-		"--external:axios",
-		"--external:hpagent",
-	];
+	const flags = ["--bundle", "--platform=node", "--external:puppeteer-core"];
 
 	await $`esbuild src/cjs/index.cjs --outfile=dist/index.cjs ${flags}`;
 	await $`esbuild src/index.ts --format=esm --outfile=dist/index.mjs ${flags}`;

src/cache.test.ts

@@ -1,7 +1,5 @@
 /* eslint-disable @typescript-eslint/no-unused-vars */
 /* eslint-disable no-restricted-syntax */
-import { vi } from "vitest";
-
 import fletcher from "./index";
 
 const requestSpy = vi.fn();

src/fletcher.d.ts

@@ -1,21 +1,21 @@
 import { type IncomingHttpHeaders } from "node:http";
 import type * as VM from "node:vm";
 
+import { type AnyNode, type Cheerio, type Element } from "cheerio";
 import {
-	type AxiosRequestConfig,
-	type AxiosResponse,
-	type AxiosResponseHeaders,
+	type Headers,
 	type Method,
-} from "axios";
-import { type AnyNode, type Cheerio, type Element } from "cheerio";
+	type OptionsInit,
+	type Response,
+} from "got-scraping";
 import { type Page, type ScreenshotOptions } from "puppeteer-core";
 
 import { type IOptions as IRetryOptions } from "./helpers/async-retry";
 import { type CookieJar } from "./helpers/cookie-jar";
 
 export type UrlSearchParams = Record<string, number | string | undefined>;
 
-export type FetchOptions = AxiosRequestConfig;
+export type FetchOptions = OptionsInit;
 
 export { type CookieJar, type ICookie } from "./helpers/cookie-jar";
 
@@ -27,12 +27,10 @@ export type IProxyConfig = {
 	username?: string;
 };
 
-type RequestRedirect = "error" | "follow" | "manual";
-
 type RequestData = Record<string, unknown>;
 
 type IOnAfterRequestArgs = {
-	response: AxiosResponse;
+	response: Response;
 };
 
 export type IOnAfterRequestFn = {
@@ -145,7 +143,7 @@ export type IInstance = {
 
 export type IResponse = {
 	// body: Readable & Dispatcher.BodyMixin;
-	headers: AxiosResponseHeaders;
+	headers: Headers;
 	statusCode: number;
 	statusMessage?: string;
 	text: () => Promise<string>;

src/helpers/cookie-jar.ts

@@ -44,7 +44,12 @@ export class CookieJar {
 		this.cookies.push(c);
 	}
 
-	setCookies(cookies: ICookie[] | string[] = []) {
+	setCookies(cookies_: ICookie[] | string | string[] | undefined) {
+		const cookies = Array.isArray(cookies_)
+			? cookies_
+			: typeof cookies_ === "string"
+				? cookies_.split("; ")
+				: [];
 		for (const cookie of cookies) {
 			this.setCookie(cookie);
 		}

src/index.ts

@@ -60,7 +60,9 @@ function fletcher(
 					console.error(error);
 				}
 
-				if (!res) throw new Error(error as string);
+				if (!res) {
+					throw new Error(error as string);
+				}
 
 				if (!validateStatus(res.statusCode)) {
 					throw new Error(`${res.statusCode}: ${res.statusMessage}`);

src/options/index.ts

@@ -1,5 +1,5 @@
 /* eslint-disable no-case-declarations */
-import { type Method } from "axios";
+import { type Method } from "got-scraping";
 
 import type {
 	IFletcherOptions,

src/retry.test.ts

@@ -1,5 +1,3 @@
-import { vi } from "vitest";
-
 import { toFletcherOptions } from "./options";
 import fletcher from "./index";
 

src/services/request.test.ts

@@ -1,6 +1,6 @@
 import { STATUS_CODES } from "node:http";
 
-import { type IFletcherOptions } from "src/fletcher";
+import { type IFletcherOptions } from "src/fletcher.d";
 import { getRandomPort, server } from "src/mocks";
 import { server as httpsServer } from "src/mocks/https";
 
@@ -45,18 +45,9 @@ describe("request", () => {
 
 			const expectedRequest = {
 				body: '{"foo":"bar"}',
-				headers: {
-					accept: "application/json, text/plain, */*",
-					"accept-encoding": "gzip, compress, deflate, br",
-					connection: expect.anything(),
-					"content-length": "13",
-					"content-type": "application/x-www-form-urlencoded",
-					host: `localhost:${port}`,
-					"user-agent": expect.anything(),
-				},
 				method: "POST",
 			};
-			expect(JSON.parse(req)).toStrictEqual(expectedRequest);
+			expect(JSON.parse(req)).toMatchObject(expectedRequest);
 		});
 	});
 
@@ -199,55 +190,4 @@ describe("request", () => {
 			expect(actual.statusMessage).toBe(expected);
 		});
 	});
-
-	describe("proxy", () => {
-		it.skip("uses a proxy", async () => {
-			uri.pathname = "/ip";
-			const proxy = {
-				host: "<ip>",
-				password: "<password>",
-				port: 666,
-				username: "<username>",
-			};
-
-			const uri2 = new URL("https://httpbin.org/ip");
-			const response = await request(uri2.href, { proxy });
-			const body = await response.text();
-			const actual = JSON.parse(body);
-
-			expect(actual.origin).toBe(proxy.host);
-		});
-	});
-
-	describe("onAfterRequest", () => {
-		it("calls postRequest (sync)", async () => {
-			const onAfterRequestSpy = vi.fn();
-			uri.pathname = "/anything";
-			const options = { onAfterRequest: onAfterRequestSpy };
-			await request(uri.href, options);
-
-			const expected = {
-				response: expect.anything(),
-			};
-			expect(onAfterRequestSpy).toHaveBeenCalledTimes(1);
-			expect(onAfterRequestSpy).toHaveBeenCalledWith(expected);
-		});
-
-		it("calls postRequest (async)", async () => {
-			const onAfterRequestSpy = vi.fn().mockReturnValue(
-				new Promise<void>((resolve) => {
-					resolve();
-				})
-			);
-			uri.pathname = "/anything";
-			const options = { onAfterRequest: onAfterRequestSpy };
-			await request(uri.href, options);
-
-			const expected = {
-				response: expect.anything(),
-			};
-			expect(onAfterRequestSpy).toHaveBeenCalledTimes(1);
-			expect(onAfterRequestSpy).toHaveBeenCalledWith(expected);
-		});
-	});
 });

src/services/request.ts

@@ -1,70 +1,47 @@
 import { STATUS_CODES } from "node:http";
-import https from "node:https";
 
-import axios, {
-	type AxiosError,
-	type AxiosRequestConfig,
-	type AxiosResponse,
-	type AxiosResponseHeaders,
-} from "axios";
-import { HttpsProxyAgent } from "hpagent";
+import {
+	gotScraping as got,
+	type Method,
+	type OptionsInit,
+	type RequestError,
+	type Response,
+} from "got-scraping";
 
-import { type IFletcherOptions, type IResponse } from "../fletcher.d";
+import { type IFletcherOptions, type IResponse } from "../fletcher";
 
-function toAxiosOptions(fletcherOptions?: Partial<IFletcherOptions>) {
+function toGotOptions(
+	url: string,
+	fletcherOptions?: Partial<IFletcherOptions>
+) {
 	const {
 		body,
 		encoding,
 		headers,
-		maxRedirections = 999, // follow all redirects by default
+		maxRedirections = 999,
 		method = "GET",
 		proxy,
-		rejectUnauthorized,
+		rejectUnauthorized = false,
 		timeout = 30_000,
-		validateStatus,
 	} = fletcherOptions || {};
 
-	const options: AxiosRequestConfig = {
+	const options: OptionsInit = {
+		body,
+		encoding,
+		followRedirect: maxRedirections > 0,
+		headers,
+		https: { rejectUnauthorized },
 		maxRedirects: maxRedirections,
-		method,
-		responseType: "text",
-		timeout,
+		method: method as Method,
+		retry: { limit: 0 },
+		timeout: { request: timeout },
+		url,
 	};
 
-	if (body) {
-		options.data = body;
-	}
-
-	if (encoding) {
-		options.responseEncoding = encoding;
-	}
-
-	if (headers) {
-		options.headers = headers;
-	}
-
-	if (validateStatus) {
-		options.validateStatus = validateStatus;
-	}
-
-	if (options.maxRedirects === 0) {
-		options.validateStatus =
-			validateStatus || ((statusCode) => statusCode >= 200 && statusCode < 400);
-	}
-
-	if (rejectUnauthorized !== undefined && !proxy) {
-		options.httpsAgent = new https.Agent({
-			rejectUnauthorized: rejectUnauthorized ?? false,
-		});
-	}
-
 	if (proxy) {
 		const { host, password, port, protocol = "http", username } = proxy;
 		const auth = `${username}:${password}`;
-		options.httpsAgent = new HttpsProxyAgent({
-			proxy: `${protocol}://${auth}@${host}:${port}`,
-			rejectUnauthorized: rejectUnauthorized ?? false,
-		});
+		options.proxyUrl = `${protocol}://${auth}@${host}:${port}`;
 	}
 
 	return options;
@@ -75,39 +52,35 @@ export async function request(
 	userOptions?: Partial<IFletcherOptions>
 ) {
 	try {
-		// encode special characters on the URL
-		const uri = new URL(url);
-		const options = toAxiosOptions(userOptions);
-		const response = await axios(uri.href, options);
-		const {
-			data,
-			headers,
-			status: statusCode,
-			statusText: statusMessage,
-		} = response;
+		const options = toGotOptions(url, userOptions);
+		const response_ = await got(options);
+		const response = response_ as Response;
 
-		if (userOptions?.onAfterRequest) {
-			const r = userOptions.onAfterRequest({ response });
-			await Promise.resolve(r);
+		const { headers, statusCode } = response;
+
+		const statusMessage = response.statusMessage || STATUS_CODES[statusCode];
+
+		if (userOptions?.validateStatus) {
+			response.ok = userOptions.validateStatus(statusCode);
 		}
 
 		return {
-			headers: headers as AxiosResponseHeaders,
+			headers,
 			statusCode,
-			statusMessage,
-			text: async () => data,
-		};
+			statusMessage: response.ok ? statusMessage : `${statusMessage} - ${url}`,
+			text: async () => response.body as string,
+		} as IResponse;
 	} catch (error_) {
-		const error = error_ as AxiosError;
-		const { cause, response = {} as AxiosResponse } = error;
-		const { headers, status: statusCode } = response;
+		// @ts-expect-error caus is not a standard property
+		const { cause, code, options } = error_ as RequestError;
+		const { headers } = options;
+		const { message: statusMessage } = cause;
 
-		const statusMessage =
-			response.statusText || error.message || STATUS_CODES[statusCode];
+		const statusCode = code || 500;
 
 		return {
 			headers,
-			statusCode: statusCode || error.code,
+			statusCode,
 			statusMessage: `${statusMessage} - ${url}`,
 			text: async () => JSON.stringify({ cause, statusCode, statusMessage }),
 		} as IResponse;