import fetch from "node-fetch"; import { HttpsProxyAgent } from "https-proxy-agent"; import { ProxieProvider } from "./proxies.js"; import logger from "./logger.js"; export async function fetchHandler(target, options = {}, proxy = {}, retry = 0) { // fetch handeler - // retry with different proxy and with no proxy // log the request and response // make a fetch request with proxy if provided if (retry < 1) { logger.log('Requesting:', target); // get new proxy and assign to options } else if (retry === 1) { logger.log('Retry :', retry, target); // retry with new proxy or first proxy var proxy = await (new ProxieProvider()).getOneWorkingProxy(); logger.log('using new proxy:', proxy); } else if (retry === 2) { logger.log('Retry :', retry, target, 'without proxy'); // retry with without proxy } else { logger.error('Retry at limti :', retry, target); return } // make a fetch request with proxy if provided try { var res = await fetchWithProxy(target, options, proxy); if (res.status === 200) { console.log(await res.text()); return res; } else { // retry the request with different proxy if (retry === 2) { throw new Error('Max retries exceeded', res.status, res.statusText); } return fetchHandler(target, options, proxy, ++retry); } } catch (error) { logger.log('[logger] request fails :', error); // retry the request without proxys if (retry === 2) { throw new Error(error); } return fetchHandler(target, options, proxy, ++retry); } } export async function fetchWithProxy( target, options = {}, proxy = {}, ) { var response; try { // logger.log('working with',proxy); if (proxy.ip && proxy.port) { // Create a new Proxy Agent const proxyUrl = `http://${proxy.ip}:${proxy.port}`; const proxyAgent = new HttpsProxyAgent(proxyUrl); response = fetchWithTimeout(target, { agent: proxyAgent, ...options }); } else { response = fetchWithTimeout(target, options); } return await response } catch (error) { return error; } } export const fetchWithTimeout = async (url, options = {}, ms = 5000) => { const controller = new AbortController(); const signal = controller.signal; setTimeout(() => { controller.abort(); }, ms); try { return await fetch(url, { signal, ...options }); } catch (error) { // if (error.name === 'AbortError') { // logger.log('request was aborted'); // } else { // logger.log(error); // } return error } }