diff --git a/.gitignore b/.gitignore index 91477c9..2ccbe46 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1 @@ /node_modules/ -/*.json -!/package.json -!/package-lock.json diff --git a/README.md b/README.md index 868214f..7fc5f6c 100644 --- a/README.md +++ b/README.md @@ -1,23 +1,5 @@ # fetch-booru -## `[hostname].json` fetch options - -Each request will look for a corresponding `[hostname].json` and, if present, will use it as the second argument in all `fetch` calls. - -For example, if you have a `cf_clearance` cookie for the Cloudflare protection in front of a given site, you can create - -```json -// example.com.json -{ - "headers": { - "cookie": "cf_clearance=whatever", - "user-agent": "whatever" - } -} -``` - -and this cookie and user-agent will be used for API calls and for downloading the images. - ## License [MIT](LICENSE) diff --git a/fetch_options/README.md b/fetch_options/README.md new file mode 100644 index 0000000..4ffa2e7 --- /dev/null +++ b/fetch_options/README.md @@ -0,0 +1,11 @@ +# `fetch_options/[hostname].json` + +Each request will look for a corresponding `fetch_options/[hostname].json` and, if present, will use it as the second argument in all `fetch` calls. + +For example, if you have a `cf_clearance` cookie for the Cloudflare protection in front of a given site, you can create `fetch_options/example.com.json` + +```json +{ "headers": { "cookie": "cf_clearance=whatever", "user-agent": "whatever" } } +``` + +and this cookie and user-agent will be used for network calls to `example.com`. Note that these will not automatically apply to subdomains. diff --git a/fetch_options/e621.net.json b/fetch_options/e621.net.json new file mode 100644 index 0000000..48b11cf --- /dev/null +++ b/fetch_options/e621.net.json @@ -0,0 +1 @@ +{ "headers": { "user-agent": "fetch-booru (+https://git.chor.date/Conduitry/fetch-booru)" } } diff --git a/lib/fetch.js b/lib/fetch.js index 59d3d16..e4703ed 100644 --- a/lib/fetch.js +++ b/lib/fetch.js @@ -4,7 +4,7 @@ import { log } from './log.js'; const fetch_options_lookup = {}; export async function fetch_retry(url) { const { hostname } = new URL(url); - const fetch_options = await (fetch_options_lookup[hostname] ??= fs.promises.readFile(new URL(`../${hostname}.json`, import.meta.url), 'utf8').then(JSON.parse).catch(() => ({ headers: { 'user-agent': 'fetch-booru (+https://git.chor.date/Conduitry/fetch-booru)' } }))); + const fetch_options = await (fetch_options_lookup[hostname] ??= fs.promises.readFile(new URL(`../fetch_options/${hostname}.json`, import.meta.url), 'utf8').then(JSON.parse).catch(() => {})); for (;;) { let resp; try {