63 lines
1.9 KiB
JavaScript
63 lines
1.9 KiB
JavaScript
import * as crypto from 'node:crypto';
|
|
import * as fs from 'node:fs';
|
|
import * as path from 'node:path';
|
|
|
|
const emitWarning = process.emitWarning.bind(process);
|
|
process.emitWarning = (...args) => args[1] !== 'ExperimentalWarning' && emitWarning(...args);
|
|
|
|
const fetch_200 = async (...args) => {
|
|
const resp = await fetch(...args);
|
|
if (resp.status !== 200) {
|
|
console.log(...args);
|
|
throw resp;
|
|
}
|
|
return resp;
|
|
};
|
|
|
|
const make_query = (params) => {
|
|
const filtered = {};
|
|
for (const key in params) {
|
|
if (params[key] != null) {
|
|
filtered[key] = params[key];
|
|
}
|
|
}
|
|
const str = String(new URLSearchParams(filtered));
|
|
return str && `?${str}`;
|
|
};
|
|
|
|
const fetch_options_lookup = {};
|
|
const get_fetch_options = url => {
|
|
const { hostname } = new URL(url);
|
|
return fetch_options_lookup[hostname] ??= fs.promises.readFile(new URL(`./fetch-options.${hostname}.json`, import.meta.url), 'utf8').then(JSON.parse, () => {});
|
|
};
|
|
|
|
export const dl = async get_posts => {
|
|
for (const query of process.argv.slice(2)) {
|
|
const posts = [];
|
|
for await (const page of get_posts(query, async (url, params) => fetch_200(url + make_query(params), await get_fetch_options(url)))) {
|
|
posts.push(...page);
|
|
console.log(`\x1b[K${query}: ${posts.length} posts\x1b[A`);
|
|
}
|
|
console.log();
|
|
|
|
for (let i = 0; i < posts.length; i++) {
|
|
const { url, dest, date } = posts[i];
|
|
try {
|
|
await fs.promises.access(dest);
|
|
} catch {
|
|
console.log(`${i + 1}/${posts.length}: ${dest}`);
|
|
const resp = await fetch_200(url, await get_fetch_options(url));
|
|
const buffer = Buffer.from(await resp.arrayBuffer());
|
|
const tmp = crypto.randomUUID();
|
|
await fs.promises.writeFile(tmp, buffer);
|
|
const date_obj = new Date(date ?? resp.headers.get('last-modified'));
|
|
await fs.promises.utimes(tmp, date_obj, date_obj);
|
|
await fs.promises.mkdir(path.dirname(dest), { recursive: true });
|
|
await fs.promises.rename(tmp, dest);
|
|
}
|
|
}
|
|
|
|
console.log('Done\n');
|
|
}
|
|
};
|