1
Fork 0

rework directory/query handling in dl-*

This commit is contained in:
Conduitry 2022-06-16 20:11:15 -04:00
parent 37b0663e5f
commit d472b8e360
1 changed files with 12 additions and 8 deletions

View File

@ -27,9 +27,12 @@ export const get = (url, params) => {
return fetch_200(`${url}?${search}`);
};
const get_query = arg => arg.match(/([^/]+)\/?$/)[1];
export const dl = async get_posts => {
for (const query of process.argv.slice(2)) {
for (const arg of process.argv.slice(2)) {
const posts = [];
const query = get_query(arg);
for await (const page of get_posts(query)) {
posts.push(...page);
console.log(`\x1b[K${query}: ${posts.length} posts\x1b[A`);
@ -38,17 +41,18 @@ export const dl = async get_posts => {
for (let i = 0; i < posts.length; i++) {
const { url, dest, date } = posts[i];
try {
await fs.promises.access(`${query}/${dest}`);
} catch {
console.log(`${i + 1}/${posts.length}: ${query}/${dest}`);
const path = posts.length > 1 ? `${arg}/${dest}` : dest;
if (!await fs.promises.access(path).then(() => true, () => false)) {
console.log(`${i + 1}/${posts.length}: ${path}`);
const resp = await fetch_200(url);
const tmp = crypto.randomUUID();
if (posts.length > 1) {
await fs.promises.mkdir(arg, { recursive: true });
}
const tmp = `${posts.length > 1 ? `${arg}/` : ''}.${crypto.randomUUID()}`;
await new Promise(res => stream.Readable.fromWeb(resp.body).pipe(fs.createWriteStream(tmp)).once('finish', res));
const date_obj = new Date(date ?? resp.headers.get('last-modified'));
await fs.promises.utimes(tmp, date_obj, date_obj);
await fs.promises.mkdir(query, { recursive: true });
await fs.promises.rename(tmp, `${query}/${dest}`);
await fs.promises.rename(tmp, path);
}
}