Conduitry/fetch-booru
1
Fork 0
Code Activity

limit to 32 concurrent file downloads

Browse source
This commit is contained in:
Conduitry 2024-03-27 17:47:50 -04:00
parent 21f56991ac
commit 6c76a14448
2 changed files with 32 additions and 18 deletions
Download patch file Download diff file Expand all files Collapse all files

7
lib/concurrency.js Normal file
View file

@ -0,0 +1,7 @@
export function create_concurrency_queue(concurrency) {
const waiting = [];
return {
request: () => concurrency ? concurrency-- : new Promise(res => waiting.push(res)),
release: () => waiting.length ? waiting.shift()() : concurrency++,
};
}

43
lib/fetch.js
View file

@ -1,4 +1,5 @@
import * as fs from 'node:fs';
import { create_concurrency_queue } from './concurrency.js';
import { log } from './log.js';
const fetch_options_lookup = {};
@ -26,29 +27,35 @@ export async function fetch_retry(url) {
}
}
const dl_queue = create_concurrency_queue(32);
export async function fetch_retry_dl(url, dest, mtime) {
try {
await fs.promises.access(dest);
return false;
} catch {}
log(`Downloading: ${url}`);
const tmp = dest.replace(/[^/]*$/, '.$&.part');
for (;;) {
const resp = await fetch_retry(url);
try {
await fs.promises.writeFile(tmp, resp.body);
} catch (error) {
log('ERROR - fetch', url, 'code:', error?.cause?.code ?? error, 'retrying');
await new Promise((res) => setTimeout(res, 5000));
continue;
try {
await dl_queue.request();
log(`Downloading: ${url}`);
const tmp = dest.replace(/[^/]*$/, '.$&.part');
for (;;) {
const resp = await fetch_retry(url);
try {
await fs.promises.writeFile(tmp, resp.body);
} catch (error) {
log('ERROR - fetch', url, 'code:', error?.cause?.code ?? error, 'retrying');
await new Promise((res) => setTimeout(res, 5000));
continue;
}
mtime ??= resp.headers.get('last-modified');
if (mtime) {
const date = new Date(mtime);
await fs.promises.utimes(tmp, date, date);
}
await fs.promises.rename(tmp, dest);
log(`Downloaded to: ${dest}`);
return true;
}
mtime ??= resp.headers.get('last-modified');
if (mtime) {
const date = new Date(mtime);
await fs.promises.utimes(tmp, date, date);
}
await fs.promises.rename(tmp, dest);
log(`Downloaded to: ${dest}`);
return true;
} finally {
dl_queue.release();
}
}