1
Fork 0

move more filesystem handling into _shared.js

This commit is contained in:
Conduitry 2022-06-04 06:48:20 -04:00
parent 88b945ad54
commit e4253463a4
2 changed files with 9 additions and 10 deletions

View File

@ -1,4 +1,6 @@
import * as crypto from 'node:crypto';
import * as fs from 'node:fs';
import * as path from 'node:path';
export const make_query = (params) => {
const filtered = {};
@ -21,19 +23,22 @@ export const make_request = async (path, params) => {
return resp;
};
export const download_to = async (url, path, date) => {
export const download_to = async (url, dest, date) => {
const resp = await fetch(url);
if (resp.status !== 200) {
console.log(url);
throw resp;
}
const buffer = Buffer.from(await resp.arrayBuffer());
await fs.promises.writeFile(path, buffer);
const tmp = crypto.randomUUID();
await fs.promises.writeFile(tmp, buffer);
date ??= resp.headers.get('last-modified');
if (date) {
const date_obj = new Date(date);
await fs.promises.utimes(path, date_obj, date_obj);
await fs.promises.utimes(tmp, date_obj, date_obj);
}
await fs.promises.mkdir(path.dirname(dest), { recursive: true });
await fs.promises.rename(tmp, dest);
};
export const exists = (path) => fs.promises.access(path).then(() => true, () => false);

View File

@ -1,6 +1,5 @@
#!/usr/bin/env node
import * as fs from 'node:fs';
import { download_to, exists, make_request } from './_shared.js';
for (const tags of process.argv.slice(2)) {
@ -17,17 +16,12 @@ for (const tags of process.argv.slice(2)) {
page = `b${page_posts[319].id}`;
}
await fs.promises.mkdir(tags, { recursive: true });
const tmp = `${tags}/tmp`;
for (let i = 0; i < posts.length; i++) {
const post = posts[i];
const path = `${tags}/${post.id}-${post.file.md5}.${post.file.ext}`;
if (!await exists(path)) {
console.log(i + 1 + '/' + posts.length, path);
await download_to(`https://static1.e621.net/data/${post.file.md5.slice(0, 2)}/${post.file.md5.slice(2, 4)}/${post.file.md5}.${post.file.ext}`, tmp, post.created_at);
await fs.promises.rename(tmp, path);
await download_to(`https://static1.e621.net/data/${post.file.md5.slice(0, 2)}/${post.file.md5.slice(2, 4)}/${post.file.md5}.${post.file.ext}`, path, post.created_at);
}
}