doushio/archive/dump.js

/* Dumps the given thread to JSON.
 * Not production-ready.
 */

var _ = require('../lib/underscore'),
    async = require('async'),
    caps = require('../server/caps'),
    db = require('../db'),
    etc = require('../etc'),
    fs = require('fs'),
    joinPath = require('path').join,
    render = require('../server/render'),
    util = require('util');

var DUMP_DIR = 'www/archive';
var AUTH_DUMP_DIR = 'authdump';

var DUMP_IDENT = {ip: '127.0.0.1', auth: 'dump'};

require('../server/amusement'); // load dice hook

function Dumper(reader, out) {
	this.out = out;
	this.reader = reader;
	_.bindAll(this);
	reader.on('thread', this.on_thread);
	reader.on('post', this.on_post);
	reader.on('endthread', this.on_endthread);
}
var D = Dumper.prototype;

D.on_thread = function (op_post) {
	if (this.needComma) {
		this.out.write(',\n');
		this.needComma = false;
	}
	this.op = op_post.num;
	this.out.write('[\n' + JSON.stringify(tweak_post(op_post)));
};

D.on_post = function (post) {
	this.out.write(',\n' + JSON.stringify(tweak_post(post, this.op)));
};

D.on_endthread = function () {
	this.out.write('\n]');
	this.needComma = true;
	this.op = null;
};

D.destroy = function () {
	this.reader.removeListener('thread', this.on_thread);
	this.reader.removeListener('post', this.on_post);
	this.reader.removeListener('endthread', this.on_endthread);
	this.reader = null;
	this.out = null;
};


function AuthDumper(reader, out) {
	Dumper.call(this, reader, out);
}
util.inherits(AuthDumper, Dumper);
var AD = AuthDumper.prototype;

AD.on_thread = function (post) {
	this.out.write('{"ips":{');

	if (post.num && post.ip) {
		this.out.write('"'+post.num+'":'+JSON.stringify(post.ip));
		this.needComma = true;
	}
};

AD.on_post = function (post) {
	if (post.num && post.ip) {
		if (this.needComma)
			this.out.write(',');
		else
			this.needComma = true;
		this.out.write('"'+post.num+'":'+JSON.stringify(post.ip));
	}
};

AD.on_endthread = function () {
	this.out.write('}}');
	this.needComma = false;
};

function tweak_post(post, known_op) {
	post = _.clone(post);

	/* thread-only */
	if (typeof post.tags == 'string')
		post.tags = db.parse_tags(post.tags);
	if (typeof post.origTags == 'string')
		post.origTags = db.parse_tags(post.origTags);
	if (typeof post.hctr == 'string')
		post.hctr = parseInt(post.hctr, 10);
	if (typeof post.imgctr == 'string')
		post.imgctr = parseInt(post.imgctr, 10);

	/* post-only */
	if (known_op == post.op)
		delete post.op;

	if (post.hideimg) {
		delete post.image;
		delete post.hideimg;
	}
	if (post.body == '')
		delete post.body;

	/* blacklisting is bad... */
	delete post.ip;

	return post;
}

function dump_thread(op, board, ident, outputs, cb) {
	if (!caps.can_access_board(ident, board))
		return cb(404);
	if (!caps.can_access_thread(ident, op))
		return cb(404);

	var yaku = new db.Yakusoku(board, ident);
	var reader = new db.Reader(yaku);
	reader.get_thread(board, op, {});
	reader.once('nomatch', function () {
		cb(404);
		yaku.disconnect();
	});
	reader.once('redirect', function (op) {
		cb('redirect', op);
		yaku.disconnect();
	});
	reader.once('begin', function (preThread) {
		var dumper = new Dumper(reader, outputs.json);
		var authDumper = new AuthDumper(reader, outputs.auth);

		var out = outputs.html;
		render.write_thread_head(out, '', board, op, {
			subject: preThread.subject,
		});

		var fakeReq = {ident: ident, headers: {}};
		var opts = {fullPosts: true, board: board};
		render.write_thread_html(reader, fakeReq, out, opts);

		reader.once('end', function () {
			outputs.json.write('\n');
			outputs.auth.write('\n');
			render.write_page_end(out, ident, true);
			yaku.disconnect();
			cb(null);
		});
	});

	function on_err(err) {
		yaku.disconnect();
		cb(err);
	}
	reader.once('error', on_err);
	yaku.once('error', on_err);
}

function close_stream(stream, cb) {
	if (!stream.writable)
		return cb(null);
	if (stream.write(''))
		close();
	else
		stream.once('drain', close);

	function close() {
		// deal with process.stdout not being closable
		try {
			stream.destroySoon(function (err) {
				if (cb)
					cb(err);
				cb = null;
			});
		}
		catch (e) {
			if (cb)
				cb(null);
			cb = null;
		}
	}
}

function load_state(cb) {
	async.series([
		etc.checked_mkdir.bind(null, DUMP_DIR),
		etc.checked_mkdir.bind(null, AUTH_DUMP_DIR),
		require('../server/state').reload_hot_resources,
		db.track_OPs,
	], cb);
}

if (require.main === module) (function () {
	var op = parseInt(process.argv[2], 10), board = process.argv[3];
	if (!op) {
		console.error('Usage: node upkeep/dump.js <thread>');
		process.exit(-1);
	}

	console.log('Loading state...');
	load_state(function (err) {
		if (err)
			throw err;

		if (!board)
			board = db.first_tag_of(op);
		if (!board) {
			console.error(op + ' has no tags.');
			process.exit(-1);
		}

		console.log('Dumping thread...');

		var base = joinPath(DUMP_DIR, op.toString());
		var authBase = joinPath(AUTH_DUMP_DIR, op.toString());
		var outputs = {
			auth: fs.createWriteStream(authBase + '.json'),
			json: fs.createWriteStream(base + '.json'),
			html: fs.createWriteStream(base + '.html'),
		};

		dump_thread(op, board, DUMP_IDENT, outputs, function (err) {
			if (err)
				throw err;

			var streams = [];
			for (var k in outputs)
				streams.push(outputs[k]);
			async.each(streams, close_stream, quit);
		});
	});

	function quit() {
		// crappy flush for stdout (can't close it)
		if (process.stdout.write(''))
			process.exit(0);
		else
			process.stdout.on('drain', function () {
				process.exit(0);
			});
	}
})();