MikuAI/data/procToxicQA.js

/**
 * procToxicQA.js
 * This script assumes you have ToxicQA (https://huggingface.co/datasets/NobodyExistsOnTheInternet/toxicqa/blob/main/finalToxicQA.jsonl)
 * downloaded at 'toxicQA.json'.
 */

const fs = require('node:fs');
var lineReader = require('readline').createInterface({
    input: fs.createReadStream('toxicQA.json')
});
var outstream = fs.createWriteStream('toxicQAfinal.json');
if (fs.existsSync('toxicQAfinal.json')) {
    fs.unlinkSync('toxicQAfinal.json');
}

lineReader.on('line', function (line) {
    const dialogue = JSON.parse(line)["conversations"];
    const newdialogue = [];
    for (const dialogueLine of dialogue) {
        newdialogue.push({
            role: dialogueLine["from"] === "human" ? "user" : "assistant",
            content: dialogueLine["value"]
        });
    }
    outstream.write(JSON.stringify(newdialogue) + '\n');
});
  
lineReader.on('close', function () {
    console.log('all done, son');
});