Formatting script for NobodyExistsOnTheInternet/toxicqa

data/procToxicQA.js

@@ -0,0 +1,22 @@
+const fs = require('node:fs');
+var lineReader = require('readline').createInterface({
+    input: fs.createReadStream('toxicQA.json')
+});
+var outstream = fs.createWriteStream('toxicQAfinal.json');
+fs.unlinkSync('toxicQAfinal.json');
+
+lineReader.on('line', function (line) {
+    const dialogue = JSON.parse(line)["conversations"];
+    const newdialogue = [];
+    for (const dialogueLine of dialogue) {
+        newdialogue.push({
+            role: dialogueLine["from"] === "human" ? "user" : "assistant",
+            content: dialogueLine["value"]
+        });
+    }
+    outstream.write(JSON.stringify(newdialogue) + '\n');
+});
+  
+lineReader.on('close', function () {
+    console.log('all done, son');
+});