james/MikuAI
1
0
Fork 0
Code Issues 9 Pull Requests 2 Packages Projects Releases Wiki Activity

Strip mentions, dropout for non-reaction message chains

Browse Source
This commit is contained in:
James S 2024-05-25 04:35:24 +00:00
parent c990740219
commit a572dc62f1
1 changed files with 35 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
data/process.js
View File

@ -2,6 +2,23 @@ const fs = require('node:fs');
const JSONStream = require('JSONStream'); const JSONStream = require('JSONStream');
const MIKU_FREQ = 5; // 1/5 = 20% of message chains are randomly chosen to be from Miku const MIKU_FREQ = 5; // 1/5 = 20% of message chains are randomly chosen to be from Miku
const DROPOUT_UNFUNNY = 0.75; // 75% dropout rate for messages chains which have NO reactions
const USERNAMES = [
'vinny volcano\uD83C\uDF0B (伊胜焱)',
'Server Comp!',
'Make The Map \uD83D\uDDFA',
'1981 Celical Man\uD83C\uDF41\uD83C\uDF42',
'Hatsune Miku',
'Cutie Kazerounian\uD83C\uDF41\uD83C\uDF42',
'David Pan (Fembooru)\uD83C\uDF41\uD83C\uDF42',
'Exiled Sammy \uD83D\uDD12\uD83C\uDFDD⏱',
'shibe.mp4❄☃',
'Today Man-San(1990)\uD83C\uDF41\uD83C\uDF42',
'owner',
'cj7 by stephen chow (gmod PC)\uD83C\uDF41\uD83C\uDF42',
'Nicolaid',
'epbic',
];
async function main() { async function main() {
let counter = 0; let counter = 0;
@ -14,6 +31,7 @@ async function main() {
let lastMsgTime; let lastMsgTime;
let botAuthoredMsgSequence; let botAuthoredMsgSequence;
let convoMsgSeqCount = 0; let convoMsgSeqCount = 0;
let convoReactCount = 0;
let convoMsgs = []; let convoMsgs = [];
stream.on('data', async (msg) => { stream.on('data', async (msg) => {
@ -35,11 +53,18 @@ async function main() {
*/ */
// scrub links // scrub links
const cleanContent = msg.content.replaceAll(/https?:\/\/\S+/gi, ''); let cleanContent = msg.content.replaceAll(/https?:\/\/\S+/gi, '');
// scrub @mentions
for (const username of USERNAMES) {
cleanContent = cleanContent.replaceAll(`@${username}`, "");
}
if (!cleanContent) { if (!cleanContent) {
return; return;
} }
// count reaction
convoReactCount += msg.reactions.length;
// determine continuity of message sequences // determine continuity of message sequences
let msgTime = new Date(msg.timestamp); let msgTime = new Date(msg.timestamp);
if (lastMsgAuthor !== msg.author.id || (msgTime - lastMsgTime)/60000 >= 7) { if (lastMsgAuthor !== msg.author.id || (msgTime - lastMsgTime)/60000 >= 7) {
@ -54,15 +79,19 @@ async function main() {
// 10 msg sequences per "conversation" // 10 msg sequences per "conversation"
if (convoMsgSeqCount === 10) { if (convoMsgSeqCount === 10) {
// dropout
const convoKeep = convoReactCount > 0 || Math.random() >= DROPOUT_UNFUNNY;
if (convoKeep) {
// write JSONL format // write JSONL format
fs.appendFileSync('output.json', JSON.stringify(convoMsgs) + '\n'); fs.appendFileSync('output.json', JSON.stringify(convoMsgs) + '\n');
convoMsgSeqCount = 0; }
convoMsgSeqCount = convoReactCount = 0;
convoMsgs = []; convoMsgs = [];
} }
// follow chatML chat template // follow chatML chat template
const outMsg = { const outMsg = {
role: botAuthoredMsgSequence ? "assistant" : "user", role: botAuthoredMsgSequence ? "assistant" : msg.author.name,
content: cleanContent content: cleanContent
}; };
convoMsgs.push(outMsg); convoMsgs.push(outMsg);