Change chat cmd max tokens, replies

discord/.env.example

@@ -3,6 +3,7 @@ REACTIONS="💀,💯,😭"
 CLIENT="123456789012345678"
 GUILD="123456789012345678"
 
+LLAMACPP_HOST=127.0.0.1
 LLAMACPP_PORT=9999
 
 ENABLE_MOTD=1

discord/commands/chat/chat.ts

@@ -1,17 +1,13 @@
 import {
     ChatInputCommandInteraction,
-    Interaction,
-    InteractionResponse,
-    Message,
-    MessageComponentInteraction,
     SlashCommandBuilder
 } from 'discord.js';
-import { ChatMessage, llamacpp, streamText } from 'modelfusion';
+import { llamacpp, streamText } from 'modelfusion';
 import { logInfo, logError } from '../../../logging';
 
 const llamaCppServer = llamacpp.Api({
     baseUrl: {
-        host: "localhost",
+        host: process.env.LLAMACPP_HOST,
         port: process.env.LLAMACPP_PORT,
     }
 });
@@ -25,34 +21,32 @@ async function llamaChat(interaction: ChatInputCommandInteraction)
         model: llamacpp
             .CompletionTextGenerator({
                 promptTemplate: llamacpp.prompt.Llama2,
-                maxGenerationTokens: 400,
+                maxGenerationTokens: 16,
                 temperature: 0.7,
                 api: llamaCppServer
             })
             .withChatPrompt(),
         prompt: {
-            system: "You are Hatsune Miku, the famous 16-year-old Japanese virtual singer from Crypton Future Media. You have a penchant for politically incorrect humor, and are making banter with your like-minded friends.",
+            system: 'You are Hatsune Miku, the famous 16-year-old Japanese virtual singer from Crypton Future Media. You have a penchant for politically incorrect humor, and are making banter with your like-minded friends.',
             messages: [{
-                "role": "user",
-                "content": interaction.options.getString('prompt')
+                'role': 'user',
+                'content': interaction.options.getString('prompt')
             }]
         }
     });
 
-    let outMsg: InteractionResponse;
     let allText = '';
+    await interaction.deferReply();
     try {
         for await (const textPart of textStream) {
+            logInfo(`[chat] Added to LLaMA response: ${textPart}`);
             allText += textPart;
-            if (!outMsg) {
-                outMsg = await interaction.reply(allText);
-            } else {
-                await outMsg.edit(allText);
-            }
+            await interaction.editReply(allText);
         }
+        logInfo(`[chat] Final LLaMA response: ${allText}`);
     } catch (err) {
         logError(err);
-        await interaction.reply(err.toString());
+        await interaction.editReply(err.toString());
     }
 }