Skip to content

Commit ab63267

Browse files
committed
feat(grzesiu): Use natural
1 parent 38b5a89 commit ab63267

File tree

3 files changed

+143
-23
lines changed

3 files changed

+143
-23
lines changed

package.json

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
"isolated-vm": "4.2.0",
3333
"mongodb": "4.0.0-beta.5",
3434
"monitorss": "6.13.19",
35+
"natural": "5.1.13",
3536
"node-cache": "5.1.2",
3637
"node-fetch": "2.6.1",
3738
"openai": "2.0.2",
@@ -47,6 +48,7 @@
4748
"@types/chai-as-promised": "7.1.4",
4849
"@types/dotenv": "8.2.0",
4950
"@types/mocha": "8.2.2",
51+
"@types/natural": "5.1.0",
5052
"@types/nock": "11.1.0",
5153
"@types/node": "14.14.41",
5254
"@types/node-fetch": "2.5.10",

src/commands/grzesiu.ts

Lines changed: 85 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,13 @@
11
import { Command } from '../types';
22
import { Configuration, OpenAIApi } from 'openai';
33
import grzesJson from '../../grzes.json';
4+
import Natural from 'natural';
5+
6+
const tokenizer = new Natural.AggressiveTokenizerPl();
7+
const tfidf = grzesJson.reduce((tfidf, line) => {
8+
tfidf.addDocument(tokenizer.tokenize(line));
9+
return tfidf;
10+
}, new Natural.TfIdf());
411

512
const wait = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
613

@@ -24,30 +31,14 @@ const grzesiu: Command = {
2431
cooldown: COOLDOWN,
2532
async execute(msg, args) {
2633
const username = (msg.member?.displayName || msg.author.username).trim().split(/\s/)[0];
27-
const prompt = await generateGrzesiuPrompt(username, args.join(' '));
28-
29-
const engine = 'text-davinci-001';
30-
// const engine = 'text-babbage-001';
31-
const response = await openai.createCompletion(engine, {
32-
prompt,
33-
temperature: 1,
34-
max_tokens: RESPONSE_TOKENS,
35-
top_p: 1,
36-
frequency_penalty: 0,
37-
presence_penalty: 2,
38-
best_of: 4,
39-
});
40-
41-
if (!response.data.choices?.[0]?.text) {
42-
return msg.channel.send(`Niestety, Grzesiu nie miał nic do powiedzenia!`);
43-
}
34+
const question = args.join(' ');
4435

45-
const messages = response.data.choices[0].text
46-
.split('\n')
47-
.map((l) => l.trim())
48-
.filter((l) => l.startsWith(`${GRZESIU_NAME}:`))
49-
.flatMap((l) => l.split(`${GRZESIU_NAME}:`))
50-
.filter((l) => l.trim().length > 0);
36+
const generators = {
37+
getGrzesiuAnswerFromOpenAI,
38+
getRandomGrzesiuAnswers,
39+
};
40+
41+
const messages = await generators.getRandomGrzesiuAnswers(username, question);
5142

5243
if (!messages.length) {
5344
return msg.channel.send(`Niestety, Grzesiu nie miał nic do powiedzenia!`);
@@ -66,6 +57,77 @@ export default grzesiu;
6657

6758
const getRandomInt = (len: number) => Math.floor(Math.random() * len);
6859

60+
interface GrzesiuGenerator {
61+
(username: string, question: string): Promise<string[]>;
62+
}
63+
64+
// random
65+
const getRandomGrzesiuAnswers: GrzesiuGenerator = async (_username, question) => {
66+
const g = grzesJson
67+
.map((l) => l.trim())
68+
.filter((line) => !BANNED_PATTERNS.test(line) && line.length > 0);
69+
70+
const potentialItems =
71+
question.trim().length > 0
72+
? g.filter((l, idx) => {
73+
const coeff = (tfidf.tfidf(question, idx) as unknown as number) || 1;
74+
return coeff * Natural.DiceCoefficient(question, l) > 0.35;
75+
})
76+
: [];
77+
78+
const initialIndex =
79+
potentialItems.length > 0 ? g.indexOf(potentialItems[getRandomInt(potentialItems.length)]) : -1;
80+
81+
const MAX_LINES = 5;
82+
const numberOfLines = 5 - Math.floor(Math.log2(Math.floor(Math.random() * 2 ** MAX_LINES) + 1));
83+
84+
const lines = [];
85+
let idx = initialIndex === -1 ? getRandomInt(g.length) : initialIndex;
86+
87+
for (let i = 0; i < numberOfLines; ++i) {
88+
lines.push(g[idx]);
89+
90+
if (Math.random() < 0.9 && idx < g.length) {
91+
++idx;
92+
} else {
93+
idx = getRandomInt(g.length);
94+
}
95+
}
96+
97+
return lines;
98+
};
99+
100+
// openAI
101+
const getGrzesiuAnswerFromOpenAI: GrzesiuGenerator = async (username, question) => {
102+
const prompt = await generateGrzesiuPrompt(username, question);
103+
104+
const engine = 'text-davinci-001';
105+
// const engine = 'text-babbage-001';
106+
const response = await openai.createCompletion(engine, {
107+
prompt,
108+
temperature: 1,
109+
max_tokens: RESPONSE_TOKENS,
110+
top_p: 1,
111+
frequency_penalty: 0,
112+
presence_penalty: 2,
113+
best_of: 4,
114+
});
115+
116+
console.log(response);
117+
118+
if (!response.data.choices?.[0]?.text) {
119+
return [];
120+
}
121+
122+
const messages = response.data.choices[0].text
123+
.split('\n')
124+
.map((l) => l.trim())
125+
.filter((l) => l.startsWith(`${GRZESIU_NAME}:`))
126+
.flatMap((l) => l.split(`${GRZESIU_NAME}:`))
127+
.filter((l) => l.trim().length > 0);
128+
return messages;
129+
};
130+
69131
const getRandomIndices = (num: number, max: number) => {
70132
const set = new Set<number>();
71133
while (set.size < num) set.add(getRandomInt(max));

yarn.lock

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -601,6 +601,13 @@
601601
"@types/bson" "*"
602602
"@types/node" "*"
603603

604+
605+
version "5.1.0"
606+
resolved "https://registry.yarnpkg.com/@types/natural/-/natural-5.1.0.tgz#bb003b328db83db5bf138829ecf79ce26e9997c0"
607+
integrity sha512-b6HzwfHcxK9himwAnhyth8Nz3CcbbgM/qu8bkSVEE/WzMUNR2wo2RHaVK5XVVakwP8EVcI5MAysaOkG8iWhEag==
608+
dependencies:
609+
"@types/node" "*"
610+
604611
605612
version "11.1.0"
606613
resolved "https://registry.yarnpkg.com/@types/nock/-/nock-11.1.0.tgz#0a8c1056a31ba32a959843abccf99626dd90a538"
@@ -848,6 +855,11 @@ addressparser@^1.0.1:
848855
resolved "https://registry.yarnpkg.com/addressparser/-/addressparser-1.0.1.tgz#47afbe1a2a9262191db6838e4fd1d39b40821746"
849856
integrity sha1-R6++GiqSYhkdtoOOT9HTm0CCF0Y=
850857

858+
afinn-165@^1.0.2:
859+
version "1.0.4"
860+
resolved "https://registry.yarnpkg.com/afinn-165/-/afinn-165-1.0.4.tgz#3abf6b8922dd5db84d84e0abd155924381dd73a4"
861+
integrity sha512-7+Wlx3BImrK0HiG6y3lU4xX7SpBPSSu8T9iguPMlaueRFxjbYwAQrp9lqZUuFikqKbd/en8lVREILvP2J80uJA==
862+
851863
aggregate-error@^3.0.0:
852864
version "3.1.0"
853865
resolved "https://registry.yarnpkg.com/aggregate-error/-/aggregate-error-3.1.0.tgz#92670ff50f5359bdb7a3e0d40d0ec30c5737687a"
@@ -940,6 +952,13 @@ anymatch@~3.1.1:
940952
normalize-path "^3.0.0"
941953
picomatch "^2.0.4"
942954

955+
apparatus@^0.0.10:
956+
version "0.0.10"
957+
resolved "https://registry.yarnpkg.com/apparatus/-/apparatus-0.0.10.tgz#81ea756772ada77863db54ceee8202c109bdca3e"
958+
integrity sha512-KLy/ugo33KZA7nugtQ7O0E1c8kQ52N3IvD/XgIh4w/Nr28ypfkwDfA67F1ev4N1m5D+BOk1+b2dEJDfpj/VvZg==
959+
dependencies:
960+
sylvester ">= 0.0.8"
961+
943962
append-transform@^2.0.0:
944963
version "2.0.0"
945964
resolved "https://registry.yarnpkg.com/append-transform/-/append-transform-2.0.0.tgz#99d9d29c7b38391e6f428d28ce136551f0b77e12"
@@ -3608,6 +3627,18 @@ natural-compare@^1.4.0:
36083627
resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7"
36093628
integrity sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=
36103629

3630+
3631+
version "5.1.13"
3632+
resolved "https://registry.yarnpkg.com/natural/-/natural-5.1.13.tgz#61f3e5d15aacd0e6b5c270cb5e8ed1dba7e163a8"
3633+
integrity sha512-fbV7Jvr8X0S+Yb/iggoGxTBRMSbTxN1m2sgBspNI5dSnE6x3lN2P48O1ddyoiTtMC2q4OKDkHJAGuNRs9jM9WA==
3634+
dependencies:
3635+
afinn-165 "^1.0.2"
3636+
apparatus "^0.0.10"
3637+
safe-stable-stringify "^2.2.0"
3638+
sylvester "^0.0.12"
3639+
underscore "^1.9.1"
3640+
wordnet-db "^3.1.11"
3641+
36113642
36123643
version "0.6.2"
36133644
resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.2.tgz#feacf7ccf525a77ae9634436a64883ffeca346fb"
@@ -4523,6 +4554,11 @@ [email protected], safe-buffer@^5.0.1, safe-buffer@^5.1.0, safe-buffer@^5.1.1, s
45234554
resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6"
45244555
integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==
45254556

4557+
safe-stable-stringify@^2.2.0:
4558+
version "2.3.1"
4559+
resolved "https://registry.yarnpkg.com/safe-stable-stringify/-/safe-stable-stringify-2.3.1.tgz#ab67cbe1fe7d40603ca641c5e765cb942d04fc73"
4560+
integrity sha512-kYBSfT+troD9cDA85VDnHZ1rpHC50O0g1e6WlGHVCz/g+JS+9WKLj+XwFYyR8UbrZN8ll9HUpDAAddY58MGisg==
4561+
45264562
"safer-buffer@>= 2.1.2 < 3", safer-buffer@^2.0.2, safer-buffer@^2.1.0, safer-buffer@~2.1.0:
45274563
version "2.1.2"
45284564
resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a"
@@ -4941,6 +4977,16 @@ supports-color@^7.1.0, supports-color@^7.2.0:
49414977
dependencies:
49424978
has-flag "^4.0.0"
49434979

4980+
"sylvester@>= 0.0.8":
4981+
version "0.0.21"
4982+
resolved "https://registry.yarnpkg.com/sylvester/-/sylvester-0.0.21.tgz#2987b1ce2bd2f38b0dce2a34388884bfa4400ea7"
4983+
integrity sha1-KYexzivS84sNzio0OIiEv6RADqc=
4984+
4985+
sylvester@^0.0.12:
4986+
version "0.0.12"
4987+
resolved "https://registry.yarnpkg.com/sylvester/-/sylvester-0.0.12.tgz#5a884415cd2d002c57e7a3aac99462a75ce9fdb4"
4988+
integrity sha1-WohEFc0tACxX56OqyZRip1zp/bQ=
4989+
49444990
table@^6.0.9:
49454991
version "6.7.1"
49464992
resolved "https://registry.yarnpkg.com/table/-/table-6.7.1.tgz#ee05592b7143831a8c94f3cee6aae4c1ccef33e2"
@@ -5130,6 +5176,11 @@ unbox-primitive@^1.0.0, unbox-primitive@^1.0.1:
51305176
has-symbols "^1.0.2"
51315177
which-boxed-primitive "^1.0.2"
51325178

5179+
underscore@^1.9.1:
5180+
version "1.13.2"
5181+
resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.13.2.tgz#276cea1e8b9722a8dbed0100a407dda572125881"
5182+
integrity sha512-ekY1NhRzq0B08g4bGuX4wd2jZx5GnKz6mKSqFL4nqBlfyMGiG10gDFhDTMEfYmDL6Jy0FUIZp7wiRB+0BP7J2g==
5183+
51335184
[email protected], unpipe@~1.0.0:
51345185
version "1.0.0"
51355186
resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec"
@@ -5247,6 +5298,11 @@ word-wrap@^1.2.3:
52475298
resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.3.tgz#610636f6b1f703891bd34771ccb17fb93b47079c"
52485299
integrity sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==
52495300

5301+
wordnet-db@^3.1.11:
5302+
version "3.1.14"
5303+
resolved "https://registry.yarnpkg.com/wordnet-db/-/wordnet-db-3.1.14.tgz#7ba1ec2cb5730393f0856efcc738a60085426199"
5304+
integrity sha512-zVyFsvE+mq9MCmwXUWHIcpfbrHHClZWZiVOzKSxNJruIcFn2RbY55zkhiAMMxM8zCVSmtNiViq8FsAZSFpMYag==
5305+
52505306
52515307
version "6.1.0"
52525308
resolved "https://registry.yarnpkg.com/workerpool/-/workerpool-6.1.0.tgz#a8e038b4c94569596852de7a8ea4228eefdeb37b"

0 commit comments

Comments
 (0)