From 7aa147758d4bfa243e0a6423917200b1cce83df7 Mon Sep 17 00:00:00 2001 From: nalbam Date: Wed, 5 Jun 2024 13:09:31 +0900 Subject: [PATCH 1/7] add vision --- handler.py | 68 ++++++++++++++++++++++-------------------------------- 1 file changed, 27 insertions(+), 41 deletions(-) diff --git a/handler.py b/handler.py index cc0ee03..eb8a389 100644 --- a/handler.py +++ b/handler.py @@ -30,6 +30,8 @@ # Set up the allowed channel ID ALLOWED_CHANNEL_IDS = os.environ.get("ALLOWED_CHANNEL_IDS", "AAA,CCC,EEE,GGG") +ENABLE_IMAGE = os.environ.get("ENABLE_IMAGE", "False") + # Set up System messages SYSTEM_MESSAGE = os.environ.get("SYSTEM_MESSAGE", "None") @@ -91,7 +93,7 @@ def chat_update(channel, ts, message, blocks=None): app.client.chat_update(channel=channel, ts=ts, text=message, blocks=blocks) -def invoke_claude_3(messages): +def invoke_claude_3(content): """ Invokes Anthropic Claude 3 Sonnet to run an inference using the input provided in the request body. @@ -102,6 +104,14 @@ def invoke_claude_3(messages): text = "" + messages = [] + messages.append( + { + "role": "user", + "content": content, + }, + ) + try: body = { "anthropic_version": ANTHROPIC_VERSION, @@ -118,13 +128,13 @@ def invoke_claude_3(messages): ) # Process and print the response - result = json.loads(response.get("body").read()) + body = json.loads(response.get("body").read()) - print("response: {}".format(result)) + print("response: {}".format(body)) - content = result.get("content", []) + result = body.get("content", []) - for output in content: + for output in result: text = output["text"] return text @@ -133,17 +143,6 @@ def invoke_claude_3(messages): print("Error: {}".format(e)) -# Reply to the message -def reply_text(messages, channel, ts, user): - message = invoke_claude_3(messages) - - message = message.replace("**", "*") - - chat_update(channel, ts, message) - - return message - - # Get thread messages using conversations.replies API method def conversations_replies(channel, ts, client_msg_id): messages = [] @@ -206,8 +205,8 @@ def conversation(say: Say, thread_ts, content, channel, user, client_msg_id): prompt = content[0]["text"] type = "text" - # if "그려줘" in prompt: - # type = "image" + if ENABLE_IMAGE == "True" and "그려줘" in prompt: + type = "image" prompts = [] @@ -225,16 +224,8 @@ def conversation(say: Say, thread_ts, content, channel, user, client_msg_id): content[0]["text"] = "Describe the image in great detail as if viewing a photo." - messages = [] - messages.append( - { - "role": "user", - "content": content, - }, - ) - # Send the prompt to Bedrock - message = invoke_claude_3(messages) + message = invoke_claude_3(content) prompts.append(message) @@ -246,23 +237,18 @@ def conversation(say: Say, thread_ts, content, channel, user, client_msg_id): try: chat_update(channel, latest_ts, "응답 기다리는 중... " + BOT_CURSOR) - messages = [] - messages.append( - { - "role": "user", - "content": [ - { - "type": "text", - "text": "\n\n\n".join(prompts), - } - ], - }, - ) + prompt = "\n\n\n".join(prompts) + + print("conversation: {}".format(prompt)) - print("conversation: {}".format(messages)) + content[0]["text"] = prompt # Send the prompt to Bedrock - message = reply_text(messages, channel, latest_ts, user) + message = invoke_claude_3(content) + + message = message.replace("**", "*") + + chat_update(channel, latest_ts, message) print("conversation: {}".format(message)) From f117ff8c4927f5e8467b19a85b5b2db2a8fdaa64 Mon Sep 17 00:00:00 2001 From: nalbam Date: Wed, 5 Jun 2024 14:17:39 +0900 Subject: [PATCH 2/7] add invoke_stable_diffusion --- bedrock/invoke_claude_3.py | 6 +- bedrock/invoke_claude_3_image.py | 6 +- bedrock/invoke_stable_diffusion.py | 8 +- handler.py | 125 ++++++++++++++++++++++------- 4 files changed, 110 insertions(+), 35 deletions(-) diff --git a/bedrock/invoke_claude_3.py b/bedrock/invoke_claude_3.py index 5737106..95f50f6 100644 --- a/bedrock/invoke_claude_3.py +++ b/bedrock/invoke_claude_3.py @@ -58,11 +58,11 @@ def invoke_claude_3(prompt): ) # Process and print the response - result = json.loads(response.get("body").read()) + body = json.loads(response.get("body").read()) - # print("response: {}".format(result)) + # print("response: {}".format(body)) - content = result.get("content", []) + content = body.get("content", []) for output in content: print(output["text"]) diff --git a/bedrock/invoke_claude_3_image.py b/bedrock/invoke_claude_3_image.py index f202050..d728f66 100644 --- a/bedrock/invoke_claude_3_image.py +++ b/bedrock/invoke_claude_3_image.py @@ -73,11 +73,11 @@ def invoke_claude_3(prompt): ) # Process and print the response - result = json.loads(response.get("body").read()) + body = json.loads(response.get("body").read()) - # print("response: {}".format(result)) + # print("response: {}".format(body)) - content = result.get("content", []) + content = body.get("content", []) for output in content: print(output["text"]) diff --git a/bedrock/invoke_stable_diffusion.py b/bedrock/invoke_stable_diffusion.py index 1f57a53..5de37c3 100644 --- a/bedrock/invoke_stable_diffusion.py +++ b/bedrock/invoke_stable_diffusion.py @@ -56,8 +56,12 @@ def invoke_stable_diffusion(prompt, seed=0, style_preset="photographic"): body=json.dumps(body), ) - response_body = json.loads(response["body"].read()) - base64_image = response_body.get("artifacts")[0].get("base64") + body = json.loads(response["body"].read()) + + # body["artifacts"][0]["base64"] = None + # print("response: {}".format(body)) + + base64_image = body.get("artifacts")[0].get("base64") base64_bytes = base64_image.encode("ascii") image_bytes = base64.b64decode(base64_bytes) diff --git a/handler.py b/handler.py index eb8a389..ba747fe 100644 --- a/handler.py +++ b/handler.py @@ -7,6 +7,9 @@ import time import base64 import requests +import io + +from PIL import Image from slack_bolt import App, Say from slack_bolt.adapter.aws_lambda import SlackRequestHandler @@ -102,21 +105,16 @@ def invoke_claude_3(content): :return: Inference response from the model. """ - text = "" - - messages = [] - messages.append( - { - "role": "user", - "content": content, - }, - ) - try: body = { "anthropic_version": ANTHROPIC_VERSION, "max_tokens": ANTHROPIC_TOKENS, - "messages": messages, + "messages": [ + { + "role": "user", + "content": content, + }, + ], } if SYSTEM_MESSAGE != "None": @@ -142,6 +140,58 @@ def invoke_claude_3(content): except Exception as e: print("Error: {}".format(e)) + return None + + +def invoke_stable_diffusion(prompt, seed=0, style_preset="photographic"): + """ + Invokes the Stability.ai Stable Diffusion XL model to create an image using + the input provided in the request body. + + :param prompt: The prompt that you want Stable Diffusion to use for image generation. + :param seed: Random noise seed (omit this option or use 0 for a random seed) + :param style_preset: Pass in a style preset to guide the image model towards + a particular style. + :return: Base64-encoded inference response from the model. + """ + + try: + # The different model providers have individual request and response formats. + # For the format, ranges, and available style_presets of Stable Diffusion models refer to: + # https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-stability-diffusion.html + + body = { + "text_prompts": [{"text": prompt}], + "seed": seed, + "cfg_scale": 10, + "steps": 30, + "samples": 1, + } + + if style_preset: + body["style_preset"] = style_preset + + response = bedrock.invoke_model( + modelId=IMAGE_MODEL_ID, + body=json.dumps(body), + ) + + body = json.loads(response["body"].read()) + + base64_image = body.get("artifacts")[0].get("base64") + base64_bytes = base64_image.encode("ascii") + image_bytes = base64.b64decode(base64_bytes) + + image = Image.open(io.BytesIO(image_bytes)) + image.show() + + return image_bytes + + except Exception as e: + print("Error: {}".format(e)) + + return None + # Get thread messages using conversations.replies API method def conversations_replies(channel, ts, client_msg_id): @@ -229,35 +279,56 @@ def conversation(say: Say, thread_ts, content, channel, user, client_msg_id): prompts.append(message) - # Send the prompt to Bedrock - if prompt: - prompts.append(prompt) + if prompt: + prompts.append(prompt) - # Send the prompt to Bedrock - try: - chat_update(channel, latest_ts, "응답 기다리는 중... " + BOT_CURSOR) + chat_update(channel, latest_ts, "이미지 생성 준비 중... " + BOT_CURSOR) - prompt = "\n\n\n".join(prompts) + prompts.append( + "Convert the above sentence into a command for stable-diffusion to generate an image within 1000 characters. Just give me a prompt." + ) - print("conversation: {}".format(prompt)) + prompt = "\n\n\n".join(prompts) - content[0]["text"] = prompt + content = [] + content.append({"type": "text", "text": prompt}) # Send the prompt to Bedrock message = invoke_claude_3(content) - message = message.replace("**", "*") + else: + chat_update(channel, latest_ts, "응답 기다리는 중... " + BOT_CURSOR) - chat_update(channel, latest_ts, message) + if prompt: + prompts.append(prompt) - print("conversation: {}".format(message)) + prompt = "\n\n\n".join(prompts) - except Exception as e: - print("conversation: Error handling message: {}".format(e)) + content[0]["text"] = prompt - message = f"```{e}```" + # Send the prompt to Bedrock + message = invoke_claude_3(content) - chat_update(channel, latest_ts, message) + message = message.replace("**", "*") + + # Update the message in Slack + chat_update(channel, latest_ts, message) + + # Generate an image + if type == "image" and len(content) > 1: + prompt = message + + image = invoke_stable_diffusion(prompt) + + if image: + # Send the image to Slack + app.client.files_upload( + channels=channel, + file=io.BytesIO(image), + title="Generated Image", + filename="image.jpg", + initial_comment="Here is the generated image.", + ) # Get image from URL From 2e711f85a184607a2dc3ba8f07a855a2a0541c0c Mon Sep 17 00:00:00 2001 From: nalbam Date: Wed, 5 Jun 2024 14:31:57 +0900 Subject: [PATCH 3/7] fix chat_update --- handler.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/handler.py b/handler.py index ba747fe..2b1f6e9 100644 --- a/handler.py +++ b/handler.py @@ -316,9 +316,11 @@ def conversation(say: Say, thread_ts, content, channel, user, client_msg_id): # Generate an image if type == "image" and len(content) > 1: - prompt = message + chat_update(channel, latest_ts, "이미지 그리는 중... " + BOT_CURSOR) - image = invoke_stable_diffusion(prompt) + image = invoke_stable_diffusion(message) + + chat_update(channel, latest_ts, message) if image: # Send the image to Slack From b9325223e0734936b335b79b568a219e390c8fbc Mon Sep 17 00:00:00 2001 From: nalbam Date: Wed, 5 Jun 2024 14:37:20 +0900 Subject: [PATCH 4/7] add env ENABLE_IMAGE --- .github/workflows/push.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index d023492..7edf2fc 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -14,6 +14,7 @@ env: IMAGE_MODEL_ID: ${{ secrets.IMAGE_MODEL_ID }} SYSTEM_MESSAGE: ${{ vars.SYSTEM_MESSAGE }} ALLOWED_CHANNEL_IDS: ${{ secrets.ALLOWED_CHANNEL_IDS }} + ENABLE_IMAGE: ${{ secrets.ENABLE_IMAGE }} jobs: deploy: @@ -52,6 +53,7 @@ jobs: echo "IMAGE_MODEL_ID=${IMAGE_MODEL_ID}" >> .env echo "SYSTEM_MESSAGE=${SYSTEM_MESSAGE}" >> .env echo "ALLOWED_CHANNEL_IDS=${ALLOWED_CHANNEL_IDS}" >> .env + echo "ENABLE_IMAGE=${ENABLE_IMAGE}" >> .env - name: Deploy to AWS Lambda 🚀 env: From 9072fda42a04a34e90416e45aed2842e8e94cbc4 Mon Sep 17 00:00:00 2001 From: nalbam Date: Wed, 5 Jun 2024 14:44:53 +0900 Subject: [PATCH 5/7] fix type image --- handler.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/handler.py b/handler.py index 2b1f6e9..361c260 100644 --- a/handler.py +++ b/handler.py @@ -258,6 +258,8 @@ def conversation(say: Say, thread_ts, content, channel, user, client_msg_id): if ENABLE_IMAGE == "True" and "그려줘" in prompt: type = "image" + print("conversation: {}".format(type)) + prompts = [] # Get the thread messages @@ -279,9 +281,10 @@ def conversation(say: Say, thread_ts, content, channel, user, client_msg_id): prompts.append(message) - if prompt: - prompts.append(prompt) + if prompt: + prompts.append(prompt) + if type == "image": chat_update(channel, latest_ts, "이미지 생성 준비 중... " + BOT_CURSOR) prompts.append( @@ -299,9 +302,6 @@ def conversation(say: Say, thread_ts, content, channel, user, client_msg_id): else: chat_update(channel, latest_ts, "응답 기다리는 중... " + BOT_CURSOR) - if prompt: - prompts.append(prompt) - prompt = "\n\n\n".join(prompts) content[0]["text"] = prompt From f75862e462954afd11c0f9c6c3330a90f8482728 Mon Sep 17 00:00:00 2001 From: nalbam Date: Wed, 5 Jun 2024 14:47:55 +0900 Subject: [PATCH 6/7] fix files_upload thread_ts --- handler.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/handler.py b/handler.py index 361c260..4e15cb8 100644 --- a/handler.py +++ b/handler.py @@ -299,27 +299,11 @@ def conversation(say: Say, thread_ts, content, channel, user, client_msg_id): # Send the prompt to Bedrock message = invoke_claude_3(content) - else: - chat_update(channel, latest_ts, "응답 기다리는 중... " + BOT_CURSOR) - - prompt = "\n\n\n".join(prompts) - - content[0]["text"] = prompt - - # Send the prompt to Bedrock - message = invoke_claude_3(content) - - message = message.replace("**", "*") - - # Update the message in Slack - chat_update(channel, latest_ts, message) - - # Generate an image - if type == "image" and len(content) > 1: chat_update(channel, latest_ts, "이미지 그리는 중... " + BOT_CURSOR) image = invoke_stable_diffusion(message) + # Update the message in Slack chat_update(channel, latest_ts, message) if image: @@ -330,8 +314,24 @@ def conversation(say: Say, thread_ts, content, channel, user, client_msg_id): title="Generated Image", filename="image.jpg", initial_comment="Here is the generated image.", + thread_ts=latest_ts, ) + else: + chat_update(channel, latest_ts, "응답 기다리는 중... " + BOT_CURSOR) + + prompt = "\n\n\n".join(prompts) + + content[0]["text"] = prompt + + # Send the prompt to Bedrock + message = invoke_claude_3(content) + + message = message.replace("**", "*") + + # Update the message in Slack + chat_update(channel, latest_ts, message) + # Get image from URL def get_image_from_url(image_url, token=None): From 5c4127446b80c176df53c62548046a4989a2d44a Mon Sep 17 00:00:00 2001 From: nalbam Date: Wed, 5 Jun 2024 14:51:57 +0900 Subject: [PATCH 7/7] fix files_upload_v2 --- handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/handler.py b/handler.py index 4e15cb8..5f06562 100644 --- a/handler.py +++ b/handler.py @@ -308,7 +308,7 @@ def conversation(say: Say, thread_ts, content, channel, user, client_msg_id): if image: # Send the image to Slack - app.client.files_upload( + app.client.files_upload_v2( channels=channel, file=io.BytesIO(image), title="Generated Image",