CDK Entest

Setup NextJS#

Let create a new NextJS project

npx create-next-app@latest

Then install dependencies, here is package.json

{
  "name": "next-prisma-hello",
  "version": "0.1.0",
  "private": true,
  "scripts": {
    "dev": "next dev",
    "build": "next build",
    "start": "next start",
    "lint": "next lint"
  },
  "dependencies": {
    "@aws-sdk/client-bedrock-runtime": "^3.490.0",
    "@prisma/client": "^5.8.0",
    "ai": "^2.2.33",
    "next": "14.0.4",
    "package.json": "^2.0.1",
    "react": "^18",
    "react-dom": "^18"
  },
  "devDependencies": {
    "@types/node": "^20",
    "@types/react": "^18",
    "@types/react-dom": "^18",
    "autoprefixer": "^10.0.1",
    "eslint": "^8",
    "eslint-config-next": "14.0.4",
    "postcss": "^8",
    "prisma": "^5.8.0",
    "tailwindcss": "^3.3.0",
    "typescript": "^5"
  }
}

Project structure

|--app
   |--page.tsx
   |--api
      |--chat
         |--route.ts
   |--image
      |--page.tsx
|--cdk
   |--bin
      |--cdk.ts
   |--lib
      |--cdk-stack.ts
|--next.config.js
|--package.json
|--Dockerfile
|--.dockerignore
|--build.py

Chat Function#

Simple conversation memory by using vercel SDK and bedrock
Streaming chat response
Frontend uses useChat react hook

Let implement the route.ts for chat

import {
  BedrockRuntime,
  InvokeModelWithResponseStreamCommand,
} from "@aws-sdk/client-bedrock-runtime";
import { AWSBedrockAnthropicStream, StreamingTextResponse } from "ai";
import { experimental_buildAnthropicPrompt } from "ai/prompts";

// IMPORTANT! Set the runtime to edge
// export const runtime = "edge";

const bedrockClient = new BedrockRuntime({
  region: "us-east-1",
  // region: process.env.AWS_REGION ?? "us-east-1",
  // credentials: {
  //   accessKeyId: process.env.AWS_ACCESS_KEY_ID ?? "",
  //   secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY ?? "",
  //   sessionToken: process.env.AWS_SESSION_TOKEN ?? "",
  // },
});

export async function POST(req: Request) {
  // Extract the `prompt` from the body of the request
  const { messages } = await req.json();

  console.log(messages);

  console.log(experimental_buildAnthropicPrompt(messages));

  // Ask Claude for a streaming chat completion given the prompt
  const bedrockResponse = await bedrockClient.send(
    new InvokeModelWithResponseStreamCommand({
      modelId: "anthropic.claude-v2",
      contentType: "application/json",
      accept: "application/json",
      body: JSON.stringify({
        prompt: experimental_buildAnthropicPrompt(messages),
        max_tokens_to_sample: 2048,
      }),
    })
  );

  // Convert the response into a friendly text-stream
  const stream = AWSBedrockAnthropicStream(bedrockResponse);

  // Respond with the stream
  return new StreamingTextResponse(stream);
}

The useChat hook and submit chat on enter key

const { messages, input, handleInputChange, handleSubmit } = useChat({
  api: "./api/chat",
});

<form onSubmit={handleSubmit}>
  <input
    className="fixed bottom-0 w-full max-w-md p-2 mb-8 border border-gray-300 rounded shadow-xl"
    value={input}
    placeholder="Say something..."
    onChange={handleInputChange}
  />
</form>;

Image Function#

Then let implement server function for generating image

Generate image
Upload to S3
Generate signed url

"use server";

import {
  GetObjectCommand,
  S3Client,
  PutObjectCommand,
} from "@aws-sdk/client-s3";

import { getSignedUrl } from "@aws-sdk/s3-request-presigner";

import {
  BedrockRuntime,
  InvokeModelCommand,
} from "@aws-sdk/client-bedrock-runtime";
import * as fs from "fs";

const bedrock = new BedrockRuntime({ region: "us-east-1" });

const s3Client = new S3Client({ region: "us-east-1" });

const genImage = async ({ prompt }: { prompt: string }) => {
  let url = "";

  const body = {
    text_prompts: [{ text: prompt, weight: 1 }],
    seed: 3,
    cfg_scale: 10,
    samples: 1,
    steps: 50,
    style_preset: "anime",
    height: 1024,
    width: 1024,
  };

  const command = new InvokeModelCommand({
    body: JSON.stringify(body),
    modelId: "stability.stable-diffusion-xl-v1",
    contentType: "application/json",
    accept: "image/png",
  });

  try {
    console.log(prompt);
    const imageName = "sample" + Date.now().toString() + ".png";
    const key = `next-vercel-ai/${imageName}`;
    const response = await bedrock.send(command);

    // fs.writeFile(`./public/${imageName}`, response["body"], () => {
    //   console.log("OK");
    // });

    // upload to s3 input location
    await s3Client.send(
      new PutObjectCommand({
        Bucket: process.env.BUCKET,
        Key: key,
        Body: response["body"],
      })
    );

    // generate signed url
    const commandGetUrl = new GetObjectCommand({
      Bucket: process.env.BUCKET,
      Key: key,
    });

    url = await getSignedUrl(s3Client as any, commandGetUrl as any, {
      expiresIn: 3600,
    });

    console.log(url);
  } catch (error) {
    console.log(error);
  }

  return url;
};

export { genImage };

Docker#

To build and deploy with Apprunner please take note the next.config.js

/** @type {import('next').NextConfig} */
const nextConfig = {
  output: "standalone",
};

module.exports = nextConfig;

Then prepare Dockerfile with multiple stage to optimize the image size

FROM node:18-alpine AS base

# Install dependencies only when needed
FROM base AS deps
# Check https://github.com/nodejs/docker-node/tree/b4117f9333da4138b03a546ec926ef50a31506c3#nodealpine to understand why libc6-compat might be needed.
RUN apk add --no-cache libc6-compat
WORKDIR /app

# Install dependencies based on the preferred package manager
COPY package.json yarn.lock* package-lock.json* pnpm-lock.yaml* ./
RUN \
  if [ -f yarn.lock ]; then yarn --frozen-lockfile; \
  elif [ -f package-lock.json ]; then npm ci; \
  elif [ -f pnpm-lock.yaml ]; then corepack enable pnpm && pnpm i --frozen-lockfile; \
  else echo "Lockfile not found." && exit 1; \
  fi


# Rebuild the source code only when needed
FROM base AS builder
WORKDIR /app
COPY --from=deps /app/node_modules ./node_modules
COPY . .

# Next.js collects completely anonymous telemetry data about general usage.
# Learn more here: https://nextjs.org/telemetry
# Uncomment the following line in case you want to disable telemetry during the build.
# ENV NEXT_TELEMETRY_DISABLED 1

RUN \
  if [ -f yarn.lock ]; then yarn run build; \
  elif [ -f package-lock.json ]; then npm run build; \
  elif [ -f pnpm-lock.yaml ]; then corepack enable pnpm && pnpm run build; \
  else echo "Lockfile not found." && exit 1; \
  fi

# Production image, copy all the files and run next
FROM base AS runner
WORKDIR /app

ENV NODE_ENV production
# Uncomment the following line in case you want to disable telemetry during runtime.
# ENV NEXT_TELEMETRY_DISABLED 1

RUN addgroup --system --gid 1001 nodejs
RUN adduser --system --uid 1001 nextjs

COPY --from=builder /app/public ./public

# Set the correct permission for prerender cache
RUN mkdir .next
RUN chown nextjs:nodejs .next

# Automatically leverage output traces to reduce image size
# https://nextjs.org/docs/advanced-features/output-file-tracing
COPY --from=builder --chown=nextjs:nodejs /app/.next/standalone ./
COPY --from=builder --chown=nextjs:nodejs /app/.next/static ./.next/static

USER nextjs

EXPOSE 3000

ENV PORT 3000
# set hostname to localhost
ENV HOSTNAME "0.0.0.0"

# server.js is created by next build from the standalone output
# https://nextjs.org/docs/pages/api-reference/next-config-js/output
CMD ["node", "server.js"]

And .dockerignore file

Dockerfile.dockerignore;
node_modules;
npm - debug.log;
README.md.next.git;

AppRunner#

Let create a stack to deploy the app on apprunner

Build role to pull ecr image
Task role to invoke bedrock model

import { Stack, StackProps, aws_apprunner, aws_iam } from "aws-cdk-lib";
import { Effect } from "aws-cdk-lib/aws-iam";
import { Construct } from "constructs";

interface AppRunnerProps extends StackProps {
  ecr: string;
  bucket: string;
}

export class AppRunnerStack extends Stack {
  constructor(scope: Construct, id: string, props: AppRunnerProps) {
    super(scope, id, props);

    const buildRole = new aws_iam.Role(this, "RoleForAppRunnerPullEcrBedrock", {
      assumedBy: new aws_iam.ServicePrincipal("build.apprunner.amazonaws.com"),
      roleName: "RoleForAppRunnerPullEcrBedrock",
    });

    buildRole.addToPolicy(
      new aws_iam.PolicyStatement({
        effect: Effect.ALLOW,
        resources: ["*"],
        actions: ["ecr:*"],
      })
    );

    const instanceRole = new aws_iam.Role(
      this,
      "InstanceRoleForApprunerBedrock",
      {
        assumedBy: new aws_iam.ServicePrincipal(
          "tasks.apprunner.amazonaws.com"
        ),
        roleName: "InstanceRoleForApprunnerBedrock",
      }
    );

    instanceRole.addToPolicy(
      new aws_iam.PolicyStatement({
        effect: Effect.ALLOW,
        resources: [
          "arn:aws:bedrock:us-east-1::foundation-model/anthropic.claude-v2",
          "arn:aws:bedrock:us-east-1::foundation-model/stability.stable-diffusion-xl-v1",
        ],
        actions: [
          "bedrock:InvokeModel",
          "bedrock:InvokeModelWithResponseStream",
        ],
      })
    );

    instanceRole.addToPolicy(
      new aws_iam.PolicyStatement({
        effect: Effect.ALLOW,
        resources: [`arn:aws:s3:::${props.bucket}/*`],
        actions: ["s3:PutObject", "s3:GetObject"],
      })
    );

    const autoscaling = new aws_apprunner.CfnAutoScalingConfiguration(
      this,
      "AutoScalingForGoApp",
      {
        autoScalingConfigurationName: "AutoScalingForGoApp",
        // min number instance
        minSize: 1,
        // max number instance
        maxSize: 10,
        // max concurrent request per instance
        maxConcurrency: 100,
      }
    );

    const apprunner = new aws_apprunner.CfnService(this, "NextBedrockService", {
      serviceName: "NextBedrockService",
      sourceConfiguration: {
        authenticationConfiguration: {
          accessRoleArn: buildRole.roleArn,
        },

        autoDeploymentsEnabled: false,

        imageRepository: {
          imageIdentifier: props.ecr,
          imageRepositoryType: "ECR",
          imageConfiguration: {
            port: "3000",
            runtimeEnvironmentVariables: [
              {
                name: "BUCKET",
                value: "demo",
              },
              {
                name: "HOSTNAME",
                value: "0.0.0.0",
              },
              {
                name: "PORT",
                value: "3000",
              },
            ],
            // startCommand: "",
          },
        },
      },

      instanceConfiguration: {
        cpu: "1 vCPU",
        memory: "2 GB",
        instanceRoleArn: instanceRole.roleArn,
      },

      observabilityConfiguration: {
        observabilityEnabled: false,
      },

      autoScalingConfigurationArn: autoscaling.ref,
    });

    apprunner.addDependency(autoscaling);
  }
}

Docker#

Stop all running containers and remove images

docker stop $(docker ps -a -q)
docker rmi -f $(docker images -aq)

Delete existing images

docker rmi -f $(docker images -aq)

Build and tag image

docker build --tag entest .

Stop all processing running on port 3000

sudo kill -9 $(lsof -i:3000 -t)

Here is build script

build.py

import os
import boto3


# parameters
deploy = 1

# # parameters
REGION = "us-east-1"
ACCOUNT = os.environ["ACCOUNT_ID"]

# delete all docker images
os.system("sudo docker system prune -a")

# build next-bedrock image
os.system("sudo docker build -t next-bedrock . ")

#  aws ecr login
os.system(f"aws ecr get-login-password --region {REGION} | sudo docker login --username AWS --password-stdin {ACCOUNT}.dkr.ecr.{REGION}.amazonaws.com")

# get image id
IMAGE_ID=os.popen("sudo docker images -q next-bedrock:latest").read()

# tag next-bedrock image
os.system(f"sudo docker tag {IMAGE_ID.strip()} {ACCOUNT}.dkr.ecr.{REGION}.amazonaws.com/next-bedrock:latest")

# create ecr repository
os.system(f"aws ecr create-repository --registry-id {ACCOUNT} --repository-name next-bedrock")

# push image to ecr
os.system(f"sudo docker push {ACCOUNT}.dkr.ecr.{REGION}.amazonaws.com/next-bedrock:latest")

# run locally to test
# os.system(f"sudo docker run -d -p 3000:3000 next-bedrock:latest")


# apprunner deploy
if deploy == 1:
    apprunner = boto3.client('apprunner')
    apprunner.start_deployment(
     ServiceArn=f"arn:aws:apprunner:{REGION}:{ACCOUNT}:service/NextBedrockService/xxx"
     )

FrontEnd Chat#

First let create a chat page which submit on enter key

"use client";

import { Message } from "ai";
import { useChat } from "ai/react";

export default function Chat() {
  const { messages, input, handleInputChange, handleSubmit } = useChat({
    api: "./api/chat",
  });

  // Generate a map of message role to text color
  const roleToColorMap: Record<Message["role"], string> = {
    system: "red",
    user: "black",
    function: "blue",
    tool: "purple",
    assistant: "green",
    data: "orange",
  };

  return (
    <div className="flex flex-col w-full max-w-md py-24 mx-auto stretch">
      {messages.length > 0
        ? messages.map((m) => (
            <div
              key={m.id}
              className="whitespace-pre-wrap"
              style={{ color: roleToColorMap[m.role] }}
            >
              <strong>{`${m.role}: `}</strong>
              {m.content || JSON.stringify(m.function_call)}
              <br />
              <br />
            </div>
          ))
        : null}

      <form onSubmit={handleSubmit}>
        <input
          className="fixed bottom-0 w-full max-w-md p-2 mb-8 border border-gray-300 rounded shadow-xl"
          value={input}
          placeholder="Say something..."
          onChange={handleInputChange}
        />
      </form>
    </div>
  );
}

FrontEnd Image#

Second let create an image generation page

"use client";

import { useEffect, useState } from "react";
import { genImage } from "./actions";

const SendIcon = ({ className }: { className?: string }) => {
  return (
    <svg
      xmlns="http://www.w3.org/2000/svg"
      viewBox="0 0 16 16"
      fill="none"
      className={className}
      strokeWidth="2"
    >
      <path
        d="M.5 1.163A1 1 0 0 1 1.97.28l12.868 6.837a1 1 0 0 1 0 1.766L1.969 15.72A1 1 0 0 1 .5 14.836V10.33a1 1 0 0 1 .816-.983L8.5 8 1.316 6.653A1 1 0 0 1 .5 5.67V1.163Z"
        fill="currentColor"
      ></path>
    </svg>
  );
};

const HomePage = () => {
  const [url, setUrl] = useState<string>("");

  useEffect(() => {}, [url]);

  return (
    <main>
      <div className="max-w-3xl mx-auto">
        <form className="mt-10 px-10">
          <div className="relative ">
            <textarea
              rows={2}
              id="prompt"
              name="prompt"
              className="w-[100%] bg-gray-300 px-5 py-5 rounded-sm"
              onKeyDown={async (event) => {
                if (event.key === "Enter" && event.shiftKey === false) {
                  event.preventDefault();
                  setUrl("");
                  document.getElementById("modal")!.style.display = "block";
                  const url = await genImage({
                    prompt: (
                      document.getElementById("prompt") as HTMLInputElement
                    ).value,
                  });
                  setUrl(url);
                  document.getElementById("modal")!.style.display = "none";
                }
              }}
            ></textarea>

            <button
              className="absolute top-[50%] translate-y-[-50%] right-1 flex items-center justify-center rounded-md w-10 h-16 bg-green-500 hover:bg-green-600"
              onClick={async (event) => {
                event.preventDefault();
                setUrl("");
                document.getElementById("modal")!.style.display = "block";
                const url = await genImage({
                  prompt: (
                    document.getElementById("prompt") as HTMLInputElement
                  ).value,
                });
                setUrl(url);
                document.getElementById("modal")!.style.display = "none";
              }}
            >
              <SendIcon className="h-4 w-4 text-white"></SendIcon>
            </button>
          </div>
        </form>
        <div className="mt-10 px-10">
          <img src={url}></img>
        </div>

        <div
          id="modal"
          className="fixed top-0 left-0 bg-slate-400 min-h-screen w-full opacity-60"
          hidden
        >
          <div className="min-h-screen flex justify-center items-center">
            <h1>Wait a few second!</h1>
          </div>
        </div>
      </div>
    </main>
  );
};

export default HomePage;

And here is the image generation server function

Call bedrock stability model
Save image to S3
Return signed URL and display image

"use server";

import {
  GetObjectCommand,
  S3Client,
  PutObjectCommand,
} from "@aws-sdk/client-s3";

import { getSignedUrl } from "@aws-sdk/s3-request-presigner";

import {
  BedrockRuntime,
  InvokeModelCommand,
} from "@aws-sdk/client-bedrock-runtime";
import * as fs from "fs";

const bedrock = new BedrockRuntime({ region: "us-east-1" });

const s3Client = new S3Client({ region: "us-east-1" });

const genImage = async ({ prompt }: { prompt: string }) => {
  let url = "";

  const body = {
    text_prompts: [{ text: prompt, weight: 1 }],
    seed: 3,
    cfg_scale: 10,
    samples: 1,
    steps: 50,
    style_preset: "anime",
    height: 1024,
    width: 1024,
  };

  const command = new InvokeModelCommand({
    body: JSON.stringify(body),
    modelId: "stability.stable-diffusion-xl-v1",
    contentType: "application/json",
    accept: "image/png",
  });

  try {
    console.log(prompt);
    const imageName = "sample" + Date.now().toString() + ".png";
    const key = `next-vercel-ai/${imageName}`;
    const response = await bedrock.send(command);

    // fs.writeFile(`./public/${imageName}`, response["body"], () => {
    //   console.log("OK");
    // });

    // upload to s3 input location
    await s3Client.send(
      new PutObjectCommand({
        Bucket: process.env.BUCKET,
        Key: key,
        Body: response["body"],
      })
    );

    // generate signed url
    const commandGetUrl = new GetObjectCommand({
      Bucket: process.env.BUCKET,
      Key: key,
    });

    url = await getSignedUrl(s3Client as any, commandGetUrl as any, {
      expiresIn: 3600,
    });

    console.log(url);
  } catch (error) {
    console.log(error);
  }

  return url;
};

export { genImage };