CDK Entest

Introduction#

chat using Bedrock Claude2
chat using Bedrock Claude3
embedding vector opensearch
basic prompt

Chat Claude2#

struct to form Bedrock request and parse response
stream Bedrock response to client

Let create some struct for request and response

type Request struct {
	Prompt            string   `json:"prompt"`
	MaxTokensToSample int      `json:"max_tokens_to_sample"`
	Temperature       float64  `json:"temperature,omitempty"`
	TopP              float64  `json:"top_p,omitempty"`
	TopK              int      `json:"top_k,omitempty"`
	StopSequences     []string `json:"stop_sequences,omitempty"`
}

type Response struct {
	Completion string `json:"completion"`
}

type HelloHandler struct{}

type Query struct {
	Topic string `json:"topic"`
}

Let create a function to send request to Bedrock and stream back client

func HandleBedrockClaude2Chat(w http.ResponseWriter, r *http.Request) {

	const claudePromptFormat = "\n\nHuman: %s\n\nAssistant:"

	var query Query
	var message string

	// parse mesage from request
	error := json.NewDecoder(r.Body).Decode(&query)

	if error != nil {
		message = "how to learn japanese as quick as possible?"
		panic(error)
	}

	message = query.Topic

	fmt.Println(message)

	prompt := "" + fmt.Sprintf(claudePromptFormat, message)

	payload := Request{
		Prompt:            prompt,
		MaxTokensToSample: 2048,
	}

	payloadBytes, error := json.Marshal(payload)

	if error != nil {
		fmt.Fprintf(w, "ERROR")
		// return "", error
	}

	output, error := BedrockClient.InvokeModelWithResponseStream(
		context.Background(),
		&bedrockruntime.InvokeModelWithResponseStreamInput{
			Body:        payloadBytes,
			ModelId:     aws.String("anthropic.claude-v2"),
			ContentType: aws.String("application/json"),
		},
	)

	if error != nil {
		fmt.Fprintf(w, "ERROR")
		// return "", error
	}

	for event := range output.GetStream().Events() {
		switch v := event.(type) {
		case *types.ResponseStreamMemberChunk:

			//fmt.Println("payload", string(v.Value.Bytes))

			var resp Response
			err := json.NewDecoder(bytes.NewReader(v.Value.Bytes)).Decode(&resp)
			if err != nil {
				fmt.Fprintf(w, "ERROR")
				// return "", err
			}

			fmt.Println(resp.Completion)

			fmt.Fprintf(w, resp.Completion)
			if f, ok := w.(http.Flusher); ok {
				f.Flush()
			} else {
				fmt.Println("Damn, no flush")
			}

		case *types.UnknownUnionMember:
			fmt.Println("unknown tag:", v.Tag)

		default:
			fmt.Println("union is nil or unknown type")
		}
	}
}

Here is full code

chat-claude2.go

package main

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/aws/aws-sdk-go-v2/aws"
	"github.com/aws/aws-sdk-go-v2/service/bedrockruntime"
	"github.com/aws/aws-sdk-go-v2/service/bedrockruntime/types"
)

type Content struct {
	Type string `json:"type"`
	Text string `json:"text"`
}

type Message struct {
	Role    string    `json:"role"`
	Content []Content `json:"content"`
}

type RequestBodyClaude3 struct {
	MaxTokensToSample int       `json:"max_tokens"`
	Temperature       float64   `json:"temperature,omitempty"`
	AnthropicVersion  string    `json:"anthropic_version"`
	Messages          []Message `json:"messages"`
}

type Delta struct {
	Type string `json:"type"`
	Text string `json:"text"`
}

type ResponseClaude3 struct {
	Type  string `json:"type"`
	Index int    `json:"index"`
	Delta Delta  `json:"delta"`
}

type Request struct {
	Prompt            string   `json:"prompt"`
	MaxTokensToSample int      `json:"max_tokens_to_sample"`
	Temperature       float64  `json:"temperature,omitempty"`
	TopP              float64  `json:"top_p,omitempty"`
	TopK              int      `json:"top_k,omitempty"`
	StopSequences     []string `json:"stop_sequences,omitempty"`
}

type Response struct {
	Completion string `json:"completion"`
}

type HelloHandler struct{}

type Query struct {
	Topic string `json:"topic"`
}

func HandleBedrockClaude2Chat(w http.ResponseWriter, r *http.Request) {

	const claudePromptFormat = "\n\nHuman: %s\n\nAssistant:"

	var query Query
	var message string

	// parse mesage from request
	error := json.NewDecoder(r.Body).Decode(&query)

	if error != nil {
		message = "how to learn japanese as quick as possible?"
		panic(error)
	}

	message = query.Topic

	fmt.Println(message)

	prompt := "" + fmt.Sprintf(claudePromptFormat, message)

	payload := Request{
		Prompt:            prompt,
		MaxTokensToSample: 2048,
	}

	payloadBytes, error := json.Marshal(payload)

	if error != nil {
		fmt.Fprintf(w, "ERROR")
		// return "", error
	}

	output, error := BedrockClient.InvokeModelWithResponseStream(
		context.Background(),
		&bedrockruntime.InvokeModelWithResponseStreamInput{
			Body:        payloadBytes,
			ModelId:     aws.String("anthropic.claude-v2"),
			ContentType: aws.String("application/json"),
		},
	)

	if error != nil {
		fmt.Fprintf(w, "ERROR")
		// return "", error
	}

	for event := range output.GetStream().Events() {
		switch v := event.(type) {
		case *types.ResponseStreamMemberChunk:

			//fmt.Println("payload", string(v.Value.Bytes))

			var resp Response
			err := json.NewDecoder(bytes.NewReader(v.Value.Bytes)).Decode(&resp)
			if err != nil {
				fmt.Fprintf(w, "ERROR")
				// return "", err
			}

			fmt.Println(resp.Completion)

			fmt.Fprintf(w, resp.Completion)
			if f, ok := w.(http.Flusher); ok {
				f.Flush()
			} else {
				fmt.Println("Damn, no flush")
			}

		case *types.UnknownUnionMember:
			fmt.Println("unknown tag:", v.Tag)

		default:
			fmt.Println("union is nil or unknown type")
		}
	}
}

func HandleBedrockClaude3HaikuChat(w http.ResponseWriter, r *http.Request) {

	var query Query
	var message string

	// parse mesage from request
	error := json.NewDecoder(r.Body).Decode(&query)

	if error != nil {
		message = "how to learn japanese as quick as possible?"
		panic(error)
	}

	message = query.Topic

	fmt.Println(message)

	payload := RequestBodyClaude3{
		MaxTokensToSample: 2048,
		AnthropicVersion:  "bedrock-2023-05-31",
		Temperature:       0.8,
		Messages: []Message{{
			Role: "user",
			Content: []Content{{
				Type: "text",
				Text: message,
			}},
		}},
	}

	payloadBytes, error := json.Marshal(payload)

	if error != nil {
		fmt.Fprintf(w, "ERROR")
		// return "", error
	}

	output, error := BedrockClient.InvokeModelWithResponseStream(
		context.Background(),
		&bedrockruntime.InvokeModelWithResponseStreamInput{
			Body:        payloadBytes,
			ModelId:     aws.String("anthropic.claude-3-haiku-20240307-v1:0"),
			ContentType: aws.String("application/json"),
			Accept:      aws.String("application/json"),
		},
	)

	if error != nil {
		fmt.Fprintf(w, "ERROR")
		// return "", error
	}

	for event := range output.GetStream().Events() {
		switch v := event.(type) {
		case *types.ResponseStreamMemberChunk:

			//fmt.Println("payload", string(v.Value.Bytes))

			var resp ResponseClaude3
			err := json.NewDecoder(bytes.NewReader(v.Value.Bytes)).Decode(&resp)
			if err != nil {
				fmt.Fprintf(w, "ERROR")
				// return "", err
			}

			fmt.Println(resp.Delta.Text)

			fmt.Fprintf(w, resp.Delta.Text)
			if f, ok := w.(http.Flusher); ok {
				f.Flush()
			} else {
				fmt.Println("Damn, no flush")
			}

		case *types.UnknownUnionMember:
			fmt.Println("unknown tag:", v.Tag)

		default:
			fmt.Println("union is nil or unknown type")
		}
	}
}

func TestHaiku() {

	fmt.Println("Hello")

	payload := RequestBodyClaude3{
		MaxTokensToSample: 2048,
		AnthropicVersion:  "bedrock-2023-05-31",
		Temperature:       0.8,
		Messages: []Message{{
			Role: "user",
			Content: []Content{{
				Type: "text",
				Text: "How to cook chicken soup?",
			}},
		}},
	}

	payloadBytes, error := json.Marshal(payload)

	if error != nil {
		fmt.Println(error)
	}

	output, error := BedrockClient.InvokeModelWithResponseStream(
		context.Background(),
		&bedrockruntime.InvokeModelWithResponseStreamInput{
			Body:        payloadBytes,
			ModelId:     aws.String("anthropic.claude-3-haiku-20240307-v1:0"),
			ContentType: aws.String("application/json"),
			Accept:      aws.String("application/json"),
		},
	)

	if error != nil {
		fmt.Println(error)
	}

	fmt.Println(output)

	for event := range output.GetStream().Events() {
		switch v := event.(type) {
		case *types.ResponseStreamMemberChunk:

			// fmt.Println("payload", string(v.Value.Bytes))

			// var resp map[string]interface{}
			var resp ResponseClaude3

			err := json.NewDecoder(bytes.NewReader(v.Value.Bytes)).Decode(&resp)
			if err != nil {
				fmt.Println(err)
			}

			fmt.Println(resp.Delta.Text)

		case *types.UnknownUnionMember:
			fmt.Println("unknown tag:", v.Tag)

		default:
			fmt.Println("union is nil or unknown type")
		}
	}

}

Chat Claude3#

struct to form request and parse response from Bedrock
parse and stream back to client

Let create struct for Bedrock request and response

// claude3 data type
type Content struct {
	Type string `json:"type"`
	Text string `json:"text"`
}

type Message struct {
	Role    string    `json:"role"`
	Content []Content `json:"content"`
}

type RequestBodyClaude3 struct {
	MaxTokensToSample int       `json:"max_tokens"`
	Temperature       float64   `json:"temperature,omitempty"`
	AnthropicVersion  string    `json:"anthropic_version"`
	Messages          []Message `json:"messages"`
}

Let create a function to process request for Bedrock and stream back client

func HandleBedrockClaude3HaikuChat(w http.ResponseWriter, r *http.Request) {

	var query Query
	var message string

	// parse mesage from request
	error := json.NewDecoder(r.Body).Decode(&query)

	if error != nil {
		message = "how to learn japanese as quick as possible?"
		panic(error)
	}

	message = query.Topic

	fmt.Println(message)

	payload := RequestBodyClaude3{
		MaxTokensToSample: 2048,
		AnthropicVersion:  "bedrock-2023-05-31",
		Temperature:       0.8,
		Messages: []Message{{
			Role: "user",
			Content: []Content{{
				Type: "text",
				Text: message,
			}},
		}},
	}

	payloadBytes, error := json.Marshal(payload)

	if error != nil {
		fmt.Fprintf(w, "ERROR")
		// return "", error
	}

	output, error := BedrockClient.InvokeModelWithResponseStream(
		context.Background(),
		&bedrockruntime.InvokeModelWithResponseStreamInput{
			Body:        payloadBytes,
			ModelId:     aws.String("anthropic.claude-3-haiku-20240307-v1:0"),
			ContentType: aws.String("application/json"),
			Accept:      aws.String("application/json"),
		},
	)

	if error != nil {
		fmt.Fprintf(w, "ERROR")
		// return "", error
	}

	for event := range output.GetStream().Events() {
		switch v := event.(type) {
		case *types.ResponseStreamMemberChunk:

			//fmt.Println("payload", string(v.Value.Bytes))

			var resp ResponseClaude3
			err := json.NewDecoder(bytes.NewReader(v.Value.Bytes)).Decode(&resp)
			if err != nil {
				fmt.Fprintf(w, "ERROR")
				// return "", err
			}

			fmt.Println(resp.Delta.Text)

			fmt.Fprintf(w, resp.Delta.Text)
			if f, ok := w.(http.Flusher); ok {
				f.Flush()
			} else {
				fmt.Println("Damn, no flush")
			}

		case *types.UnknownUnionMember:
			fmt.Println("unknown tag:", v.Tag)

		default:
			fmt.Println("union is nil or unknown type")
		}
	}
}

Here is full code

chat-claude3.go

package main

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/aws/aws-sdk-go-v2/aws"
	"github.com/aws/aws-sdk-go-v2/service/bedrockruntime"
	"github.com/aws/aws-sdk-go-v2/service/bedrockruntime/types"
)

type Content struct {
	Type string `json:"type"`
	Text string `json:"text"`
}

type Message struct {
	Role    string    `json:"role"`
	Content []Content `json:"content"`
}

type RequestBodyClaude3 struct {
	MaxTokensToSample int       `json:"max_tokens"`
	Temperature       float64   `json:"temperature,omitempty"`
	AnthropicVersion  string    `json:"anthropic_version"`
	Messages          []Message `json:"messages"`
}

type Delta struct {
	Type string `json:"type"`
	Text string `json:"text"`
}

type ResponseClaude3 struct {
	Type  string `json:"type"`
	Index int    `json:"index"`
	Delta Delta  `json:"delta"`
}

type Request struct {
	Prompt            string   `json:"prompt"`
	MaxTokensToSample int      `json:"max_tokens_to_sample"`
	Temperature       float64  `json:"temperature,omitempty"`
	TopP              float64  `json:"top_p,omitempty"`
	TopK              int      `json:"top_k,omitempty"`
	StopSequences     []string `json:"stop_sequences,omitempty"`
}

type Response struct {
	Completion string `json:"completion"`
}

type HelloHandler struct{}

type Query struct {
	Topic string `json:"topic"`
}

func HandleBedrockClaude2Chat(w http.ResponseWriter, r *http.Request) {

	const claudePromptFormat = "\n\nHuman: %s\n\nAssistant:"

	var query Query
	var message string

	// parse mesage from request
	error := json.NewDecoder(r.Body).Decode(&query)

	if error != nil {
		message = "how to learn japanese as quick as possible?"
		panic(error)
	}

	message = query.Topic

	fmt.Println(message)

	prompt := "" + fmt.Sprintf(claudePromptFormat, message)

	payload := Request{
		Prompt:            prompt,
		MaxTokensToSample: 2048,
	}

	payloadBytes, error := json.Marshal(payload)

	if error != nil {
		fmt.Fprintf(w, "ERROR")
		// return "", error
	}

	output, error := BedrockClient.InvokeModelWithResponseStream(
		context.Background(),
		&bedrockruntime.InvokeModelWithResponseStreamInput{
			Body:        payloadBytes,
			ModelId:     aws.String("anthropic.claude-v2"),
			ContentType: aws.String("application/json"),
		},
	)

	if error != nil {
		fmt.Fprintf(w, "ERROR")
		// return "", error
	}

	for event := range output.GetStream().Events() {
		switch v := event.(type) {
		case *types.ResponseStreamMemberChunk:

			//fmt.Println("payload", string(v.Value.Bytes))

			var resp Response
			err := json.NewDecoder(bytes.NewReader(v.Value.Bytes)).Decode(&resp)
			if err != nil {
				fmt.Fprintf(w, "ERROR")
				// return "", err
			}

			fmt.Println(resp.Completion)

			fmt.Fprintf(w, resp.Completion)
			if f, ok := w.(http.Flusher); ok {
				f.Flush()
			} else {
				fmt.Println("Damn, no flush")
			}

		case *types.UnknownUnionMember:
			fmt.Println("unknown tag:", v.Tag)

		default:
			fmt.Println("union is nil or unknown type")
		}
	}
}

func HandleBedrockClaude3HaikuChat(w http.ResponseWriter, r *http.Request) {

	var query Query
	var message string

	// parse mesage from request
	error := json.NewDecoder(r.Body).Decode(&query)

	if error != nil {
		message = "how to learn japanese as quick as possible?"
		panic(error)
	}

	message = query.Topic

	fmt.Println(message)

	payload := RequestBodyClaude3{
		MaxTokensToSample: 2048,
		AnthropicVersion:  "bedrock-2023-05-31",
		Temperature:       0.8,
		Messages: []Message{{
			Role: "user",
			Content: []Content{{
				Type: "text",
				Text: message,
			}},
		}},
	}

	payloadBytes, error := json.Marshal(payload)

	if error != nil {
		fmt.Fprintf(w, "ERROR")
		// return "", error
	}

	output, error := BedrockClient.InvokeModelWithResponseStream(
		context.Background(),
		&bedrockruntime.InvokeModelWithResponseStreamInput{
			Body:        payloadBytes,
			ModelId:     aws.String("anthropic.claude-3-haiku-20240307-v1:0"),
			ContentType: aws.String("application/json"),
			Accept:      aws.String("application/json"),
		},
	)

	if error != nil {
		fmt.Fprintf(w, "ERROR")
		// return "", error
	}

	for event := range output.GetStream().Events() {
		switch v := event.(type) {
		case *types.ResponseStreamMemberChunk:

			//fmt.Println("payload", string(v.Value.Bytes))

			var resp ResponseClaude3
			err := json.NewDecoder(bytes.NewReader(v.Value.Bytes)).Decode(&resp)
			if err != nil {
				fmt.Fprintf(w, "ERROR")
				// return "", err
			}

			fmt.Println(resp.Delta.Text)

			fmt.Fprintf(w, resp.Delta.Text)
			if f, ok := w.(http.Flusher); ok {
				f.Flush()
			} else {
				fmt.Println("Damn, no flush")
			}

		case *types.UnknownUnionMember:
			fmt.Println("unknown tag:", v.Tag)

		default:
			fmt.Println("union is nil or unknown type")
		}
	}
}

func TestHaiku() {

	fmt.Println("Hello")

	payload := RequestBodyClaude3{
		MaxTokensToSample: 2048,
		AnthropicVersion:  "bedrock-2023-05-31",
		Temperature:       0.8,
		Messages: []Message{{
			Role: "user",
			Content: []Content{{
				Type: "text",
				Text: "How to cook chicken soup?",
			}},
		}},
	}

	payloadBytes, error := json.Marshal(payload)

	if error != nil {
		fmt.Println(error)
	}

	output, error := BedrockClient.InvokeModelWithResponseStream(
		context.Background(),
		&bedrockruntime.InvokeModelWithResponseStreamInput{
			Body:        payloadBytes,
			ModelId:     aws.String("anthropic.claude-3-haiku-20240307-v1:0"),
			ContentType: aws.String("application/json"),
			Accept:      aws.String("application/json"),
		},
	)

	if error != nil {
		fmt.Println(error)
	}

	fmt.Println(output)

	for event := range output.GetStream().Events() {
		switch v := event.(type) {
		case *types.ResponseStreamMemberChunk:

			// fmt.Println("payload", string(v.Value.Bytes))

			// var resp map[string]interface{}
			var resp ResponseClaude3

			err := json.NewDecoder(bytes.NewReader(v.Value.Bytes)).Decode(&resp)
			if err != nil {
				fmt.Println(err)
			}

			fmt.Println(resp.Delta.Text)

		case *types.UnknownUnionMember:
			fmt.Println("unknown tag:", v.Tag)

		default:
			fmt.Println("union is nil or unknown type")
		}
	}

}

OpenSearch#

Query all items
Query given a vector
Get embedding vec from Bedrock Titan

Let query all item

type EmbedResponse struct {
	Embedding []float64 `json:"embedding"`
}

type Hits struct {
	Hits []map[string]interface{} `json:"hits"`
}

type AossResponse struct {
	Hits Hits `json:"hits"`
}

func QueryAOSS(vec []float64) ([]string, error) {

	// let query get all item in an index

	content := strings.NewReader(`{
	    "size": 10,
	    "query": {
	        "match_all": {}
	        }
	}`)

	vecStr := make([]string, len(vec))

	// convert array float to string
	for k, v := range vec {

		if k < len(vec)-1 {
			vecStr[k] = fmt.Sprint(v) + ","
		} else {
			vecStr[k] = fmt.Sprint(v)
		}

	}

	// create request body to titan model
	// content := strings.NewReader(fmt.Sprintf(`{
	// 	"size": 5,
	// 	"query": {
	// 		"knn": {
	// 			"vector_field": {
	// 				"vector": %s,
	// 				"k": 5
	// 			}
	// 		}
	// 	}
	// }`, vecStr))

	// fmt.Println(content)

	search := opensearchapi.SearchRequest{
		Index: []string{"demo"},
		Body:  content,
	}

	searchResponse, err := search.Do(context.Background(), AOSSClient)

	if err != nil {
		log.Fatal(err)
	}

	// fmt.Println(searchResponse)

	var answer AossResponse

	json.NewDecoder(searchResponse.Body).Decode(&answer)

	// first := answer.Hits.Hits[0]

	// fmt.Printf("id: %s\n, index: %s\n, text: %s", first["_id"], first["_index"], first["_source"].(map[string]interface{})["text"])

	// fmt.Println(answer.Hits.Hits[0]["_id"])

	queryResult := answer.Hits.Hits[0]["_source"].(map[string]interface{})["text"]

	if queryResult == nil {
		return []string{"nil"}, nil
	}

	// extract hint text only
	hits := []string{}

	for k, v := range answer.Hits.Hits {

		if k > 0 {
			hits = append(hits, v["_source"].(map[string]interface{})["text"].(string))
		}

	}

	return hits, nil

	// return fmt.Sprint(queryResult), nil

}

Let query given a vector

func GetEmbedVector(string) ([]float64, error) {

	// create request body to titan model
	body := map[string]interface{}{
		"inputText": "what is amazon bedrock?",
	}
	bodyJson, err := json.Marshal(body)

	if err != nil {
		fmt.Println(err)
		return nil, err
	}

	// invoke bedrock titan model to convert string to embedding vector
	response, error := BedrockClient.InvokeModel(
		context.Background(),
		&bedrockruntime.InvokeModelInput{
			Body:        []byte(bodyJson),
			ModelId:     aws.String("amazon.titan-embed-text-v1"),
			ContentType: aws.String("application/json"),
		},
	)

	if error != nil {
		fmt.Println(error)
		return nil, error
	}

	// assert response to map
	var embedResponse map[string]interface{}

	error = json.Unmarshal(response.Body, &embedResponse)

	if error != nil {
		fmt.Println(error)
		return nil, error
	}

	// assert response to array
	slice, ok := embedResponse["embedding"].([]interface{})

	if !ok {
		fmt.Println(ok)
	}

	// assert to array of float64
	values := make([]float64, len(slice))

	for k, v := range slice {
		values[k] = float64(v.(float64))
	}

	return values, nil
}

func TestQueryAOSS() {

	fmt.Println("Hello")

	v, error := GetEmbedVector("hello")

	if error != nil {
		fmt.Println(error)
	}

	awnsers, error := QueryAOSS(v)

	if error != nil {
		fmt.Println(error)
	}

	// fmt.Println(awnsers)

	for k, v := range awnsers {
		fmt.Println(k, v)
		fmt.Println("====================================")
	}

}

Here is full code

chat-opensearch.go

package main

import (
	"context"
	"encoding/json"
	"fmt"
	"log"
	"strings"

	"github.com/aws/aws-sdk-go-v2/aws"
	"github.com/aws/aws-sdk-go-v2/service/bedrockruntime"
	"github.com/opensearch-project/opensearch-go/v2/opensearchapi"
)

type EmbedResponse struct {
	Embedding []float64 `json:"embedding"`
}

type Hits struct {
	Hits []map[string]interface{} `json:"hits"`
}

type AossResponse struct {
	Hits Hits `json:"hits"`
}

func QueryAOSS(vec []float64) ([]string, error) {

	// let query get all item in an index

	content := strings.NewReader(`{
	    "size": 10,
	    "query": {
	        "match_all": {}
	        }
	}`)

	vecStr := make([]string, len(vec))

	// convert array float to string
	for k, v := range vec {

		if k < len(vec)-1 {
			vecStr[k] = fmt.Sprint(v) + ","
		} else {
			vecStr[k] = fmt.Sprint(v)
		}

	}

	// create request body to titan model
	// content := strings.NewReader(fmt.Sprintf(`{
	// 	"size": 5,
	// 	"query": {
	// 		"knn": {
	// 			"vector_field": {
	// 				"vector": %s,
	// 				"k": 5
	// 			}
	// 		}
	// 	}
	// }`, vecStr))

	// fmt.Println(content)

	search := opensearchapi.SearchRequest{
		Index: []string{"demo"},
		Body:  content,
	}

	searchResponse, err := search.Do(context.Background(), AOSSClient)

	if err != nil {
		log.Fatal(err)
	}

	// fmt.Println(searchResponse)

	var answer AossResponse

	json.NewDecoder(searchResponse.Body).Decode(&answer)

	// first := answer.Hits.Hits[0]

	// fmt.Printf("id: %s\n, index: %s\n, text: %s", first["_id"], first["_index"], first["_source"].(map[string]interface{})["text"])

	// fmt.Println(answer.Hits.Hits[0]["_id"])

	queryResult := answer.Hits.Hits[0]["_source"].(map[string]interface{})["text"]

	if queryResult == nil {
		return []string{"nil"}, nil
	}

	// extract hint text only
	hits := []string{}

	for k, v := range answer.Hits.Hits {

		if k > 0 {
			hits = append(hits, v["_source"].(map[string]interface{})["text"].(string))
		}

	}

	return hits, nil

	// return fmt.Sprint(queryResult), nil

}

func GetEmbedVector(string) ([]float64, error) {

	// create request body to titan model
	body := map[string]interface{}{
		"inputText": "what is amazon bedrock?",
	}
	bodyJson, err := json.Marshal(body)

	if err != nil {
		fmt.Println(err)
		return nil, err
	}

	// invoke bedrock titan model to convert string to embedding vector
	response, error := BedrockClient.InvokeModel(
		context.Background(),
		&bedrockruntime.InvokeModelInput{
			Body:        []byte(bodyJson),
			ModelId:     aws.String("amazon.titan-embed-text-v1"),
			ContentType: aws.String("application/json"),
		},
	)

	if error != nil {
		fmt.Println(error)
		return nil, error
	}

	// assert response to map
	var embedResponse map[string]interface{}

	error = json.Unmarshal(response.Body, &embedResponse)

	if error != nil {
		fmt.Println(error)
		return nil, error
	}

	// assert response to array
	slice, ok := embedResponse["embedding"].([]interface{})

	if !ok {
		fmt.Println(ok)
	}

	// assert to array of float64
	values := make([]float64, len(slice))

	for k, v := range slice {
		values[k] = float64(v.(float64))
	}

	return values, nil
}

func TestQueryAOSS() {

	fmt.Println("Hello")

	v, error := GetEmbedVector("hello")

	if error != nil {
		fmt.Println(error)
	}

	awnsers, error := QueryAOSS(v)

	if error != nil {
		fmt.Println(error)
	}

	// fmt.Println(awnsers)

	for k, v := range awnsers {
		fmt.Println(k, v)
		fmt.Println("====================================")
	}

}

HTTP Server#

create http server and multiplixer
frontend and javascript

http-server.go

package main

import (
	"context"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"time"

	"github.com/aws/aws-sdk-go-v2/config"
	"github.com/aws/aws-sdk-go-v2/service/bedrockruntime"
	opensearch "github.com/opensearch-project/opensearch-go/v2"
	requestsigner "github.com/opensearch-project/opensearch-go/v2/signer/awsv2"
)

// opensearch severless client
var AOSSClient *opensearch.Client

// bedrock client
var BedrockClient *bedrockruntime.Client

// create an init function to initializing opensearch client
func init() {

	//
	fmt.Println("init and create an opensearch client")

	// load aws credentials from profile demo using config
	awsCfg, err := config.LoadDefaultConfig(context.Background(),
		config.WithRegion("us-east-1"),
		config.WithSharedConfigProfile("demo"),
	)

	if err != nil {
		log.Fatal(err)
	}

	// create a aws request signer using requestsigner
	signer, err := requestsigner.NewSignerWithService(awsCfg, "aoss")

	if err != nil {
		log.Fatal(err)
	}

	// create an opensearch client using opensearch package
	AOSSClient, err = opensearch.NewClient(opensearch.Config{
		Addresses: []string{AOSS_ENDPOINT},
		Signer:    signer,
	})

	if err != nil {
		log.Fatal(err)
	}

	// create bedorck runtime client
	BedrockClient = bedrockruntime.NewFromConfig(awsCfg)

}

func main() {

	// create handler multiplexer
	mux := http.NewServeMux()

	// hello message
	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte("Hello"))
	})

	// response simple json
	mux.HandleFunc("/json", func(w http.ResponseWriter, r *http.Request) {
		json.NewEncoder(w).Encode(struct {
			Message string `json:"Message"`
		}{Message: "Hello"})
	})

	// handle a simple post request from a form
	mux.HandleFunc("/form", func(w http.ResponseWriter, r *http.Request) {
		if r.Method == "POST" {
			r.ParseForm()
			name := r.FormValue("name")
			w.Write([]byte(fmt.Sprintf("Hello %s", name)))
		}

		if r.Method == "GET" {
			// w.Write([]byte("Hello"))
			http.ServeFile(w, r, "./static/index.html")
		}
	})

	// query opensarch
	mux.HandleFunc("/query", func(w http.ResponseWriter, r *http.Request) {

		// parse question from user
		question := r.FormValue("query")

		// convert question to embedding vector
		vec, error := GetEmbedVector(question)

		if error != nil {
			fmt.Println(error)
		}

		// do query opensearch
		answers, err := QueryAOSS(vec)

		if err != nil {
			fmt.Println(err)
		}

		json.NewEncoder(w).Encode(struct {
			Messages []string `json:"Messages"`
		}{Messages: answers})
	})

	// handle chat with bedrock
	mux.HandleFunc("/bedrock-stream", HandleBedrockClaude2Chat)


	// bedrock chat frontend
	mux.HandleFunc("/chat", func(w http.ResponseWriter, r *http.Request) {
		http.ServeFile(w, r, "./static/chat.html")
	})

		// handle chat with bedrock
	mux.HandleFunc("/bedrock-haiku", HandleBedrockClaude3HaikuChat)


	// bedrock chat frontend
	mux.HandleFunc("/haiku", func(w http.ResponseWriter, r *http.Request) {
		http.ServeFile(w, r, "./static/haiku.html")
	})

	// create a http server using http
	server := http.Server{
		Addr:           ":3000",
		Handler:        mux,
		ReadTimeout:    10 * time.Second,
		WriteTimeout:   10 * time.Second,
		MaxHeaderBytes: 1 << 20,
	}

	server.ListenAndServe()

}

FrontEnd for chat page

chat.html

<html>
  <head>
    <style>
      :root {
        box-sizing: border-box;
      }
      *,
      ::before,
      ::after {
        box-sizing: inherit;
      }

      .container {
        width: 80%;
        max-width: 600px;
        margin: auto;
        /* background-color: antiquewhite; */
      }

      .question-form {
        position: relative;
      }

      button {
        cursor: pointer;
        background-color: orange;
        padding: 1em;
        padding-left: 1.5em;
        padding-right: 1.5em;
        position: absolute;
        top: 50%;
        transform: translateY(-50%);
        right: 1em;
        opacity: 80;
      }

      .text-area {
        background-color: azure;
        margin-top: 1em;
      }

      .text-input {
        background-color: aquamarine;
        width: 100%;
        padding: 1em;
        font-size: large;
      }
    </style>
  </head>
  <body>
    <div class="container">
      <div class="question-form">
        <form id="form" onkeydown="return event.key != 'Enter';">
          <input class="text-input" type="text" id="text-input" />
        </form>
        <button id="submit">Submit</button>
      </div>
      <div class="text-area">
        <p id="story-output"></p>
      </div>
    </div>
    <script>
      const storyOutput = document.getElementById('story-output')

      const callBedrockStream = async () => {
        storyOutput.innerText = ''

        const topic = document.getElementById('text-input').value

        if (topic) {
          try {
            const response = await fetch('/bedrock-haiku', {
              method: 'POST',
              headers: {
                'Content-Type': 'application/json'
              },
              body: JSON.stringify({ topic: topic })
            })

            console.log(response)
            const reader = response.body.getReader()
            const decoder = new TextDecoder()

            while (true) {
              const { done, value } = await reader.read()
              if (done) {
                break
              }
              const text = decoder.decode(value)
              console.log(text)
              storyOutput.innerText += text
            }
          } catch (error) {
            console.log(error)
          }
        } else {
          console.log('Please enter question ...')
        }
      }

      document
        .getElementById('submit')
        .addEventListener('click', callBedrockStream)

      document
        .getElementById('text-input')
        .addEventListener('keydown', async event => {
          if (event.code === 'Enter') {
            await callBedrockStream()
          }
        })
    </script>
  </body>
</html>

FrontEnd for OpenSearch page

opensearch.html

<html>
  <head>
    <title>Query OpenSearch</title>
    <style>
      body {
        background-color: antiquewhite;
      }

      .container {
        width: 80%;
        max-width: 600px;
        margin: auto;
      }

      .button-submit {
        background-color: orange;
        padding: 5px 20px;
        border-radius: 5px;
        border: none;
        cursor: pointer;
        position: absolute;
        transform: translateY(-50%);
        top: 50%;
        right: 10px;
      }

      .input-query {
        padding: 10px 15px;
        width: 100%;
      }

      .container-input {
        position: relative;
      }

      .form {
        margin-top: 20px;
      }
    </style>
  </head>
  <body>
    <div class="container">
      <form id="form" class="form">
        <div class="container-input">
          <input type="text" id="query" name="query" class="input-query" />
          <button class="button-submit">Submit</button>
        </div>
      </form>
      <div id="list"></div>
    </div>
  </body>

  <script>
    console.log('hello')

    const query = async () => {
      // call post request
      const response = await fetch('/query')

      const json = await response.json()

      console.log(json)

      // update ui
      // document.getElementById("content").innerHTML = json.Messages;

      // Create an array of items
      var items = json.Messages

      // Get the list container element
      var listContainer = document.getElementById('list')

      // Loop through the items array and create list items (<li>)
      for (var i = 0; i < items.length; i++) {
        var listItem = document.createElement('div')
        listItem.style.marginBottom = '15px'
        listItem.style.borderBottom = '1px solid #0000FF'

        var header = document.createElement('h4')
        header.textContent = `Chunk ${i}`

        var itemText = document.createTextNode(items[i])
        listItem.appendChild(header)
        listItem.appendChild(itemText)
        listContainer.appendChild(listItem)
      }
    }

    document.getElementById('form').addEventListener('submit', async event => {
      event.preventDefault()
      console.log('submit form')

      await query()
    })
  </script>
</html>

Deploy#

CDK stack to create an AppRunder service
Dockerfile
Build script

Here is the AppRunder stack

AppRunnerStack

import { Stack, StackProps, aws_apprunner, aws_iam } from 'aws-cdk-lib'
import { Effect } from 'aws-cdk-lib/aws-iam'
import { Construct } from 'constructs'

interface AppRunnerProps extends StackProps {
  ecr: string
  bucket: string
  aossCollectionArn: string
}

export class AppRunnerStack extends Stack {
  constructor(scope: Construct, id: string, props: AppRunnerProps) {
    super(scope, id, props)

    const buildRole = new aws_iam.Role(
      this,
      'RoleForAppRunnerToPullGoBedrockAppImage',
      {
        assumedBy: new aws_iam.ServicePrincipal(
          'build.apprunner.amazonaws.com'
        ),
        roleName: 'RoleForAppRunnerToPullGoBedrockAppImage'
      }
    )

    buildRole.addToPolicy(
      new aws_iam.PolicyStatement({
        effect: Effect.ALLOW,
        resources: ['*'],
        actions: ['ecr:*']
      })
    )

    const instanceRole = new aws_iam.Role(this, 'InstanceRoleForGoBedrockApp', {
      assumedBy: new aws_iam.ServicePrincipal('tasks.apprunner.amazonaws.com'),
      roleName: 'InstanceRoleForApprunnerBedrock'
    })

    instanceRole.addToPolicy(
      new aws_iam.PolicyStatement({
        effect: Effect.ALLOW,
        resources: [
          'arn:aws:bedrock:us-east-1::foundation-model/anthropic.claude-v2',
          'arn:aws:bedrock:us-east-1::foundation-model/stability.stable-diffusion-xl-v1',
          'arn:aws:bedrock:us-east-1::foundation-model/amazon.titan-embed-text-v1'
        ],
        actions: [
          'bedrock:InvokeModel',
          'bedrock:InvokeModelWithResponseStream'
        ]
      })
    )

    instanceRole.addToPolicy(
      new aws_iam.PolicyStatement({
        effect: Effect.ALLOW,
        resources: [`arn:aws:s3:::${props.bucket}/*`],
        actions: ['s3:PutObject', 's3:GetObject']
      })
    )

    instanceRole.addToPolicy(
      new aws_iam.PolicyStatement({
        effect: Effect.ALLOW,
        resources: [props.aossCollectionArn],
        actions: ['aoss:APIAccessAll']
      })
    )

    const autoscaling = new aws_apprunner.CfnAutoScalingConfiguration(
      this,
      'AutoScalingForGoBedrockApp',
      {
        autoScalingConfigurationName: 'AutoScalingForGoBedrockApp',
        // min number instance
        minSize: 1,
        // max number instance
        maxSize: 10,
        // max concurrent request per instance
        maxConcurrency: 100
      }
    )

    const apprunner = new aws_apprunner.CfnService(
      this,
      'GoBedrockAppRunnerService',
      {
        serviceName: 'GoBedrockAppRunnerService',
        sourceConfiguration: {
          authenticationConfiguration: {
            accessRoleArn: buildRole.roleArn
          },

          autoDeploymentsEnabled: false,

          imageRepository: {
            imageIdentifier: props.ecr,
            imageRepositoryType: 'ECR',
            imageConfiguration: {
              port: '3000',
              runtimeEnvironmentVariables: [
                {
                  name: 'BUCKET',
                  value: 'demo'
                },
                {
                  name: 'HOSTNAME',
                  value: '0.0.0.0'
                },
                {
                  name: 'PORT',
                  value: '3000'
                }
              ]
              // startCommand: "",
            }
          }
        },

        instanceConfiguration: {
          cpu: '1 vCPU',
          memory: '2 GB',
          instanceRoleArn: instanceRole.roleArn
        },

        observabilityConfiguration: {
          observabilityEnabled: false
        },

        autoScalingConfigurationArn: autoscaling.ref
      }
    )

    apprunner.addDependency(autoscaling)
  }
}

Here is Dockerfile

# syntax=docker/dockerfile:1

# Build the application from source
FROM golang:1.21.5 AS build-stage

WORKDIR /app

COPY go.mod go.sum ./
RUN go mod download

COPY *.go ./

RUN CGO_ENABLED=0 GOOS=linux go build -o /entest

# Run the tests in the container
FROM build-stage AS run-test-stage

# Deploy the application binary into a lean image
FROM gcr.io/distroless/base-debian11 AS build-release-stage

WORKDIR /

COPY --from=build-stage /entest /entest
COPY *.html ./
COPY static ./static

EXPOSE 3000

USER nonroot:nonroot

ENTRYPOINT ["/entest"]

And build script

import os

# parameters
REGION = "us-east-1"
ACCOUNT = os.environ["ACCOUNT_ID"]

# delete all docker images
os.system("sudo docker system prune -a")

# build go-bedrock-app image
os.system("sudo docker build -t go-bedrock-app . ")

#  aws ecr login
os.system(f"aws ecr get-login-password --region {REGION} | sudo docker login --username AWS --password-stdin {ACCOUNT}.dkr.ecr.{REGION}.amazonaws.com")

# get image id
IMAGE_ID=os.popen("sudo docker images -q go-bedrock-app:latest").read()

# tag go-bedrock-app image
os.system(f"sudo docker tag {IMAGE_ID.strip()} {ACCOUNT}.dkr.ecr.{REGION}.amazonaws.com/go-bedrock-app:latest")

# create ecr repository
os.system(f"aws ecr create-repository --registry-id {ACCOUNT} --repository-name go-bedrock-app")

# push image to ecr
os.system(f"sudo docker push {ACCOUNT}.dkr.ecr.{REGION}.amazonaws.com/go-bedrock-app:latest")

# run locally to test
# os.system(f"sudo docker run -d -p 3001:3000 go-bedrock-app:latest")

Basic Prompt#

Let do a simple multiple conversation turns which mean client web send a list of messaages to bedrock like [user,bot,user, ...].

update client code to store messages
update server code to parese the messages and send request to bedrock

In frontend, we need a buffer to store chat history

// conversation turns
let messages = []

// get html component for model answer
const modelAnswer = document.getElementById('model-answer')

const callBedrockStream = async () => {
  // present model answer to frontend
  modelAnswer.innerText = ''

  // get user question
  const userQuestion = document.getElementById('text-input').value

  // push user question to messages
  messages.push({
    role: 'user',
    content: [{ type: 'text', text: userQuestion }]
  })

  if (userQuestion) {
    try {
      const response = await fetch('/bedrock-haiku', {
        method: 'POST',
        headers: {
          'Content-Type': 'application/json'
        },
        body: JSON.stringify({ messages: messages })
      })

      console.log(response)
      const reader = response.body.getReader()
      const decoder = new TextDecoder()

      while (true) {
        const { done, value } = await reader.read()
        if (done) {
          break
        }
        const text = decoder.decode(value)
        console.log(text)
        modelAnswer.innerText += text
      }

      // push model answer to converstion turn
      messages.push({
        role: 'assistant',
        content: [{ type: 'text', text: modelAnswer.innerText }]
      })
    } catch (error) {
      console.log(error)
    }
  } else {
    console.log('Please enter question ...')
  }
}

document.getElementById('submit').addEventListener('click', async event => {
  event.preventDefault()
  await callBedrockStream()
})

document
  .getElementById('text-input')
  .addEventListener('keydown', async event => {
    if (event.code === 'Enter') {
      await callBedrockStream()
    }
  })

Here is frondend code

haiku.html

<html>
  <head>
    <meta name="viewport" content="width=device-width" />
    <style>
      :root {
        box-sizing: border-box;
      }
      *,
      ::before,
      ::after {
        box-sizing: inherit;
      }

      body {
        /* background-color: antiquewhite; */
      }

      .container {
        width: 100%;
        max-width: 500px;
        margin: auto;
        /* background-color: antiquewhite; */
      }

      .button {
        background-color: #43a047;
        padding: 8px 20px;
        border-radius: 5px;
        border: none;
        cursor: pointer;
        position: absolute;
        transform: translateY(-50%);
        top: 50%;
        right: 10px;
        opacity: 0.8;
      }

      .button:hover {
        background-color: orange;
      }

      .text-input {
        padding: 10px 15px;
        width: 100%;
        outline: none;
        border: solid black 1px;
        background-color: #e0e0e0;
        box-shadow: 0 10px 15px -3px #e0e0e0;

        font-family: system-ui, -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto,
          Oxygen, Ubuntu, Cantarell, 'Open Sans', 'Helvetica Neue', sans-serif;
        font-size: medium;
        font-weight: 400;
        letter-spacing: normal;
        line-height: 25px;
      }

      .text-input:focus {
        border: solid #4caf50 1.5px;
        outline: none;
      }

      .container-input {
        position: relative;
      }

      .form {
        margin-top: 20px;
      }

      .text-model {
        /* color: #4caf50; */
        font-family: system-ui, -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto,
          Oxygen, Ubuntu, Cantarell, 'Open Sans', 'Helvetica Neue', sans-serif;
        font-size: medium;
        font-weight: 400;
        letter-spacing: normal;
        line-height: 25px;
      }
    </style>
  </head>
  <body>
    <div class="container">
      <form id="form" onkeydown="return event.key != 'Enter';" class="form">
        <div class="container-input">
          <input class="text-input" type="text" id="text-input" />
          <button id="submit" class="button">Ask</button>
        </div>
      </form>
      <div>
        <p id="model-answer" class="text-model"></p>
      </div>
    </div>
    <script>
      // conversation turns
      let messages = []

      // get html component for model answer
      const modelAnswer = document.getElementById('model-answer')

      const callBedrockStream = async () => {
        // present model answer to frontend
        modelAnswer.innerText = ''

        // get user question
        const userQuestion = document.getElementById('text-input').value

        // push user question to messages
        messages.push({
          role: 'user',
          content: [{ type: 'text', text: userQuestion }]
        })

        if (userQuestion) {
          try {
            const response = await fetch('/bedrock-haiku', {
              method: 'POST',
              headers: {
                'Content-Type': 'application/json'
              },
              body: JSON.stringify({ messages: messages })
            })

            console.log(response)
            const reader = response.body.getReader()
            const decoder = new TextDecoder()

            while (true) {
              const { done, value } = await reader.read()
              if (done) {
                break
              }
              const text = decoder.decode(value)
              console.log(text)
              modelAnswer.innerText += text
            }

            // push model answer to converstion turn
            messages.push({
              role: 'assistant',
              content: [{ type: 'text', text: modelAnswer.innerText }]
            })
          } catch (error) {
            console.log(error)
          }
        } else {
          console.log('Please enter question ...')
        }
      }

      document
        .getElementById('submit')
        .addEventListener('click', async event => {
          event.preventDefault()
          await callBedrockStream()
        })

      document
        .getElementById('text-input')
        .addEventListener('keydown', async event => {
          if (event.code === 'Enter') {
            await callBedrockStream()
          }
        })
    </script>
  </body>
</html>

And here is server code. First we need some structs to parse client request and form correct prompt for bedrock

// claude3 request data type
type Content struct {
	Type string `json:"type"`
	Text string `json:"text"`
}

type Message struct {
	Role    string    `json:"role"`
	Content []Content `json:"content"`
}

type RequestBodyClaude3 struct {
	MaxTokensToSample int       `json:"max_tokens"`
	Temperature       float64   `json:"temperature,omitempty"`
	AnthropicVersion  string    `json:"anthropic_version"`
	Messages          []Message `json:"messages"`
}

// frontend request data type
type FrontEndRequest struct {
	Messages []Message `json:"messages"`
}

// claude3 response data type
type Delta struct {
	Type string `json:"type"`
	Text string `json:"text"`
}

type ResponseClaude3 struct {
	Type  string `json:"type"`
	Index int    `json:"index"`
	Delta Delta  `json:"delta"`
}

Then here is the full code

bedrock.go

package main

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/aws/aws-sdk-go-v2/aws"
	"github.com/aws/aws-sdk-go-v2/service/bedrockruntime"
	"github.com/aws/aws-sdk-go-v2/service/bedrockruntime/types"
)

// claude3 request data type
type Content struct {
	Type string `json:"type"`
	Text string `json:"text"`
}

type Message struct {
	Role    string    `json:"role"`
	Content []Content `json:"content"`
}

type RequestBodyClaude3 struct {
	MaxTokensToSample int       `json:"max_tokens"`
	Temperature       float64   `json:"temperature,omitempty"`
	AnthropicVersion  string    `json:"anthropic_version"`
	Messages          []Message `json:"messages"`
}

// frontend request data type
type FrontEndRequest struct {
	Messages []Message `json:"messages"`
}

// claude3 response data type
type Delta struct {
	Type string `json:"type"`
	Text string `json:"text"`
}

type ResponseClaude3 struct {
	Type  string `json:"type"`
	Index int    `json:"index"`
	Delta Delta  `json:"delta"`
}

// claude2 data type
type Request struct {
	Prompt            string   `json:"prompt"`
	MaxTokensToSample int      `json:"max_tokens_to_sample"`
	Temperature       float64  `json:"temperature,omitempty"`
	TopP              float64  `json:"top_p,omitempty"`
	TopK              int      `json:"top_k,omitempty"`
	StopSequences     []string `json:"stop_sequences,omitempty"`
}

type Response struct {
	Completion string `json:"completion"`
}

type HelloHandler struct{}

type Query struct {
	Topic string `json:"topic"`
}

func HandleBedrockClaude2Chat(w http.ResponseWriter, r *http.Request) {

	const claudePromptFormat = "\n\nHuman: %s\n\nAssistant:"

	var query Query
	var message string

	// parse mesage from request
	error := json.NewDecoder(r.Body).Decode(&query)

	if error != nil {
		message = "how to learn japanese as quick as possible?"
		panic(error)
	}

	message = query.Topic

	fmt.Println(message)

	prompt := "" + fmt.Sprintf(claudePromptFormat, message)

	payload := Request{
		Prompt:            prompt,
		MaxTokensToSample: 2048,
	}

	payloadBytes, error := json.Marshal(payload)

	if error != nil {
		fmt.Fprintf(w, "ERROR")
		// return "", error
	}

	output, error := BedrockClient.InvokeModelWithResponseStream(
		context.Background(),
		&bedrockruntime.InvokeModelWithResponseStreamInput{
			Body:        payloadBytes,
			ModelId:     aws.String("anthropic.claude-v2"),
			ContentType: aws.String("application/json"),
		},
	)

	if error != nil {
		fmt.Fprintf(w, "ERROR")
		// return "", error
	}

	for event := range output.GetStream().Events() {
		switch v := event.(type) {
		case *types.ResponseStreamMemberChunk:

			//fmt.Println("payload", string(v.Value.Bytes))

			var resp Response
			err := json.NewDecoder(bytes.NewReader(v.Value.Bytes)).Decode(&resp)
			if err != nil {
				fmt.Fprintf(w, "ERROR")
				// return "", err
			}

			fmt.Println(resp.Completion)

			fmt.Fprintf(w, resp.Completion)
			if f, ok := w.(http.Flusher); ok {
				f.Flush()
			} else {
				fmt.Println("Damn, no flush")
			}

		case *types.UnknownUnionMember:
			fmt.Println("unknown tag:", v.Tag)

		default:
			fmt.Println("union is nil or unknown type")
		}
	}
}

func HandleBedrockClaude3HaikuChat(w http.ResponseWriter, r *http.Request) {

	// list of messages sent from frontend client
	var request FrontEndRequest

	// parse mesage from request
	error := json.NewDecoder(r.Body).Decode(&request)

	if error != nil {
		panic(error)
	}

	messages := request.Messages

	fmt.Println(messages)

	payload := RequestBodyClaude3{
		MaxTokensToSample: 2048,
		AnthropicVersion:  "bedrock-2023-05-31",
		Temperature:       0.9,
		Messages:          messages,
	}

	payloadBytes, error := json.Marshal(payload)

	if error != nil {
		fmt.Fprintf(w, "ERROR")
		// return "", error
	}

	output, error := BedrockClient.InvokeModelWithResponseStream(
		context.Background(),
		&bedrockruntime.InvokeModelWithResponseStreamInput{
			Body:        payloadBytes,
			ModelId:     aws.String("anthropic.claude-3-haiku-20240307-v1:0"),
			ContentType: aws.String("application/json"),
			Accept:      aws.String("application/json"),
		},
	)

	if error != nil {
		fmt.Fprintf(w, "ERROR")
		// return "", error
	}

	for event := range output.GetStream().Events() {
		switch v := event.(type) {
		case *types.ResponseStreamMemberChunk:

			//fmt.Println("payload", string(v.Value.Bytes))

			var resp ResponseClaude3
			err := json.NewDecoder(bytes.NewReader(v.Value.Bytes)).Decode(&resp)
			if err != nil {
				fmt.Fprintf(w, "ERROR")
				// return "", err
			}

			fmt.Println(resp.Delta.Text)

			fmt.Fprintf(w, resp.Delta.Text)
			if f, ok := w.(http.Flusher); ok {
				f.Flush()
			} else {
				fmt.Println("Damn, no flush")
			}

		case *types.UnknownUnionMember:
			fmt.Println("unknown tag:", v.Tag)

		default:
			fmt.Println("union is nil or unknown type")
		}
	}
}

func TestHaiku() {

	fmt.Println("Hello")

	payload := RequestBodyClaude3{
		MaxTokensToSample: 2048,
		AnthropicVersion:  "bedrock-2023-05-31",
		Temperature:       0.8,
		Messages: []Message{{
			Role: "user",
			Content: []Content{{
				Type: "text",
				Text: "How to cook chicken soup?",
			}},
		}, {
			Role: "assistant",
			Content: []Content{{
				Type: "text",
				Text: `Here is a basic recipe for cooking chicken soup:

				Ingredients:
				- 1 whole chicken or 3-4 lbs of chicken parts (breasts, thighs, drumsticks)
				- 8 cups of water or chicken broth
				- 2 large carrots, peeled and sliced
				- 2 celery stalks, sliced
				- 1 onion, diced
				- 2 cloves garlic, minced
				- 1 bay leaf
				- 1 tsp dried thyme
				- Salt and pepper to taste
				- Egg noodles or other pasta (optional)

				Instructions:

				1. Place the whole chicken or chicken parts in a large pot and cover with the water or broth. Bring to a boil over high heat.

				2. Once boiling, reduce heat to medium-low, cover and simmer for 45-60 minutes, until the chicken is cooked through.

				3. Remove the chicken from the pot and set aside. Skim any foam or fat that rises to the surface.

				4. Add the carrots, celery, onion, garlic, bay leaf and thyme to the pot. Simmer for 15-20 minutes until the vegetables are tender.

				5. Meanwhile, remove the meat from the chicken bones and shred or chop it into bite-size pieces.

				6. Add the cooked chicken back to the pot and season with salt and pepper to taste.

				7. If using, add the egg noodles or other pasta and cook for the time specified on the package.

				8. Serve the chicken soup hot. You can garnish with fresh parsley or dill if desired.

				The longer you simmer the soup, the more flavorful it will become. You can also add other vegetables like potatoes, peas, or corn. Adjust seasoning as needed.`,
			}},
		},
			{
				Role: "user",
				Content: []Content{{
					Type: "text",
					Text: "How to customize it for 3 years old girl?",
				}},
			},
		},
	}

	payloadBytes, error := json.Marshal(payload)

	if error != nil {
		fmt.Println(error)
	}

	output, error := BedrockClient.InvokeModelWithResponseStream(
		context.Background(),
		&bedrockruntime.InvokeModelWithResponseStreamInput{
			Body:        payloadBytes,
			ModelId:     aws.String("anthropic.claude-3-haiku-20240307-v1:0"),
			ContentType: aws.String("application/json"),
			Accept:      aws.String("application/json"),
		},
	)

	if error != nil {
		fmt.Println(error)
	}

	fmt.Println(output)

	for event := range output.GetStream().Events() {
		switch v := event.(type) {
		case *types.ResponseStreamMemberChunk:

			// fmt.Println("payload", string(v.Value.Bytes))

			// var resp map[string]interface{}
			var resp ResponseClaude3

			err := json.NewDecoder(bytes.NewReader(v.Value.Bytes)).Decode(&resp)
			if err != nil {
				fmt.Println(err)
			}

			fmt.Println(resp.Delta.Text)

		case *types.UnknownUnionMember:
			fmt.Println("unknown tag:", v.Tag)

		default:
			fmt.Println("union is nil or unknown type")
		}
	}
}

// func main() {

// 	TestHaiku()
// }

Image Analyser#

simple frontend to view image from local
invoke bedorck claude haiku to analyse the image

Here is a sample code to call haiku for image analyzing

func testHaikuImage() {

	// read image from local file
	imageData, error := ioutil.ReadFile("demo.jpeg")

	if error != nil {
		fmt.Println(error)
	}

	// encode image to base64
	base64Image := base64.StdEncoding.EncodeToString(imageData)

	source := map[string]interface{}{
		"type":       "base64",
		"media_type": "image/jpeg",
		"data":       base64Image,
	}

	messages := []map[string]interface{}{{
		"role":    "user",
		"content": []map[string]interface{}{{"type": "image", "source": source}, {"type": "text", "text": "what is in this image?"}},
	}}

	payload := map[string]interface{}{
		"max_tokens":        2048,
		"anthropic_version": "bedrock-2023-05-31",
		"temperature":       0.9,
		"messages":          messages,
	}

	// convert payload struct to bytes
	payloadBytes, error := json.Marshal(payload)

	if error != nil {
		fmt.Println(error)
		// fmt.Fprintf(w, "ERROR")
		// return "", error
	}

	// fmt.Println("invoke bedrock ...")

	// invoke bedrock claude3 haiku
	output, error := BedrockClient.InvokeModel(
		context.Background(),
		&bedrockruntime.InvokeModelInput{
			Body:        payloadBytes,
			ModelId:     aws.String("anthropic.claude-3-haiku-20240307-v1:0"),
			ContentType: aws.String("application/json"),
			Accept:      aws.String("application/json"),
		},
	)

	if error != nil {
		fmt.Println(error)
		// fmt.Fprintf(w, "ERROR")
		// return "", error
	}

	// response
	fmt.Println(string(output.Body))
}

Here is the frontend

image.html

<html>
  <head>
    <title>Image Prompt</title>
    <meta name="viewport" content="width=device-width" />
    <style>
      :root {
        box-sizing: border-box;
      }

      *,
      ::before,
      ::after {
        box-sizing: inherit;
      }

      body {
        background-color: antiquewhite;
      }

      .container {
        max-width: 800px;
        margin: auto;
      }

      .container-form {
        position: relative;
      }

      .input-question {
        width: 100%;
        padding: 15px 10px;
      }
      .button-submit {
        background-color: orange;
        padding: 10px 25px;
        border-radius: 2px;
        border: none;
        outline: none;
        position: absolute;
        top: 50%;
        right: 10px;
        transform: translateY(-50%);
        cursor: pointer;
      }

      .input-file {
        width: 100%;
        padding: 10px 10px;
        background-color: aquamarine;
        cursor: pointer;
      }

      .container-image {
        position: relative;
        background-color: gainsboro;
        padding: 10px 10px;
        align-items: center;
        justify-content: center;
        display: flex;
        max-height: 600px;
        height: 50%;
      }

      .description-image {
        position: absolute;
        bottom: 0;
        left: 0;
        background-color: azure;
        padding: 10px;
        opacity: 0.8;
      }
    </style>
  </head>
  <body>
    <div class="container">
      <div>
        <div class="container-form">
          <form onkeydown="return event.key != 'Enter';">
            <input
              type="text"
              id="question"
              name="question"
              class="input-question"
            />
          </form>
          <button class="button-submit" id="submit" type="submit">
            Submit
          </button>
        </div>
        <input type="file" id="file" class="input-file" />
        <div class="container-image">
          <img src="" alt="image" id="image" width="auto" height="90%" hidden />
          <p class="description-image" id="description-image">
            Lorem ipsum dolor sit amet consectetur, adipisicing elit. Quas
            mollitia magnam repellat, laudantium tempore voluptatibus qui
            temporibus eos? Quasi distinctio ut voluptatum vel fugiat
            exercitationem incidunt, dolorem rem iusto rerum? Lorem ipsum dolor
            sit amet consectetur adipisicing elit. Unde, quaerat dicta aliquid
            sapiente cupiditate totam est voluptatem odio. Exercitationem,
            architecto! Blanditiis rerum ex provident, laborum minus eligendi a
            aspernatur eum!
          </p>
        </div>
      </div>
    </div>
  </body>

  <script>
    const fileInput = document.getElementById('file')
    const image = document.getElementById('image')
    const submit = document.getElementById('submit')
    const desc = document.getElementById('description-image')

    //
    desc.innerText = ''
    let messages = []

    // buffer to store image base64 data
    let imageBase64

    fileInput.addEventListener('change', event => {
      // reset desc
      desc.innerText = ''
      //
      let file = event.target.files[0]
      if (file) {
        var reader = new FileReader()
        reader.onload = e => {
          // console.log(e.target.result);
          image.src = e.target.result

          // base64 image data
          imageBase64 = e.target.result.split(',')[1]
          image.style.display = 'block'
        }
        reader.readAsDataURL(file)
      }
    })

    // call bedrock to analyse image
    const analyseImage = async () => {
      // get user prompt
      const question = document.getElementById('question').value

      // console.log("submit ", question, imageBase64);

      // form message for bedrock
      messages.push({
        role: 'user',
        content: [
          {
            type: 'image',
            source: {
              type: 'base64',
              media_type: 'image/jpeg',
              data: imageBase64
            }
          },
          { type: 'text', text: question }
        ]
      })

      // call post request to analyse image
      try {
        const response = await fetch('/claude-haiku-image', {
          method: 'POST',
          headers: {
            'Content-Type': 'application/json'
          },
          body: JSON.stringify({ messages: messages })
        })

        // console.log(response);
        const reader = response.body.getReader()
        const decoder = new TextDecoder()

        while (true) {
          const { done, value } = await reader.read()
          if (done) {
            break
          }
          const text = decoder.decode(value)
          desc.innerText += text
          // console.log(text);
        }
      } catch (error) {
        console.log(error)
      }

      console.log('DONE')
    }

    submit.addEventListener('click', async event => {
      event.preventDefault()

      await analyseImage()
    })

    // listen on enter
    document
      .getElementById('question')
      .addEventListener('keydown', async event => {
        if (event.code === 'Enter') {
          await analyseImage()
        }
      })
  </script>
</html>

Here is the backend

image.go

package main

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/aws/aws-sdk-go-v2/aws"
	"github.com/aws/aws-sdk-go-v2/service/bedrockruntime"
	"github.com/aws/aws-sdk-go-v2/service/bedrockruntime/types"
)

func HandleHaikuImageAnalyzer(w http.ResponseWriter, r *http.Request) {

	// data type request
	type Message struct {
		Role    string        `json:"role"`
		Content []interface{} `json:"content"`
	}

	type Request struct {
		Messages []Message `json:"messages"`
	}

	type RequestBodyClaude3 struct {
		MaxTokensToSample int       `json:"max_tokens"`
		Temperature       float64   `json:"temperature,omitempty"`
		AnthropicVersion  string    `json:"anthropic_version"`
		Messages          []Message `json:"messages"`
	}

	// data type response
	// type ResponseContent struct {
	// 	Type string `json:"type"`
	// 	Text string `json:"text"`
	// }

	// type Response struct {
	// 	Content []ResponseContent `json:"content"`
	// }

	// parse request
	var request Request
	error := json.NewDecoder(r.Body).Decode(&request)

	if error != nil {
		panic(error)
	}

	// fmt.Println(request)

	// payload for bedrock claude3 haikue
	messages := request.Messages

	payload := RequestBodyClaude3{
		MaxTokensToSample: 2048,
		AnthropicVersion:  "bedrock-2023-05-31",
		Temperature:       0.9,
		Messages:          messages,
	}

	// convert payload struct to bytes
	payloadBytes, error := json.Marshal(payload)

	if error != nil {
		fmt.Println(error)
		fmt.Fprintf(w, "ERROR")
		// return "", error
	}

	// fmt.Println("invoke bedrock ...")

	// invoke bedrock claude3 haiku
	output, error := BedrockClient.InvokeModelWithResponseStream(
		context.Background(),
		&bedrockruntime.InvokeModelWithResponseStreamInput{
			Body:        payloadBytes,
			ModelId:     aws.String("anthropic.claude-3-haiku-20240307-v1:0"),
			ContentType: aws.String("application/json"),
			Accept:      aws.String("application/json"),
		},
	)

	// response
	// var response Response
	// json.NewDecoder(bytes.NewReader(output.Body)).Decode(&response)
	// fmt.Println(response)

	if error != nil {
		fmt.Println(error)
		fmt.Fprintf(w, "ERROR")
		// return "", error
	}

	// stream result to client
	for event := range output.GetStream().Events() {

		// fmt.Println(event)

		switch v := event.(type) {
		case *types.ResponseStreamMemberChunk:

			// fmt.Println("payload", string(v.Value.Bytes))

			var resp ResponseClaude3
			err := json.NewDecoder(bytes.NewReader(v.Value.Bytes)).Decode(&resp)
			if err != nil {
				fmt.Fprintf(w, "ERROR")
				// return "", err
			}

			// fmt.Println(resp.Delta.Text)

			fmt.Fprintf(w, resp.Delta.Text)
			if f, ok := w.(http.Flusher); ok {
				f.Flush()
			} else {
				fmt.Println("Damn, no flush")
			}

		case *types.UnknownUnionMember:
			fmt.Println("unknown tag:", v.Tag)

		default:
			fmt.Println("union is nil or unknown type")
		}
	}
}

And here is a test script

Bedrock Batch#

call claude3 batch mode for chat
call claude3 batch mode for image

Let write a sample code to invoke claude3 in batch mode

package main

import (
	"bytes"
	"context"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io/ioutil"

	"github.com/aws/aws-sdk-go-v2/service/bedrockruntime/types"

	"github.com/aws/aws-sdk-go-v2/aws"
	"github.com/aws/aws-sdk-go-v2/service/bedrockruntime"
)

func testHaikuBatch(promt string) {

	// list of messages
	messages := []map[string]interface{}{{
		"role":    "user",
		"content": []map[string]interface{}{{"type": "text", "text": promt}},
	}}

	// form request body
	payload := map[string]interface{}{
		"max_tokens":        2048,
		"anthropic_version": "bedrock-2023-05-31",
		"temperature":       0.9,
		"messages":          messages,
	}

	// marshal payload to bytes
	payloadBytes, err := json.Marshal(payload)

	if err != nil {
		fmt.Println(err)
		return
	}

	// create request to bedrock
	output, error := BedrockClient.InvokeModel(
		context.Background(),
		&bedrockruntime.InvokeModelInput{
			Body:        payloadBytes,
			ModelId:     aws.String("anthropic.claude-3-haiku-20240307-v1:0"),
			ContentType: aws.String("application/json"),
			Accept:      aws.String("application/json"),
		},
	)

	if error != nil {
		fmt.Println(error)
		return
	}

	// unmarshal response
	fmt.Println(string(output.Body))
}

And in stream mode

func testHaikuStream(promt string) {

	// list of messages
	messages := []map[string]interface{}{{
		"role":    "user",
		"content": []map[string]interface{}{{"type": "text", "text": promt}},
	}}

	// form request body
	payload := map[string]interface{}{
		"max_tokens":        2048,
		"anthropic_version": "bedrock-2023-05-31",
		"temperature":       0.9,
		"messages":          messages,
	}

	// marshal payload to bytes
	payloadBytes, err := json.Marshal(payload)

	if err != nil {
		fmt.Println(err)
		return
	}

	// create request to bedrock
	output, error := BedrockClient.InvokeModelWithResponseStream(
		context.Background(),
		&bedrockruntime.InvokeModelWithResponseStreamInput{
			Body:        payloadBytes,
			ModelId:     aws.String("anthropic.claude-3-haiku-20240307-v1:0"),
			ContentType: aws.String("application/json"),
			Accept:      aws.String("application/json"),
		},
	)

	if error != nil {
		fmt.Println(error)
		return
	}

	// parse response stream
	for event := range output.GetStream().Events() {
		switch v := event.(type) {
		case *types.ResponseStreamMemberChunk:

			//fmt.Println("payload", string(v.Value.Bytes))

			var resp ResponseClaude3
			err := json.NewDecoder(bytes.NewReader(v.Value.Bytes)).Decode(&resp)
			if err != nil {
				fmt.Println(err)
				// fmt.Fprintf(w, "ERROR")
				// return "", err
			}

			fmt.Println(resp.Delta.Text)

			// fmt.Fprintf(w, resp.Delta.Text)
			// if f, ok := w.(http.Flusher); ok {
			// 	f.Flush()
			// } else {
			// 	fmt.Println("Damn, no flush")
			// }

		case *types.UnknownUnionMember:
			fmt.Println("unknown tag:", v.Tag)

		default:
			fmt.Println("union is nil or unknown type")
		}
	}
}

Here is full script

backend.go

package main

import (
"bytes"
"context"
"encoding/base64"
"encoding/json"
"fmt"
"io/ioutil"

    "github.com/aws/aws-sdk-go-v2/service/bedrockruntime/types"

    "github.com/aws/aws-sdk-go-v2/aws"
    "github.com/aws/aws-sdk-go-v2/service/bedrockruntime"

)

func testHaikuBatch(promt string) {

    // list of messages
    messages := []map[string]interface{}{{
    	"role":    "user",
    	"content": []map[string]interface{}{{"type": "text", "text": promt}},
    }}

    // form request body
    payload := map[string]interface{}{
    	"max_tokens":        2048,
    	"anthropic_version": "bedrock-2023-05-31",
    	"temperature":       0.9,
    	"messages":          messages,
    }

    // marshal payload to bytes
    payloadBytes, err := json.Marshal(payload)

    if err != nil {
    	fmt.Println(err)
    	return
    }

    // create request to bedrock
    output, error := BedrockClient.InvokeModel(
    	context.Background(),
    	&bedrockruntime.InvokeModelInput{
    		Body:        payloadBytes,
    		ModelId:     aws.String("anthropic.claude-3-haiku-20240307-v1:0"),
    		ContentType: aws.String("application/json"),
    		Accept:      aws.String("application/json"),
    	},
    )

    if error != nil {
    	fmt.Println(error)
    	return
    }

    // unmarshal response
    fmt.Println(string(output.Body))

}

func testHaikuStream(promt string) {

    // list of messages
    messages := []map[string]interface{}{{
    	"role":    "user",
    	"content": []map[string]interface{}{{"type": "text", "text": promt}},
    }}

    // form request body
    payload := map[string]interface{}{
    	"max_tokens":        2048,
    	"anthropic_version": "bedrock-2023-05-31",
    	"temperature":       0.9,
    	"messages":          messages,
    }

    // marshal payload to bytes
    payloadBytes, err := json.Marshal(payload)

    if err != nil {
    	fmt.Println(err)
    	return
    }

    // create request to bedrock
    output, error := BedrockClient.InvokeModelWithResponseStream(
    	context.Background(),
    	&bedrockruntime.InvokeModelWithResponseStreamInput{
    		Body:        payloadBytes,
    		ModelId:     aws.String("anthropic.claude-3-haiku-20240307-v1:0"),
    		ContentType: aws.String("application/json"),
    		Accept:      aws.String("application/json"),
    	},
    )

    if error != nil {
    	fmt.Println(error)
    	return
    }

    // parse response stream
    for event := range output.GetStream().Events() {
    	switch v := event.(type) {
    	case *types.ResponseStreamMemberChunk:

    		//fmt.Println("payload", string(v.Value.Bytes))

    		var resp ResponseClaude3
    		err := json.NewDecoder(bytes.NewReader(v.Value.Bytes)).Decode(&resp)
    		if err != nil {
    			fmt.Println(err)
    			// fmt.Fprintf(w, "ERROR")
    			// return "", err
    		}

    		fmt.Println(resp.Delta.Text)

    		// fmt.Fprintf(w, resp.Delta.Text)
    		// if f, ok := w.(http.Flusher); ok {
    		// 	f.Flush()
    		// } else {
    		// 	fmt.Println("Damn, no flush")
    		// }

    	case *types.UnknownUnionMember:
    		fmt.Println("unknown tag:", v.Tag)

    	default:
    		fmt.Println("union is nil or unknown type")
    	}
    }

}

func testHaikuImage() {

    // read image from local file
    imageData, error := ioutil.ReadFile("demo.jpeg")

    if error != nil {
    	fmt.Println(error)
    }

    // encode image to base64
    base64Image := base64.StdEncoding.EncodeToString(imageData)

    // data type request
    // type Message struct {
    // 	Role    string        `json:"role"`
    // 	Content []interface{} `json:"content"`
    // }

    // type Request struct {
    // 	Messages []Message `json:"messages"`
    // }

    // type RequestBodyClaude3 struct {
    // 	MaxTokensToSample int       `json:"max_tokens"`
    // 	Temperature       float64   `json:"temperature,omitempty"`
    // 	AnthropicVersion  string    `json:"anthropic_version"`
    // 	Messages          []Message `json:"messages"`
    // }

    // data type response
    // type ResponseContent struct {
    // 	Type string `json:"type"`
    // 	Text string `json:"text"`
    // }

    // type Response struct {
    // 	Content []ResponseContent `json:"content"`
    // }

    // parse request
    // var request Request
    // error := json.NewDecoder(r.Body).Decode(&request)

    // if error != nil {
    //   panic(error)
    // }

    // fmt.Println(request)

    // payload for bedrock claude3 haikue
    // messages := request.Messages

    //

    source := map[string]interface{}{
    	"type":       "base64",
    	"media_type": "image/jpeg",
    	"data":       base64Image,
    }

    messages := []map[string]interface{}{{
    	"role":    "user",
    	"content": []map[string]interface{}{{"type": "image", "source": source}, {"type": "text", "text": "what is in this image?"}},
    }}

    payload := map[string]interface{}{
    	"max_tokens":        2048,
    	"anthropic_version": "bedrock-2023-05-31",
    	"temperature":       0.9,
    	"messages":          messages,
    }

    // convert payload struct to bytes
    payloadBytes, error := json.Marshal(payload)

    if error != nil {
    	fmt.Println(error)
    	// fmt.Fprintf(w, "ERROR")
    	// return "", error
    }

    // fmt.Println("invoke bedrock ...")

    // invoke bedrock claude3 haiku
    output, error := BedrockClient.InvokeModel(
    	context.Background(),
    	&bedrockruntime.InvokeModelInput{
    		Body:        payloadBytes,
    		ModelId:     aws.String("anthropic.claude-3-haiku-20240307-v1:0"),
    		ContentType: aws.String("application/json"),
    		Accept:      aws.String("application/json"),
    	},
    )

    if error != nil {
    	fmt.Println(error)
    	// fmt.Fprintf(w, "ERROR")
    	// return "", error
    }

    // response
    fmt.Println(string(output.Body))

    // stream result to client
    // for event := range output.GetStream().Events() {

    // 	// fmt.Println(event)

    // 	switch v := event.(type) {
    // 	case *types.ResponseStreamMemberChunk:

    // 		// fmt.Println("payload", string(v.Value.Bytes))

    // 		var resp ResponseClaude3
    // 		err := json.NewDecoder(bytes.NewReader(v.Value.Bytes)).Decode(&resp)
    // 		if err != nil {
    // 			// fmt.Fprintf(w, "ERROR")
    // 			// return "", err
    // 			fmt.Println(err)
    // 		}

    // 		// fmt.Println(resp.Delta.Text)

    // 		// fmt.Fprintf(w, resp.Delta.Text)
    // 		// if f, ok := w.(http.Flusher); ok {
    // 		//   f.Flush()
    // 		// } else {
    // 		//   fmt.Println("Damn, no flush")
    // 		// }

    // 	case *types.UnknownUnionMember:
    // 		fmt.Println("unknown tag:", v.Tag)

    // 	default:
    // 		fmt.Println("union is nil or unknown type")
    // 	}
    // }

}

func main1() {
// testHaikuBatch("how to cook chicken soup?")
// testHaikuStream("how to cook chicken soup?")
testHaikuImage()
}

Update Prompt#

messages to store chat history
update frontend and backend with messaages buffer

Here is frontend

chat.html

<html>
  <head>
    <meta name="viewport" content="width=device-width" />
    <style>
      :root {
        box-sizing: border-box;
      }
      *,
      ::before,
      ::after {
        box-sizing: inherit;
      }

      body {
        /* background-color: antiquewhite; */
      }

      .container {
        width: 100%;
        max-width: 500px;
        margin: auto;
        /* background-color: antiquewhite; */
      }

      .button {
        background-color: #43a047;
        padding: 8px 20px;
        border-radius: 5px;
        border: none;
        cursor: pointer;
        position: absolute;
        transform: translateY(-50%);
        top: 50%;
        right: 10px;
        opacity: 0.8;
      }

      .button:hover {
        background-color: orange;
      }

      .text-input {
        padding: 10px 15px;
        width: 100%;
        outline: none;
        border: solid black 1px;
        background-color: #e0e0e0;
        box-shadow: 0 10px 15px -3px #e0e0e0;

        font-family: system-ui, -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto,
          Oxygen, Ubuntu, Cantarell, 'Open Sans', 'Helvetica Neue', sans-serif;
        font-size: medium;
        font-weight: 400;
        letter-spacing: normal;
        line-height: 25px;
      }

      .text-input:focus {
        border: solid #4caf50 1.5px;
        outline: none;
      }

      .container-input {
        position: relative;
      }

      .form {
        margin-top: 20px;
      }

      .text-model {
        /* color: #4caf50; */
        font-family: system-ui, -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto,
          Oxygen, Ubuntu, Cantarell, 'Open Sans', 'Helvetica Neue', sans-serif;
        font-size: medium;
        font-weight: 400;
        letter-spacing: normal;
        line-height: 25px;
      }
    </style>
  </head>
  <body>
    <div class="container">
      <form id="form" onkeydown="return event.key != 'Enter';" class="form">
        <div class="container-input">
          <input class="text-input" type="text" id="text-input" />
          <button id="submit" class="button">Ask</button>
        </div>
      </form>
      <div>
        <p id="model-answer" class="text-model"></p>
      </div>
    </div>
    <script>
      // conversation turns
      let messages = []

      // get html component for model answer
      const modelAnswer = document.getElementById('model-answer')

      const callBedrockStream = async () => {
        // present model answer to frontend
        modelAnswer.innerText = ''

        // get user question
        const userQuestion = document.getElementById('text-input').value

        // push user question to messages
        messages.push({
          role: 'user',
          content: [{ type: 'text', text: userQuestion }]
        })

        if (userQuestion) {
          try {
            const response = await fetch('/bedrock-haiku', {
              method: 'POST',
              headers: {
                'Content-Type': 'application/json'
              },
              body: JSON.stringify({ messages: messages })
            })

            console.log(response)
            const reader = response.body.getReader()
            const decoder = new TextDecoder()

            while (true) {
              const { done, value } = await reader.read()
              if (done) {
                break
              }
              const text = decoder.decode(value)
              console.log(text)
              modelAnswer.innerText += text
            }

            // push model answer to converstion turn
            messages.push({
              role: 'assistant',
              content: [{ type: 'text', text: modelAnswer.innerText }]
            })
          } catch (error) {
            console.log(error)
          }
        } else {
          console.log('Please enter question ...')
        }
      }

      document
        .getElementById('submit')
        .addEventListener('click', async event => {
          event.preventDefault()
          await callBedrockStream()
        })

      document
        .getElementById('text-input')
        .addEventListener('keydown', async event => {
          if (event.code === 'Enter') {
            await callBedrockStream()
          }
        })
    </script>
  </body>
</html>

Here is backend

haiku.go

package main

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/aws/aws-sdk-go-v2/aws"
	"github.com/aws/aws-sdk-go-v2/service/bedrockruntime"
	"github.com/aws/aws-sdk-go-v2/service/bedrockruntime/types"
)

// claude3 request data type
type Content struct {
	Type string `json:"type"`
	Text string `json:"text"`
}

type Message struct {
	Role    string    `json:"role"`
	Content []Content `json:"content"`
}

type RequestBodyClaude3 struct {
	MaxTokensToSample int       `json:"max_tokens"`
	Temperature       float64   `json:"temperature,omitempty"`
	AnthropicVersion  string    `json:"anthropic_version"`
	Messages          []Message `json:"messages"`
}

// frontend request data type
type FrontEndRequest struct {
	Messages []Message `json:"messages"`
}

// claude3 response data type
type Delta struct {
	Type string `json:"type"`
	Text string `json:"text"`
}

type ResponseClaude3 struct {
	Type  string `json:"type"`
	Index int    `json:"index"`
	Delta Delta  `json:"delta"`
}

// claude2 data type
type Request struct {
	Prompt            string   `json:"prompt"`
	MaxTokensToSample int      `json:"max_tokens_to_sample"`
	Temperature       float64  `json:"temperature,omitempty"`
	TopP              float64  `json:"top_p,omitempty"`
	TopK              int      `json:"top_k,omitempty"`
	StopSequences     []string `json:"stop_sequences,omitempty"`
}

type Response struct {
	Completion string `json:"completion"`
}

type HelloHandler struct{}

type Query struct {
	Topic string `json:"topic"`
}

func HandleBedrockClaude2Chat(w http.ResponseWriter, r *http.Request) {

	const claudePromptFormat = "\n\nHuman: %s\n\nAssistant:"

	var query Query
	var message string

	// parse mesage from request
	error := json.NewDecoder(r.Body).Decode(&query)

	if error != nil {
		message = "how to learn japanese as quick as possible?"
		panic(error)
	}

	message = query.Topic

	fmt.Println(message)

	prompt := "" + fmt.Sprintf(claudePromptFormat, message)

	payload := Request{
		Prompt:            prompt,
		MaxTokensToSample: 2048,
	}

	payloadBytes, error := json.Marshal(payload)

	if error != nil {
		fmt.Fprintf(w, "ERROR")
		// return "", error
	}

	output, error := BedrockClient.InvokeModelWithResponseStream(
		context.Background(),
		&bedrockruntime.InvokeModelWithResponseStreamInput{
			Body:        payloadBytes,
			ModelId:     aws.String("anthropic.claude-v2"),
			ContentType: aws.String("application/json"),
		},
	)

	if error != nil {
		fmt.Fprintf(w, "ERROR")
		// return "", error
	}

	for event := range output.GetStream().Events() {
		switch v := event.(type) {
		case *types.ResponseStreamMemberChunk:

			//fmt.Println("payload", string(v.Value.Bytes))

			var resp Response
			err := json.NewDecoder(bytes.NewReader(v.Value.Bytes)).Decode(&resp)
			if err != nil {
				fmt.Fprintf(w, "ERROR")
				// return "", err
			}

			fmt.Println(resp.Completion)

			fmt.Fprintf(w, resp.Completion)
			if f, ok := w.(http.Flusher); ok {
				f.Flush()
			} else {
				fmt.Println("Damn, no flush")
			}

		case *types.UnknownUnionMember:
			fmt.Println("unknown tag:", v.Tag)

		default:
			fmt.Println("union is nil or unknown type")
		}
	}
}

func HandleBedrockClaude3HaikuChat(w http.ResponseWriter, r *http.Request) {

	// list of messages sent from frontend client
	var request FrontEndRequest

	// parse mesage from request
	error := json.NewDecoder(r.Body).Decode(&request)

	if error != nil {
		panic(error)
	}

	messages := request.Messages

	fmt.Println(messages)

	payload := RequestBodyClaude3{
		MaxTokensToSample: 2048,
		AnthropicVersion:  "bedrock-2023-05-31",
		Temperature:       0.9,
		Messages:          messages,
	}

	payloadBytes, error := json.Marshal(payload)

	if error != nil {
		fmt.Fprintf(w, "ERROR")
		// return "", error
	}

	output, error := BedrockClient.InvokeModelWithResponseStream(
		context.Background(),
		&bedrockruntime.InvokeModelWithResponseStreamInput{
			Body:        payloadBytes,
			ModelId:     aws.String("anthropic.claude-3-haiku-20240307-v1:0"),
			ContentType: aws.String("application/json"),
			Accept:      aws.String("application/json"),
		},
	)

	if error != nil {
		fmt.Fprintf(w, "ERROR")
		// return "", error
	}

	for event := range output.GetStream().Events() {
		switch v := event.(type) {
		case *types.ResponseStreamMemberChunk:

			//fmt.Println("payload", string(v.Value.Bytes))

			var resp ResponseClaude3
			err := json.NewDecoder(bytes.NewReader(v.Value.Bytes)).Decode(&resp)
			if err != nil {
				fmt.Fprintf(w, "ERROR")
				// return "", err
			}

			fmt.Println(resp.Delta.Text)

			fmt.Fprintf(w, resp.Delta.Text)
			if f, ok := w.(http.Flusher); ok {
				f.Flush()
			} else {
				fmt.Println("Damn, no flush")
			}

		case *types.UnknownUnionMember:
			fmt.Println("unknown tag:", v.Tag)

		default:
			fmt.Println("union is nil or unknown type")
		}
	}
}

func HandleHaikuImageAnalyzer(w http.ResponseWriter, r *http.Request) {

	// data type request
	type Message struct {
		Role    string        `json:"role"`
		Content []interface{} `json:"content"`
	}

	type Request struct {
		Messages []Message `json:"messages"`
	}

	type RequestBodyClaude3 struct {
		MaxTokensToSample int       `json:"max_tokens"`
		Temperature       float64   `json:"temperature,omitempty"`
		AnthropicVersion  string    `json:"anthropic_version"`
		Messages          []Message `json:"messages"`
	}

	// data type response
	// type ResponseContent struct {
	// 	Type string `json:"type"`
	// 	Text string `json:"text"`
	// }

	// type Response struct {
	// 	Content []ResponseContent `json:"content"`
	// }

	// parse request
	var request Request
	error := json.NewDecoder(r.Body).Decode(&request)

	if error != nil {
		panic(error)
	}

	// fmt.Println(request)

	// payload for bedrock claude3 haikue
	messages := request.Messages

	payload := RequestBodyClaude3{
		MaxTokensToSample: 2048,
		AnthropicVersion:  "bedrock-2023-05-31",
		Temperature:       0.9,
		Messages:          messages,
	}

	// convert payload struct to bytes
	payloadBytes, error := json.Marshal(payload)

	if error != nil {
		fmt.Println(error)
		fmt.Fprintf(w, "ERROR")
		// return "", error
	}

	// fmt.Println("invoke bedrock ...")

	// invoke bedrock claude3 haiku
	output, error := BedrockClient.InvokeModelWithResponseStream(
		context.Background(),
		&bedrockruntime.InvokeModelWithResponseStreamInput{
			Body:        payloadBytes,
			ModelId:     aws.String("anthropic.claude-3-haiku-20240307-v1:0"),
			ContentType: aws.String("application/json"),
			Accept:      aws.String("application/json"),
		},
	)

	// response
	// var response Response
	// json.NewDecoder(bytes.NewReader(output.Body)).Decode(&response)
	// fmt.Println(response)

	if error != nil {
		fmt.Println(error)
		fmt.Fprintf(w, "ERROR")
		// return "", error
	}

	// stream result to client
	for event := range output.GetStream().Events() {

		// fmt.Println(event)

		switch v := event.(type) {
		case *types.ResponseStreamMemberChunk:

			// fmt.Println("payload", string(v.Value.Bytes))

			var resp ResponseClaude3
			err := json.NewDecoder(bytes.NewReader(v.Value.Bytes)).Decode(&resp)
			if err != nil {
				fmt.Fprintf(w, "ERROR")
				// return "", err
			}

			// fmt.Println(resp.Delta.Text)

			fmt.Fprintf(w, resp.Delta.Text)
			if f, ok := w.(http.Flusher); ok {
				f.Flush()
			} else {
				fmt.Println("Damn, no flush")
			}

		case *types.UnknownUnionMember:
			fmt.Println("unknown tag:", v.Tag)

		default:
			fmt.Println("union is nil or unknown type")
		}
	}
}

func TestHaiku() {

	fmt.Println("Hello")

	payload := RequestBodyClaude3{
		MaxTokensToSample: 2048,
		AnthropicVersion:  "bedrock-2023-05-31",
		Temperature:       0.8,
		Messages: []Message{{
			Role: "user",
			Content: []Content{{
				Type: "text",
				Text: "How to cook chicken soup?",
			}},
		}, {
			Role: "assistant",
			Content: []Content{{
				Type: "text",
				Text: `Here is a basic recipe for cooking chicken soup`}},
		},
			{
				Role: "user",
				Content: []Content{{
					Type: "text",
					Text: "How to customize it for 3 years old girl?",
				}},
			},
		},
	}

	payloadBytes, error := json.Marshal(payload)

	if error != nil {
		fmt.Println(error)
	}

	output, error := BedrockClient.InvokeModelWithResponseStream(
		context.Background(),
		&bedrockruntime.InvokeModelWithResponseStreamInput{
			Body:        payloadBytes,
			ModelId:     aws.String("anthropic.claude-3-haiku-20240307-v1:0"),
			ContentType: aws.String("application/json"),
			Accept:      aws.String("application/json"),
		},
	)

	if error != nil {
		fmt.Println(error)
	}

	// fmt.Println(output)

	for event := range output.GetStream().Events() {
		switch v := event.(type) {
		case *types.ResponseStreamMemberChunk:

			// fmt.Println("payload", string(v.Value.Bytes))

			// var resp map[string]interface{}
			var resp ResponseClaude3

			err := json.NewDecoder(bytes.NewReader(v.Value.Bytes)).Decode(&resp)
			if err != nil {
				fmt.Println(err)
			}

			// fmt.Println(resp.Delta.Text)

		case *types.UnknownUnionMember:
			fmt.Println("unknown tag:", v.Tag)

		default:
			fmt.Println("union is nil or unknown type")
		}
	}

}

// func main() {

// 	TestHaiku()
// }

Claude2#

Let write a function to call Claude2 batch mode

func testClaude2Stream() {

	type Response struct {
		Completion string `json:"completion"`
	}

	// build prompt for claude2
	const claudePromptFormat = "\n\nHuman: %s\n\nAssistant:"
	message := "how to cook chicken soup"
	prompt := "" + fmt.Sprintf(claudePromptFormat, message)

	// payload
	payload := Request{
		Prompt:            prompt,
		MaxTokensToSample: 2048,
	}

	// convert payload to bytes
	payloadBytes, error := json.Marshal(payload)

	//
	if error != nil {
		return
	}

	// invoke bedrock stream
	output, error := BedrockClient.InvokeModelWithResponseStream(
		context.Background(),
		&bedrockruntime.InvokeModelWithResponseStreamInput{
			Body:        payloadBytes,
			ModelId:     aws.String("anthropic.claude-v2"),
			ContentType: aws.String("application/json"),
		},
	)

	if error != nil {
		return
	}

	for event := range output.GetStream().Events() {
		switch v := event.(type) {
		case *types.ResponseStreamMemberChunk:

			var resp Response
			err := json.NewDecoder(bytes.NewReader(v.Value.Bytes)).Decode(&resp)
			if err != nil {
				return
			}

			fmt.Println(resp.Completion)

		case *types.UnknownUnionMember:
			fmt.Println("unknown tag:", v.Tag)

		default:
			fmt.Println("union is nil or unknown type")
		}
	}
}

Let write a function to call Claude2 stream mode

func testClaude2Batch() {
	// build prompt for claude2
	const claudePromptFormat = "\n\nHuman: %s\n\nAssistant:"
	message := "how to cook chicken soup"
	prompt := "" + fmt.Sprintf(claudePromptFormat, message)

	// payload
	payload := Request{
		Prompt:            prompt,
		MaxTokensToSample: 2048,
	}

	// convert payload to bytes
	payloadBytes, error := json.Marshal(payload)

	//
	if error != nil {
		return
	}

	// invoke bedork
	output, error := BedrockClient.InvokeModel(
		context.Background(),
		&bedrockruntime.InvokeModelInput{
			Body:        payloadBytes,
			ModelId:     aws.String("anthropic.claude-v2"),
			ContentType: aws.String("application/json"),
		},
	)

	if error != nil {
		fmt.Println(error)
		return
	}

	// parse bedrock output
	var out Response
	err := json.NewDecoder(bytes.NewReader(output.Body)).Decode(&out)

	if err != nil {
		fmt.Println(err)
		return
	}
	fmt.Println(out)
}

Here is the full test script

testBedrockClaude.go

package main

import (
	"bytes"
	"context"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io/ioutil"

	"github.com/aws/aws-sdk-go-v2/service/bedrockruntime/types"

	"github.com/aws/aws-sdk-go-v2/aws"
	"github.com/aws/aws-sdk-go-v2/service/bedrockruntime"
)

func testClaude2Stream() {

	type Response struct {
		Completion string `json:"completion"`
	}

	// build prompt for claude2
	const claudePromptFormat = "\n\nHuman: %s\n\nAssistant:"
	message := "how to cook chicken soup"
	prompt := "" + fmt.Sprintf(claudePromptFormat, message)

	// payload
	payload := Request{
		Prompt:            prompt,
		MaxTokensToSample: 2048,
	}

	// convert payload to bytes
	payloadBytes, error := json.Marshal(payload)

	//
	if error != nil {
		return
	}

	// invoke bedrock stream
	output, error := BedrockClient.InvokeModelWithResponseStream(
		context.Background(),
		&bedrockruntime.InvokeModelWithResponseStreamInput{
			Body:        payloadBytes,
			ModelId:     aws.String("anthropic.claude-v2"),
			ContentType: aws.String("application/json"),
		},
	)

	if error != nil {
		return
	}

	for event := range output.GetStream().Events() {
		switch v := event.(type) {
		case *types.ResponseStreamMemberChunk:

			var resp Response
			err := json.NewDecoder(bytes.NewReader(v.Value.Bytes)).Decode(&resp)
			if err != nil {
				return
			}

			fmt.Println(resp.Completion)

		case *types.UnknownUnionMember:
			fmt.Println("unknown tag:", v.Tag)

		default:
			fmt.Println("union is nil or unknown type")
		}
	}
}

func testClaude2Batch() {

	// build prompt for claude2
	const claudePromptFormat = "\n\nHuman: %s\n\nAssistant:"
	message := "how to cook chicken soup"
	prompt := "" + fmt.Sprintf(claudePromptFormat, message)

	// payload
	payload := Request{
		Prompt:            prompt,
		MaxTokensToSample: 2048,
	}

	// convert payload to bytes
	payloadBytes, error := json.Marshal(payload)

	//
	if error != nil {
		return
	}

	// invoke bedork
	output, error := BedrockClient.InvokeModel(
		context.Background(),
		&bedrockruntime.InvokeModelInput{
			Body:        payloadBytes,
			ModelId:     aws.String("anthropic.claude-v2"),
			ContentType: aws.String("application/json"),
		},
	)

	if error != nil {
		fmt.Println(error)
		return
	}

	// parse bedrock output
	var out Response
	err := json.NewDecoder(bytes.NewReader(output.Body)).Decode(&out)

	if err != nil {
		fmt.Println(err)
		return
	}

	fmt.Println(out)

}

func testHaikuBatch(promt string) {

	// list of messages
	messages := []map[string]interface{}{{
		"role":    "user",
		"content": []map[string]interface{}{{"type": "text", "text": promt}},
	}}

	// form request body
	payload := map[string]interface{}{
		"max_tokens":        2048,
		"anthropic_version": "bedrock-2023-05-31",
		"temperature":       0.9,
		"messages":          messages,
	}

	// marshal payload to bytes
	payloadBytes, err := json.Marshal(payload)

	if err != nil {
		fmt.Println(err)
		return
	}

	// create request to bedrock
	output, error := BedrockClient.InvokeModel(
		context.Background(),
		&bedrockruntime.InvokeModelInput{
			Body:        payloadBytes,
			ModelId:     aws.String("anthropic.claude-3-haiku-20240307-v1:0"),
			ContentType: aws.String("application/json"),
			Accept:      aws.String("application/json"),
		},
	)

	if error != nil {
		fmt.Println(error)
		return
	}

	// unmarshal response
	fmt.Println(string(output.Body))
}

func testHaikuStream(promt string) {

	// data format for claude3
	type Content struct {
		Type string `json:"type"`
		Text string `json:"text"`
	}

	type Message struct {
		Role    string    `json:"role"`
		Content []Content `json:"content"`
	}

	type Body struct {
		MaxTokensToSample int       `json:"max_tokens"`
		Temperature       float64   `json:"temperature,omitempty"`
		AnthropicVersion  string    `json:"anthropic_version"`
		Messages          []Message `json:"messages"`
	}

	// list of messages
	messages := []Message{{
		Role:    "user",
		Content: []Content{{Type: "text", Text: promt}},
	}}

	// form request body
	payload := Body{
		MaxTokensToSample: 2048,
		Temperature:       0.9,
		AnthropicVersion:  "bedrock-2023-05-31",
		Messages:          messages,
	}

	// marshal payload to bytes
	payloadBytes, err := json.Marshal(payload)

	if err != nil {
		fmt.Println(err)
		return
	}

	// create request to bedrock
	output, error := BedrockClient.InvokeModelWithResponseStream(
		context.Background(),
		&bedrockruntime.InvokeModelWithResponseStreamInput{
			Body:        payloadBytes,
			ModelId:     aws.String("anthropic.claude-3-haiku-20240307-v1:0"),
			ContentType: aws.String("application/json"),
			Accept:      aws.String("application/json"),
		},
	)

	if error != nil {
		fmt.Println(error)
		return
	}

	// parse response stream
	for event := range output.GetStream().Events() {
		switch v := event.(type) {
		case *types.ResponseStreamMemberChunk:

			//fmt.Println("payload", string(v.Value.Bytes))

			var resp ResponseClaude3
			err := json.NewDecoder(bytes.NewReader(v.Value.Bytes)).Decode(&resp)
			if err != nil {
				fmt.Println(err)
			}

			fmt.Println(resp.Delta.Text)

		case *types.UnknownUnionMember:
			fmt.Println("unknown tag:", v.Tag)

		default:
			fmt.Println("union is nil or unknown type")
		}
	}

}

func testHaikuStream2(promt string) {

	// list of messages
	messages := []map[string]interface{}{{
		"role":    "user",
		"content": []map[string]interface{}{{"type": "text", "text": promt}},
	}}

	// form request body
	payload := map[string]interface{}{
		"max_tokens":        2048,
		"anthropic_version": "bedrock-2023-05-31",
		"temperature":       0.9,
		"messages":          messages,
	}

	// marshal payload to bytes
	payloadBytes, err := json.Marshal(payload)

	if err != nil {
		fmt.Println(err)
		return
	}

	// create request to bedrock
	output, error := BedrockClient.InvokeModelWithResponseStream(
		context.Background(),
		&bedrockruntime.InvokeModelWithResponseStreamInput{
			Body:        payloadBytes,
			ModelId:     aws.String("anthropic.claude-3-haiku-20240307-v1:0"),
			ContentType: aws.String("application/json"),
			Accept:      aws.String("application/json"),
		},
	)

	if error != nil {
		fmt.Println(error)
		return
	}

	// parse response stream
	for event := range output.GetStream().Events() {
		switch v := event.(type) {
		case *types.ResponseStreamMemberChunk:

			//fmt.Println("payload", string(v.Value.Bytes))

			var resp ResponseClaude3
			err := json.NewDecoder(bytes.NewReader(v.Value.Bytes)).Decode(&resp)
			if err != nil {
				fmt.Println(err)
			}

			fmt.Println(resp.Delta.Text)

		case *types.UnknownUnionMember:
			fmt.Println("unknown tag:", v.Tag)

		default:
			fmt.Println("union is nil or unknown type")
		}
	}
}

func testHaikuImage2() {

	// read image from local file
	imageData, error := ioutil.ReadFile("demo.jpeg")

	if error != nil {
		fmt.Println(error)
	}

	// encode image to base64
	base64Image := base64.StdEncoding.EncodeToString(imageData)

	// data type request
	// type Message struct {
	// 	Role    string        `json:"role"`
	// 	Content []interface{} `json:"content"`
	// }

	// type Request struct {
	// 	Messages []Message `json:"messages"`
	// }

	// type RequestBodyClaude3 struct {
	// 	MaxTokensToSample int       `json:"max_tokens"`
	// 	Temperature       float64   `json:"temperature,omitempty"`
	// 	AnthropicVersion  string    `json:"anthropic_version"`
	// 	Messages          []Message `json:"messages"`
	// }

	// data type response
	// type ResponseContent struct {
	// 	Type string `json:"type"`
	// 	Text string `json:"text"`
	// }

	// type Response struct {
	// 	Content []ResponseContent `json:"content"`
	// }

	// parse request
	// var request Request
	// error := json.NewDecoder(r.Body).Decode(&request)

	// if error != nil {
	//   panic(error)
	// }

	// fmt.Println(request)

	// payload for bedrock claude3 haikue
	// messages := request.Messages

	//

	source := map[string]interface{}{
		"type":       "base64",
		"media_type": "image/jpeg",
		"data":       base64Image,
	}

	messages := []map[string]interface{}{{
		"role":    "user",
		"content": []map[string]interface{}{{"type": "image", "source": source}, {"type": "text", "text": "what is in this image?"}},
	}}

	payload := map[string]interface{}{
		"max_tokens":        2048,
		"anthropic_version": "bedrock-2023-05-31",
		"temperature":       0.9,
		"messages":          messages,
	}

	// convert payload struct to bytes
	payloadBytes, error := json.Marshal(payload)

	if error != nil {
		fmt.Println(error)
		// fmt.Fprintf(w, "ERROR")
		// return "", error
	}

	// fmt.Println("invoke bedrock ...")

	// invoke bedrock claude3 haiku
	output, error := BedrockClient.InvokeModel(
		context.Background(),
		&bedrockruntime.InvokeModelInput{
			Body:        payloadBytes,
			ModelId:     aws.String("anthropic.claude-3-haiku-20240307-v1:0"),
			ContentType: aws.String("application/json"),
			Accept:      aws.String("application/json"),
		},
	)

	if error != nil {
		fmt.Println(error)
		// fmt.Fprintf(w, "ERROR")
		// return "", error
	}

	// response
	fmt.Println(string(output.Body))

	// stream result to client
	// for event := range output.GetStream().Events() {

	// 	// fmt.Println(event)

	// 	switch v := event.(type) {
	// 	case *types.ResponseStreamMemberChunk:

	// 		// fmt.Println("payload", string(v.Value.Bytes))

	// 		var resp ResponseClaude3
	// 		err := json.NewDecoder(bytes.NewReader(v.Value.Bytes)).Decode(&resp)
	// 		if err != nil {
	// 			// fmt.Fprintf(w, "ERROR")
	// 			// return "", err
	// 			fmt.Println(err)
	// 		}

	// 		// fmt.Println(resp.Delta.Text)

	// 		// fmt.Fprintf(w, resp.Delta.Text)
	// 		// if f, ok := w.(http.Flusher); ok {
	// 		//   f.Flush()
	// 		// } else {
	// 		//   fmt.Println("Damn, no flush")
	// 		// }

	// 	case *types.UnknownUnionMember:
	// 		fmt.Println("unknown tag:", v.Tag)

	// 	default:
	// 		fmt.Println("union is nil or unknown type")
	// 	}
	// }

}

func main() {
	// testHaikuBatch("how to cook chicken soup?")
	// testHaikuStream("how to cook chicken soup?")
	// testHaikuImage()
	// testClaude2Batch()
	testClaude2Stream()
}

Here is http server for Claude2 batch and stream mode

claude2.go

package main

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"time"

	"github.com/aws/aws-sdk-go-v2/aws"
	"github.com/aws/aws-sdk-go-v2/config"
	"github.com/aws/aws-sdk-go-v2/service/bedrockruntime"
	"github.com/aws/aws-sdk-go-v2/service/bedrockruntime/types"
)

var brc *bedrockruntime.Client

func init() {

	cfg, err := config.LoadDefaultConfig(context.Background(), config.WithRegion("us-east-1"))

	if err != nil {
		log.Fatal(err)
	}
	brc = bedrockruntime.NewFromConfig(cfg)

}

func main() {

	mux := http.NewServeMux()

	// home page
	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		http.ServeFile(w, r, "./static/bedrock-stream.html")
	})

	// chat streaming response
	mux.HandleFunc("/chat", func(w http.ResponseWriter, r *http.Request) {
		http.ServeFile(w, r, "./static/bedrock-stream.html")
	})

	// bedrock batch response
	mux.HandleFunc("/bedrock-batch", func(w http.ResponseWriter, r *http.Request) {

		var query Query
		var message string

		if r.Method == "GET" {
			fmt.Fprintf(w, "GET METHOD")
		}

		if r.Method == "POST" {
			// parse mesage from request
			error := json.NewDecoder(r.Body).Decode(&query)

			if error != nil {
				message = "how to learn japanese as quick as possible?"
				fmt.Fprintf(w, "ERROR parse query message")
				// panic(error)
			}

			message = query.Topic
			prompt := "" + fmt.Sprintf(claudePromptFormat, message)
			payload := Request{
				Prompt:            prompt,
				MaxTokensToSample: 2048,
			}

			payloadBytes, error := json.Marshal(payload)

			if error != nil {
				http.Error(w, "ERROR create payload bytes", 400)
				return
			}

			// invoke bedork
			output, error := brc.InvokeModel(
				context.Background(),
				&bedrockruntime.InvokeModelInput{
					Body:        payloadBytes,
					ModelId:     aws.String("anthropic.claude-v2"),
					ContentType: aws.String("application/json"),
				},
			)

			if error != nil {
				http.Error(w, "ERROR when invoke bedrock", 400)
				fmt.Println(error)
				return
			}

			// parse bedrock output
			var out Response
			err := json.NewDecoder(bytes.NewReader(output.Body)).Decode(&out)

			if err != nil {
				fmt.Println(err)
				http.Error(w, "ERROR decode bedrock response", 400)
				return
			}

			if out.Completion != "" {
				fmt.Fprintf(w, out.Completion)
			} else {
				http.Error(w, "ERROR bedrock empty response", 400)
			}
		}


	})

	// bedrock batch response
	mux.HandleFunc("/bedrock-stream", func(w http.ResponseWriter, r *http.Request) {

		var query Query
		var message string

		if r.Method == "GET" {
			fmt.Fprintf(w, "GET METHOD")
		}

		if r.Method == "POST" {
			// parse mesage from request
			error := json.NewDecoder(r.Body).Decode(&query)

			if error != nil {
				message = "how to learn japanese as quick as possible?"
				fmt.Fprintf(w, "ERROR parse query message")
				// panic(error)
			}

			message = query.Topic
			prompt := "" + fmt.Sprintf(claudePromptFormat, message)
			payload := Request{
				Prompt:            prompt,
				MaxTokensToSample: 2048,
			}

			payloadBytes, error := json.Marshal(payload)

			if error != nil {
				http.Error(w, "ERROR create payload bytes", 400)
				return
			}

			// invoke bedrock stream
			output, error := brc.InvokeModelWithResponseStream(
				context.Background(),
				&bedrockruntime.InvokeModelWithResponseStreamInput{
					Body:        payloadBytes,
					ModelId:     aws.String("anthropic.claude-v2"),
					ContentType: aws.String("application/json"),
				},
			)

			if error != nil {
				http.Error(w, "ERROR invoke bedrock stream", 400)
				return
			}

			for event := range output.GetStream().Events() {
				switch v := event.(type) {
				case *types.ResponseStreamMemberChunk:

					var resp Response
					err := json.NewDecoder(bytes.NewReader(v.Value.Bytes)).Decode(&resp)
					if err != nil {
						http.Error(w, "ERROR decode bedrock response", 400)
						return
					}

					fmt.Fprintf(w, resp.Completion)
					if f, ok := w.(http.Flusher); ok {
						f.Flush()
					} else {
						fmt.Println("Damn, no flush")
					}

				case *types.UnknownUnionMember:
					fmt.Println("unknown tag:", v.Tag)

				default:
					fmt.Println("union is nil or unknown type")
				}
			}

		}
	})

	// create web server
	server := &http.Server{
		Addr:           ":3000",
		Handler:        mux,
		ReadTimeout:    30 * time.Second,
		WriteTimeout:   30 * time.Second,
		MaxHeaderBytes: 1 << 20,
	}

	// enable logging
	log.Fatal(server.ListenAndServe())

}

// promt format
const claudePromptFormat = "\n\nHuman: %s\n\nAssistant:"

// bedrock handler request
type Request struct {
	Prompt            string   `json:"prompt"`
	MaxTokensToSample int      `json:"max_tokens_to_sample"`
	Temperature       float64  `json:"temperature,omitempty"`
	TopP              float64  `json:"top_p,omitempty"`
	TopK              int      `json:"top_k,omitempty"`
	StopSequences     []string `json:"stop_sequences,omitempty"`
}

type Response struct {
	Completion string `json:"completion"`
}

type HelloHandler struct{}

type Query struct {
	Topic string `json:"topic"`
}

And the simple frontend without memory

main.go

<html>
  <head>
    <style>
      :root {
        box-sizing: border-box;
      }
      *,
      ::before,
      ::after {
        box-sizing: inherit;
      }

      .container {
        width: 80%;
        margin: auto;
        /* background-color: antiquewhite; */
      }

      .question-form {
        position: relative;
      }

      button {
        cursor: pointer;
        background-color: orange;
        padding: 1em;
        padding-left: 1.5em;
        padding-right: 1.5em;
        position: absolute;
        top: 50%;
        transform: translateY(-50%);
        right: 1em;
      }

      .text-area {
        background-color: azure;
        margin-top: 1em;
      }

      .text-input {
        background-color: aquamarine;
        width: 100%;
        padding: 1em;
        font-size: large;
      }
    </style>
  </head>
  <body>
    <div class="container">
      <div class="question-form">
        <form id="form" onkeydown="return event.key != 'Enter';">
          <input class="text-input" type="text" id="text-input" />
        </form>
        <button id="submit">Submit</button>
      </div>
      <div class="text-area">
        <p id="story-output"></p>
      </div>
    </div>
    <script>
      const storyOutput = document.getElementById("story-output");

      const callBedrockStream = async () => {
        storyOutput.innerText = "";

        const topic = document.getElementById("text-input").value;

        if (topic) {
          try {
            const response = await fetch("/bedrock-stream", {
              method: "POST",
              headers: {
                "Content-Type": "application/json",
              },
              body: JSON.stringify({ topic: topic }),
            });

            console.log(response);
            const reader = response.body.getReader();
            const decoder = new TextDecoder();

            while (true) {
              const { done, value } = await reader.read();
              if (done) {
                break;
              }
              const text = decoder.decode(value);
              console.log(text);
              storyOutput.innerText += text;
            }
          } catch (error) {
            console.log(error);
          }
        } else {
          console.log("Please enter question ...");
        }
      };

      document
        .getElementById("submit")
        .addEventListener("click", callBedrockStream);

      document
        .getElementById("text-input")
        .addEventListener("keydown", async (event) => {
          if (event.code === "Enter") {
            await callBedrockStream();
          }
        });
    </script>
  </body>
</html>

Chat Claude2#

Chat Claude3#

OpenSearch#

HTTP Server#

Deploy#

Basic Prompt#

Image Analyser#

Bedrock Batch#

Update Prompt#

Claude2#

Reference#