CDK Entest

Introduction#

chat using Bedrock Claude2
chat using Bedrock Claude3
embedding vector opensearch
basic prompt

Chat Claude2#

struct to form Bedrock request and parse response
stream Bedrock response to client

Let create some struct for request and response

type Request struct {
	Prompt            string   `json:"prompt"`
	MaxTokensToSample int      `json:"max_tokens_to_sample"`
	Temperature       float64  `json:"temperature,omitempty"`
	TopP              float64  `json:"top_p,omitempty"`
	TopK              int      `json:"top_k,omitempty"`
	StopSequences     []string `json:"stop_sequences,omitempty"`
}

type Response struct {
	Completion string `json:"completion"`
}

type HelloHandler struct{}

type Query struct {
	Topic string `json:"topic"`
}

Let create a function to send request to Bedrock and stream back client

func HandleBedrockClaude2Chat(w http.ResponseWriter, r *http.Request) {

	const claudePromptFormat = "\n\nHuman: %s\n\nAssistant:"

	var query Query
	var message string

	// parse mesage from request
	error := json.NewDecoder(r.Body).Decode(&query)

	if error != nil {
		message = "how to learn japanese as quick as possible?"
		panic(error)
	}

	message = query.Topic

	fmt.Println(message)

	prompt := "" + fmt.Sprintf(claudePromptFormat, message)

	payload := Request{
		Prompt:            prompt,
		MaxTokensToSample: 2048,
	}

	payloadBytes, error := json.Marshal(payload)

	if error != nil {
		fmt.Fprintf(w, "ERROR")
		// return "", error
	}

	output, error := BedrockClient.InvokeModelWithResponseStream(
		context.Background(),
		&bedrockruntime.InvokeModelWithResponseStreamInput{
			Body:        payloadBytes,
			ModelId:     aws.String("anthropic.claude-v2"),
			ContentType: aws.String("application/json"),
		},
	)

	if error != nil {
		fmt.Fprintf(w, "ERROR")
		// return "", error
	}

	for event := range output.GetStream().Events() {
		switch v := event.(type) {
		case *types.ResponseStreamMemberChunk:

			//fmt.Println("payload", string(v.Value.Bytes))

			var resp Response
			err := json.NewDecoder(bytes.NewReader(v.Value.Bytes)).Decode(&resp)
			if err != nil {
				fmt.Fprintf(w, "ERROR")
				// return "", err
			}

			fmt.Println(resp.Completion)

			fmt.Fprintf(w, resp.Completion)
			if f, ok := w.(http.Flusher); ok {
				f.Flush()
			} else {
				fmt.Println("Damn, no flush")
			}

		case *types.UnknownUnionMember:
			fmt.Println("unknown tag:", v.Tag)

		default:
			fmt.Println("union is nil or unknown type")
		}
	}
}

Here is full code

chat-claude2.go

package main

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/aws/aws-sdk-go-v2/aws"
	"github.com/aws/aws-sdk-go-v2/service/bedrockruntime"
	"github.com/aws/aws-sdk-go-v2/service/bedrockruntime/types"
)

type Content struct {
	Type string `json:"type"`
	Text string `json:"text"`
}

type Message struct {
	Role    string    `json:"role"`
	Content []Content `json:"content"`
}

type RequestBodyClaude3 struct {
	MaxTokensToSample int       `json:"max_tokens"`
	Temperature       float64   `json:"temperature,omitempty"`
	AnthropicVersion  string    `json:"anthropic_version"`
	Messages          []Message `json:"messages"`
}

type Delta struct {
	Type string `json:"type"`
	Text string `json:"text"`
}

type ResponseClaude3 struct {
	Type  string `json:"type"`
	Index int    `json:"index"`
	Delta Delta  `json:"delta"`
}

type Request struct {
	Prompt            string   `json:"prompt"`
	MaxTokensToSample int      `json:"max_tokens_to_sample"`
	Temperature       float64  `json:"temperature,omitempty"`
	TopP              float64  `json:"top_p,omitempty"`
	TopK              int      `json:"top_k,omitempty"`
	StopSequences     []string `json:"stop_sequences,omitempty"`
}

type Response struct {
	Completion string `json:"completion"`
}

type HelloHandler struct{}

type Query struct {
	Topic string `json:"topic"`
}

func HandleBedrockClaude2Chat(w http.ResponseWriter, r *http.Request) {

	const claudePromptFormat = "\n\nHuman: %s\n\nAssistant:"

	var query Query
	var message string

	// parse mesage from request
	error := json.NewDecoder(r.Body).Decode(&query)

	if error != nil {
		message = "how to learn japanese as quick as possible?"
		panic(error)
	}

	message = query.Topic

	fmt.Println(message)

	prompt := "" + fmt.Sprintf(claudePromptFormat, message)

	payload := Request{
		Prompt:            prompt,
		MaxTokensToSample: 2048,
	}

	payloadBytes, error := json.Marshal(payload)

	if error != nil {
		fmt.Fprintf(w, "ERROR")
		// return "", error
	}

	output, error := BedrockClient.InvokeModelWithResponseStream(
		context.Background(),
		&bedrockruntime.InvokeModelWithResponseStreamInput{
			Body:        payloadBytes,
			ModelId:     aws.String("anthropic.claude-v2"),
			ContentType: aws.String("application/json"),
		},
	)

	if error != nil {
		fmt.Fprintf(w, "ERROR")
		// return "", error
	}

	for event := range output.GetStream().Events() {
		switch v := event.(type) {
		case *types.ResponseStreamMemberChunk:

			//fmt.Println("payload", string(v.Value.Bytes))

			var resp Response
			err := json.NewDecoder(bytes.NewReader(v.Value.Bytes)).Decode(&resp)
			if err != nil {
				fmt.Fprintf(w, "ERROR")
				// return "", err
			}

			fmt.Println(resp.Completion)

			fmt.Fprintf(w, resp.Completion)
			if f, ok := w.(http.Flusher); ok {
				f.Flush()
			} else {
				fmt.Println("Damn, no flush")
			}

		case *types.UnknownUnionMember:
			fmt.Println("unknown tag:", v.Tag)

		default:
			fmt.Println("union is nil or unknown type")
		}
	}
}

func HandleBedrockClaude3HaikuChat(w http.ResponseWriter, r *http.Request) {

	var query Query
	var message string

	// parse mesage from request
	error := json.NewDecoder(r.Body).Decode(&query)

	if error != nil {
		message = "how to learn japanese as quick as possible?"
		panic(error)
	}

	message = query.Topic

	fmt.Println(message)

	payload := RequestBodyClaude3{
		MaxTokensToSample: 2048,
		AnthropicVersion:  "bedrock-2023-05-31",
		Temperature:       0.8,
		Messages: []Message{{
			Role: "user",
			Content: []Content{{
				Type: "text",
				Text: message,
			}},
		}},
	}

	payloadBytes, error := json.Marshal(payload)

	if error != nil {
		fmt.Fprintf(w, "ERROR")
		// return "", error
	}

	output, error := BedrockClient.InvokeModelWithResponseStream(
		context.Background(),
		&bedrockruntime.InvokeModelWithResponseStreamInput{
			Body:        payloadBytes,
			ModelId:     aws.String("anthropic.claude-3-haiku-20240307-v1:0"),
			ContentType: aws.String("application/json"),
			Accept:      aws.String("application/json"),
		},
	)

	if error != nil {
		fmt.Fprintf(w, "ERROR")
		// return "", error
	}

	for event := range output.GetStream().Events() {
		switch v := event.(type) {
		case *types.ResponseStreamMemberChunk:

			//fmt.Println("payload", string(v.Value.Bytes))

			var resp ResponseClaude3
			err := json.NewDecoder(bytes.NewReader(v.Value.Bytes)).Decode(&resp)
			if err != nil {
				fmt.Fprintf(w, "ERROR")
				// return "", err
			}

			fmt.Println(resp.Delta.Text)

			fmt.Fprintf(w, resp.Delta.Text)
			if f, ok := w.(http.Flusher); ok {
				f.Flush()
			} else {
				fmt.Println("Damn, no flush")
			}

		case *types.UnknownUnionMember:
			fmt.Println("unknown tag:", v.Tag)

		default:
			fmt.Println("union is nil or unknown type")
		}
	}
}

func TestHaiku() {

	fmt.Println("Hello")

	payload := RequestBodyClaude3{
		MaxTokensToSample: 2048,
		AnthropicVersion:  "bedrock-2023-05-31",
		Temperature:       0.8,
		Messages: []Message{{
			Role: "user",
			Content: []Content{{
				Type: "text",
				Text: "How to cook chicken soup?",
			}},
		}},
	}

	payloadBytes, error := json.Marshal(payload)

	if error != nil {
		fmt.Println(error)
	}

	output, error := BedrockClient.InvokeModelWithResponseStream(
		context.Background(),
		&bedrockruntime.InvokeModelWithResponseStreamInput{
			Body:        payloadBytes,
			ModelId:     aws.String("anthropic.claude-3-haiku-20240307-v1:0"),
			ContentType: aws.String("application/json"),
			Accept:      aws.String("application/json"),
		},
	)

	if error != nil {
		fmt.Println(error)
	}

	fmt.Println(output)

	for event := range output.GetStream().Events() {
		switch v := event.(type) {
		case *types.ResponseStreamMemberChunk:

			// fmt.Println("payload", string(v.Value.Bytes))

			// var resp map[string]interface{}
			var resp ResponseClaude3

			err := json.NewDecoder(bytes.NewReader(v.Value.Bytes)).Decode(&resp)
			if err != nil {
				fmt.Println(err)
			}

			fmt.Println(resp.Delta.Text)

		case *types.UnknownUnionMember:
			fmt.Println("unknown tag:", v.Tag)

		default:
			fmt.Println("union is nil or unknown type")
		}
	}

}

Chat Claude3#

struct to form request and parse response from Bedrock
parse and stream back to client

Let create struct for Bedrock request and response

// claude3 data type
type Content struct {
	Type string `json:"type"`
	Text string `json:"text"`
}

type Message struct {
	Role    string    `json:"role"`
	Content []Content `json:"content"`
}

type RequestBodyClaude3 struct {
	MaxTokensToSample int       `json:"max_tokens"`
	Temperature       float64   `json:"temperature,omitempty"`
	AnthropicVersion  string    `json:"anthropic_version"`
	Messages          []Message `json:"messages"`
}

Let create a function to process request for Bedrock and stream back client

func HandleBedrockClaude3HaikuChat(w http.ResponseWriter, r *http.Request) {

	var query Query
	var message string

	// parse mesage from request
	error := json.NewDecoder(r.Body).Decode(&query)

	if error != nil {
		message = "how to learn japanese as quick as possible?"
		panic(error)
	}

	message = query.Topic

	fmt.Println(message)

	payload := RequestBodyClaude3{
		MaxTokensToSample: 2048,
		AnthropicVersion:  "bedrock-2023-05-31",
		Temperature:       0.8,
		Messages: []Message{{
			Role: "user",
			Content: []Content{{
				Type: "text",
				Text: message,
			}},
		}},
	}

	payloadBytes, error := json.Marshal(payload)

	if error != nil {
		fmt.Fprintf(w, "ERROR")
		// return "", error
	}

	output, error := BedrockClient.InvokeModelWithResponseStream(
		context.Background(),
		&bedrockruntime.InvokeModelWithResponseStreamInput{
			Body:        payloadBytes,
			ModelId:     aws.String("anthropic.claude-3-haiku-20240307-v1:0"),
			ContentType: aws.String("application/json"),
			Accept:      aws.String("application/json"),
		},
	)

	if error != nil {
		fmt.Fprintf(w, "ERROR")
		// return "", error
	}

	for event := range output.GetStream().Events() {
		switch v := event.(type) {
		case *types.ResponseStreamMemberChunk:

			//fmt.Println("payload", string(v.Value.Bytes))

			var resp ResponseClaude3
			err := json.NewDecoder(bytes.NewReader(v.Value.Bytes)).Decode(&resp)
			if err != nil {
				fmt.Fprintf(w, "ERROR")
				// return "", err
			}

			fmt.Println(resp.Delta.Text)

			fmt.Fprintf(w, resp.Delta.Text)
			if f, ok := w.(http.Flusher); ok {
				f.Flush()
			} else {
				fmt.Println("Damn, no flush")
			}

		case *types.UnknownUnionMember:
			fmt.Println("unknown tag:", v.Tag)

		default:
			fmt.Println("union is nil or unknown type")
		}
	}
}

Here is full code

chat-claude3.go

package main

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/aws/aws-sdk-go-v2/aws"
	"github.com/aws/aws-sdk-go-v2/service/bedrockruntime"
	"github.com/aws/aws-sdk-go-v2/service/bedrockruntime/types"
)

type Content struct {
	Type string `json:"type"`
	Text string `json:"text"`
}

type Message struct {
	Role    string    `json:"role"`
	Content []Content `json:"content"`
}

type RequestBodyClaude3 struct {
	MaxTokensToSample int       `json:"max_tokens"`
	Temperature       float64   `json:"temperature,omitempty"`
	AnthropicVersion  string    `json:"anthropic_version"`
	Messages          []Message `json:"messages"`
}

type Delta struct {
	Type string `json:"type"`
	Text string `json:"text"`
}

type ResponseClaude3 struct {
	Type  string `json:"type"`
	Index int    `json:"index"`
	Delta Delta  `json:"delta"`
}

type Request struct {
	Prompt            string   `json:"prompt"`
	MaxTokensToSample int      `json:"max_tokens_to_sample"`
	Temperature       float64  `json:"temperature,omitempty"`
	TopP              float64  `json:"top_p,omitempty"`
	TopK              int      `json:"top_k,omitempty"`
	StopSequences     []string `json:"stop_sequences,omitempty"`
}

type Response struct {
	Completion string `json:"completion"`
}

type HelloHandler struct{}

type Query struct {
	Topic string `json:"topic"`
}

func HandleBedrockClaude2Chat(w http.ResponseWriter, r *http.Request) {

	const claudePromptFormat = "\n\nHuman: %s\n\nAssistant:"

	var query Query
	var message string

	// parse mesage from request
	error := json.NewDecoder(r.Body).Decode(&query)

	if error != nil {
		message = "how to learn japanese as quick as possible?"
		panic(error)
	}

	message = query.Topic

	fmt.Println(message)

	prompt := "" + fmt.Sprintf(claudePromptFormat, message)

	payload := Request{
		Prompt:            prompt,
		MaxTokensToSample: 2048,
	}

	payloadBytes, error := json.Marshal(payload)

	if error != nil {
		fmt.Fprintf(w, "ERROR")
		// return "", error
	}

	output, error := BedrockClient.InvokeModelWithResponseStream(
		context.Background(),
		&bedrockruntime.InvokeModelWithResponseStreamInput{
			Body:        payloadBytes,
			ModelId:     aws.String("anthropic.claude-v2"),
			ContentType: aws.String("application/json"),
		},
	)

	if error != nil {
		fmt.Fprintf(w, "ERROR")
		// return "", error
	}

	for event := range output.GetStream().Events() {
		switch v := event.(type) {
		case *types.ResponseStreamMemberChunk:

			//fmt.Println("payload", string(v.Value.Bytes))

			var resp Response
			err := json.NewDecoder(bytes.NewReader(v.Value.Bytes)).Decode(&resp)
			if err != nil {
				fmt.Fprintf(w, "ERROR")
				// return "", err
			}

			fmt.Println(resp.Completion)

			fmt.Fprintf(w, resp.Completion)
			if f, ok := w.(http.Flusher); ok {
				f.Flush()
			} else {
				fmt.Println("Damn, no flush")
			}

		case *types.UnknownUnionMember:
			fmt.Println("unknown tag:", v.Tag)

		default:
			fmt.Println("union is nil or unknown type")
		}
	}
}

func HandleBedrockClaude3HaikuChat(w http.ResponseWriter, r *http.Request) {

	var query Query
	var message string

	// parse mesage from request
	error := json.NewDecoder(r.Body).Decode(&query)

	if error != nil {
		message = "how to learn japanese as quick as possible?"
		panic(error)
	}

	message = query.Topic

	fmt.Println(message)

	payload := RequestBodyClaude3{
		MaxTokensToSample: 2048,
		AnthropicVersion:  "bedrock-2023-05-31",
		Temperature:       0.8,
		Messages: []Message{{
			Role: "user",
			Content: []Content{{
				Type: "text",
				Text: message,
			}},
		}},
	}

	payloadBytes, error := json.Marshal(payload)

	if error != nil {
		fmt.Fprintf(w, "ERROR")
		// return "", error
	}

	output, error := BedrockClient.InvokeModelWithResponseStream(
		context.Background(),
		&bedrockruntime.InvokeModelWithResponseStreamInput{
			Body:        payloadBytes,
			ModelId:     aws.String("anthropic.claude-3-haiku-20240307-v1:0"),
			ContentType: aws.String("application/json"),
			Accept:      aws.String("application/json"),
		},
	)

	if error != nil {
		fmt.Fprintf(w, "ERROR")
		// return "", error
	}

	for event := range output.GetStream().Events() {
		switch v := event.(type) {
		case *types.ResponseStreamMemberChunk:

			//fmt.Println("payload", string(v.Value.Bytes))

			var resp ResponseClaude3
			err := json.NewDecoder(bytes.NewReader(v.Value.Bytes)).Decode(&resp)
			if err != nil {
				fmt.Fprintf(w, "ERROR")
				// return "", err
			}

			fmt.Println(resp.Delta.Text)

			fmt.Fprintf(w, resp.Delta.Text)
			if f, ok := w.(http.Flusher); ok {
				f.Flush()
			} else {
				fmt.Println("Damn, no flush")
			}

		case *types.UnknownUnionMember:
			fmt.Println("unknown tag:", v.Tag)

		default:
			fmt.Println("union is nil or unknown type")
		}
	}
}

func TestHaiku() {

	fmt.Println("Hello")

	payload := RequestBodyClaude3{
		MaxTokensToSample: 2048,
		AnthropicVersion:  "bedrock-2023-05-31",
		Temperature:       0.8,
		Messages: []Message{{
			Role: "user",
			Content: []Content{{
				Type: "text",
				Text: "How to cook chicken soup?",
			}},
		}},
	}

	payloadBytes, error := json.Marshal(payload)

	if error != nil {
		fmt.Println(error)
	}

	output, error := BedrockClient.InvokeModelWithResponseStream(
		context.Background(),
		&bedrockruntime.InvokeModelWithResponseStreamInput{
			Body:        payloadBytes,
			ModelId:     aws.String("anthropic.claude-3-haiku-20240307-v1:0"),
			ContentType: aws.String("application/json"),
			Accept:      aws.String("application/json"),
		},
	)

	if error != nil {
		fmt.Println(error)
	}

	fmt.Println(output)

	for event := range output.GetStream().Events() {
		switch v := event.(type) {
		case *types.ResponseStreamMemberChunk:

			// fmt.Println("payload", string(v.Value.Bytes))

			// var resp map[string]interface{}
			var resp ResponseClaude3

			err := json.NewDecoder(bytes.NewReader(v.Value.Bytes)).Decode(&resp)
			if err != nil {
				fmt.Println(err)
			}

			fmt.Println(resp.Delta.Text)

		case *types.UnknownUnionMember:
			fmt.Println("unknown tag:", v.Tag)

		default:
			fmt.Println("union is nil or unknown type")
		}
	}

}

OpenSearch#

Query all items
Query given a vector
Get embedding vec from Bedrock Titan

Let query all item

type EmbedResponse struct {
	Embedding []float64 `json:"embedding"`
}

type Hits struct {
	Hits []map[string]interface{} `json:"hits"`
}

type AossResponse struct {
	Hits Hits `json:"hits"`
}

func QueryAOSS(vec []float64) ([]string, error) {

	// let query get all item in an index

	content := strings.NewReader(`{
	    "size": 10,
	    "query": {
	        "match_all": {}
	        }
	}`)

	vecStr := make([]string, len(vec))

	// convert array float to string
	for k, v := range vec {

		if k < len(vec)-1 {
			vecStr[k] = fmt.Sprint(v) + ","
		} else {
			vecStr[k] = fmt.Sprint(v)
		}

	}

	// create request body to titan model
	// content := strings.NewReader(fmt.Sprintf(`{
	// 	"size": 5,
	// 	"query": {
	// 		"knn": {
	// 			"vector_field": {
	// 				"vector": %s,
	// 				"k": 5
	// 			}
	// 		}
	// 	}
	// }`, vecStr))

	// fmt.Println(content)

	search := opensearchapi.SearchRequest{
		Index: []string{"demo"},
		Body:  content,
	}

	searchResponse, err := search.Do(context.Background(), AOSSClient)

	if err != nil {
		log.Fatal(err)
	}

	// fmt.Println(searchResponse)

	var answer AossResponse

	json.NewDecoder(searchResponse.Body).Decode(&answer)

	// first := answer.Hits.Hits[0]

	// fmt.Printf("id: %s\n, index: %s\n, text: %s", first["_id"], first["_index"], first["_source"].(map[string]interface{})["text"])

	// fmt.Println(answer.Hits.Hits[0]["_id"])

	queryResult := answer.Hits.Hits[0]["_source"].(map[string]interface{})["text"]

	if queryResult == nil {
		return []string{"nil"}, nil
	}

	// extract hint text only
	hits := []string{}

	for k, v := range answer.Hits.Hits {

		if k > 0 {
			hits = append(hits, v["_source"].(map[string]interface{})["text"].(string))
		}

	}

	return hits, nil

	// return fmt.Sprint(queryResult), nil

}

Let query given a vector

func GetEmbedVector(string) ([]float64, error) {

	// create request body to titan model
	body := map[string]interface{}{
		"inputText": "what is amazon bedrock?",
	}
	bodyJson, err := json.Marshal(body)

	if err != nil {
		fmt.Println(err)
		return nil, err
	}

	// invoke bedrock titan model to convert string to embedding vector
	response, error := BedrockClient.InvokeModel(
		context.Background(),
		&bedrockruntime.InvokeModelInput{
			Body:        []byte(bodyJson),
			ModelId:     aws.String("amazon.titan-embed-text-v1"),
			ContentType: aws.String("application/json"),
		},
	)

	if error != nil {
		fmt.Println(error)
		return nil, error
	}

	// assert response to map
	var embedResponse map[string]interface{}

	error = json.Unmarshal(response.Body, &embedResponse)

	if error != nil {
		fmt.Println(error)
		return nil, error
	}

	// assert response to array
	slice, ok := embedResponse["embedding"].([]interface{})

	if !ok {
		fmt.Println(ok)
	}

	// assert to array of float64
	values := make([]float64, len(slice))

	for k, v := range slice {
		values[k] = float64(v.(float64))
	}

	return values, nil
}

func TestQueryAOSS() {

	fmt.Println("Hello")

	v, error := GetEmbedVector("hello")

	if error != nil {
		fmt.Println(error)
	}

	awnsers, error := QueryAOSS(v)

	if error != nil {
		fmt.Println(error)
	}

	// fmt.Println(awnsers)

	for k, v := range awnsers {
		fmt.Println(k, v)
		fmt.Println("====================================")
	}

}

Here is full code

chat-opensearch.go

package main

import (
	"context"
	"encoding/json"
	"fmt"
	"log"
	"strings"

	"github.com/aws/aws-sdk-go-v2/aws"
	"github.com/aws/aws-sdk-go-v2/service/bedrockruntime"
	"github.com/opensearch-project/opensearch-go/v2/opensearchapi"
)

type EmbedResponse struct {
	Embedding []float64 `json:"embedding"`
}

type Hits struct {
	Hits []map[string]interface{} `json:"hits"`
}

type AossResponse struct {
	Hits Hits `json:"hits"`
}

func QueryAOSS(vec []float64) ([]string, error) {

	// let query get all item in an index

	content := strings.NewReader(`{
	    "size": 10,
	    "query": {
	        "match_all": {}
	        }
	}`)

	vecStr := make([]string, len(vec))

	// convert array float to string
	for k, v := range vec {

		if k < len(vec)-1 {
			vecStr[k] = fmt.Sprint(v) + ","
		} else {
			vecStr[k] = fmt.Sprint(v)
		}

	}

	// create request body to titan model
	// content := strings.NewReader(fmt.Sprintf(`{
	// 	"size": 5,
	// 	"query": {
	// 		"knn": {
	// 			"vector_field": {
	// 				"vector": %s,
	// 				"k": 5
	// 			}
	// 		}
	// 	}
	// }`, vecStr))

	// fmt.Println(content)

	search := opensearchapi.SearchRequest{
		Index: []string{"demo"},
		Body:  content,
	}

	searchResponse, err := search.Do(context.Background(), AOSSClient)

	if err != nil {
		log.Fatal(err)
	}

	// fmt.Println(searchResponse)

	var answer AossResponse

	json.NewDecoder(searchResponse.Body).Decode(&answer)

	// first := answer.Hits.Hits[0]

	// fmt.Printf("id: %s\n, index: %s\n, text: %s", first["_id"], first["_index"], first["_source"].(map[string]interface{})["text"])

	// fmt.Println(answer.Hits.Hits[0]["_id"])

	queryResult := answer.Hits.Hits[0]["_source"].(map[string]interface{})["text"]

	if queryResult == nil {
		return []string{"nil"}, nil
	}

	// extract hint text only
	hits := []string{}

	for k, v := range answer.Hits.Hits {

		if k > 0 {
			hits = append(hits, v["_source"].(map[string]interface{})["text"].(string))
		}

	}

	return hits, nil

	// return fmt.Sprint(queryResult), nil

}

func GetEmbedVector(string) ([]float64, error) {

	// create request body to titan model
	body := map[string]interface{}{
		"inputText": "what is amazon bedrock?",
	}
	bodyJson, err := json.Marshal(body)

	if err != nil {
		fmt.Println(err)
		return nil, err
	}

	// invoke bedrock titan model to convert string to embedding vector
	response, error := BedrockClient.InvokeModel(
		context.Background(),
		&bedrockruntime.InvokeModelInput{
			Body:        []byte(bodyJson),
			ModelId:     aws.String("amazon.titan-embed-text-v1"),
			ContentType: aws.String("application/json"),
		},
	)

	if error != nil {
		fmt.Println(error)
		return nil, error
	}

	// assert response to map
	var embedResponse map[string]interface{}

	error = json.Unmarshal(response.Body, &embedResponse)

	if error != nil {
		fmt.Println(error)
		return nil, error
	}

	// assert response to array
	slice, ok := embedResponse["embedding"].([]interface{})

	if !ok {
		fmt.Println(ok)
	}

	// assert to array of float64
	values := make([]float64, len(slice))

	for k, v := range slice {
		values[k] = float64(v.(float64))
	}

	return values, nil
}

func TestQueryAOSS() {

	fmt.Println("Hello")

	v, error := GetEmbedVector("hello")

	if error != nil {
		fmt.Println(error)
	}

	awnsers, error := QueryAOSS(v)

	if error != nil {
		fmt.Println(error)
	}

	// fmt.Println(awnsers)

	for k, v := range awnsers {
		fmt.Println(k, v)
		fmt.Println("====================================")
	}

}

HTTP Server#

create http server and multiplixer
frontend and javascript

http-server.go

package main

import (
	"context"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"time"

	"github.com/aws/aws-sdk-go-v2/config"
	"github.com/aws/aws-sdk-go-v2/service/bedrockruntime"
	opensearch "github.com/opensearch-project/opensearch-go/v2"
	requestsigner "github.com/opensearch-project/opensearch-go/v2/signer/awsv2"
)

// opensearch severless client
var AOSSClient *opensearch.Client

// bedrock client
var BedrockClient *bedrockruntime.Client

// create an init function to initializing opensearch client
func init() {

	//
	fmt.Println("init and create an opensearch client")

	// load aws credentials from profile demo using config
	awsCfg, err := config.LoadDefaultConfig(context.Background(),
		config.WithRegion("us-east-1"),
		config.WithSharedConfigProfile("demo"),
	)

	if err != nil {
		log.Fatal(err)
	}

	// create a aws request signer using requestsigner
	signer, err := requestsigner.NewSignerWithService(awsCfg, "aoss")

	if err != nil {
		log.Fatal(err)
	}

	// create an opensearch client using opensearch package
	AOSSClient, err = opensearch.NewClient(opensearch.Config{
		Addresses: []string{AOSS_ENDPOINT},
		Signer:    signer,
	})

	if err != nil {
		log.Fatal(err)
	}

	// create bedorck runtime client
	BedrockClient = bedrockruntime.NewFromConfig(awsCfg)

}

func main() {

	// create handler multiplexer
	mux := http.NewServeMux()

	// hello message
	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte("Hello"))
	})

	// response simple json
	mux.HandleFunc("/json", func(w http.ResponseWriter, r *http.Request) {
		json.NewEncoder(w).Encode(struct {
			Message string `json:"Message"`
		}{Message: "Hello"})
	})

	// handle a simple post request from a form
	mux.HandleFunc("/form", func(w http.ResponseWriter, r *http.Request) {
		if r.Method == "POST" {
			r.ParseForm()
			name := r.FormValue("name")
			w.Write([]byte(fmt.Sprintf("Hello %s", name)))
		}

		if r.Method == "GET" {
			// w.Write([]byte("Hello"))
			http.ServeFile(w, r, "./static/index.html")
		}
	})

	// query opensarch
	mux.HandleFunc("/query", func(w http.ResponseWriter, r *http.Request) {

		// parse question from user
		question := r.FormValue("query")

		// convert question to embedding vector
		vec, error := GetEmbedVector(question)

		if error != nil {
			fmt.Println(error)
		}

		// do query opensearch
		answers, err := QueryAOSS(vec)

		if err != nil {
			fmt.Println(err)
		}

		json.NewEncoder(w).Encode(struct {
			Messages []string `json:"Messages"`
		}{Messages: answers})
	})

	// handle chat with bedrock
	mux.HandleFunc("/bedrock-stream", HandleBedrockClaude2Chat)


	// bedrock chat frontend
	mux.HandleFunc("/chat", func(w http.ResponseWriter, r *http.Request) {
		http.ServeFile(w, r, "./static/chat.html")
	})

		// handle chat with bedrock
	mux.HandleFunc("/bedrock-haiku", HandleBedrockClaude3HaikuChat)


	// bedrock chat frontend
	mux.HandleFunc("/haiku", func(w http.ResponseWriter, r *http.Request) {
		http.ServeFile(w, r, "./static/haiku.html")
	})

	// create a http server using http
	server := http.Server{
		Addr:           ":3000",
		Handler:        mux,
		ReadTimeout:    10 * time.Second,
		WriteTimeout:   10 * time.Second,
		MaxHeaderBytes: 1 << 20,
	}

	server.ListenAndServe()

}

FrontEnd for chat page

chat.html

<html>
  <head>
    <style>
      :root {
        box-sizing: border-box;
      }
      *,
      ::before,
      ::after {
        box-sizing: inherit;
      }

      .container {
        width: 80%;
        max-width: 600px;
        margin: auto;
        /* background-color: antiquewhite; */
      }

      .question-form {
        position: relative;
      }

      button {
        cursor: pointer;
        background-color: orange;
        padding: 1em;
        padding-left: 1.5em;
        padding-right: 1.5em;
        position: absolute;
        top: 50%;
        transform: translateY(-50%);
        right: 1em;
        opacity: 80;
      }

      .text-area {
        background-color: azure;
        margin-top: 1em;
      }

      .text-input {
        background-color: aquamarine;
        width: 100%;
        padding: 1em;
        font-size: large;
      }
    </style>
  </head>
  <body>
    <div class="container">
      <div class="question-form">
        <form id="form" onkeydown="return event.key != 'Enter';">
          <input class="text-input" type="text" id="text-input" />
        </form>
        <button id="submit">Submit</button>
      </div>
      <div class="text-area">
        <p id="story-output"></p>
      </div>
    </div>
    <script>
      const storyOutput = document.getElementById('story-output')

      const callBedrockStream = async () => {
        storyOutput.innerText = ''

        const topic = document.getElementById('text-input').value

        if (topic) {
          try {
            const response = await fetch('/bedrock-haiku', {
              method: 'POST',
              headers: {
                'Content-Type': 'application/json'
              },
              body: JSON.stringify({ topic: topic })
            })

            console.log(response)
            const reader = response.body.getReader()
            const decoder = new TextDecoder()

            while (true) {
              const { done, value } = await reader.read()
              if (done) {
                break
              }
              const text = decoder.decode(value)
              console.log(text)
              storyOutput.innerText += text
            }
          } catch (error) {
            console.log(error)
          }
        } else {
          console.log('Please enter question ...')
        }
      }

      document
        .getElementById('submit')
        .addEventListener('click', callBedrockStream)

      document
        .getElementById('text-input')
        .addEventListener('keydown', async event => {
          if (event.code === 'Enter') {
            await callBedrockStream()
          }
        })
    </script>
  </body>
</html>

FrontEnd for OpenSearch page

opensearch.html

<html>
  <head>
    <title>Query OpenSearch</title>
    <style>
      body {
        background-color: antiquewhite;
      }

      .container {
        width: 80%;
        max-width: 600px;
        margin: auto;
      }

      .button-submit {
        background-color: orange;
        padding: 5px 20px;
        border-radius: 5px;
        border: none;
        cursor: pointer;
        position: absolute;
        transform: translateY(-50%);
        top: 50%;
        right: 10px;
      }

      .input-query {
        padding: 10px 15px;
        width: 100%;
      }

      .container-input {
        position: relative;
      }

      .form {
        margin-top: 20px;
      }
    </style>
  </head>
  <body>
    <div class="container">
      <form id="form" class="form">
        <div class="container-input">
          <input type="text" id="query" name="query" class="input-query" />
          <button class="button-submit">Submit</button>
        </div>
      </form>
      <div id="list"></div>
    </div>
  </body>

  <script>
    console.log('hello')

    const query = async () => {
      // call post request
      const response = await fetch('/query')

      const json = await response.json()

      console.log(json)

      // update ui
      // document.getElementById("content").innerHTML = json.Messages;

      // Create an array of items
      var items = json.Messages

      // Get the list container element
      var listContainer = document.getElementById('list')

      // Loop through the items array and create list items (<li>)
      for (var i = 0; i < items.length; i++) {
        var listItem = document.createElement('div')
        listItem.style.marginBottom = '15px'
        listItem.style.borderBottom = '1px solid #0000FF'

        var header = document.createElement('h4')
        header.textContent = `Chunk ${i}`

        var itemText = document.createTextNode(items[i])
        listItem.appendChild(header)
        listItem.appendChild(itemText)
        listContainer.appendChild(listItem)
      }
    }

    document.getElementById('form').addEventListener('submit', async event => {
      event.preventDefault()
      console.log('submit form')

      await query()
    })
  </script>
</html>

Deploy#

CDK stack to create an AppRunder service
Dockerfile
Build script

Here is the AppRunder stack

AppRunnerStack

import { Stack, StackProps, aws_apprunner, aws_iam } from 'aws-cdk-lib'
import { Effect } from 'aws-cdk-lib/aws-iam'
import { Construct } from 'constructs'

interface AppRunnerProps extends StackProps {
  ecr: string
  bucket: string
  aossCollectionArn: string
}

export class AppRunnerStack extends Stack {
  constructor(scope: Construct, id: string, props: AppRunnerProps) {
    super(scope, id, props)

    const buildRole = new aws_iam.Role(
      this,
      'RoleForAppRunnerToPullGoBedrockAppImage',
      {
        assumedBy: new aws_iam.ServicePrincipal(
          'build.apprunner.amazonaws.com'
        ),
        roleName: 'RoleForAppRunnerToPullGoBedrockAppImage'
      }
    )

    buildRole.addToPolicy(
      new aws_iam.PolicyStatement({
        effect: Effect.ALLOW,
        resources: ['*'],
        actions: ['ecr:*']
      })
    )

    const instanceRole = new aws_iam.Role(this, 'InstanceRoleForGoBedrockApp', {
      assumedBy: new aws_iam.ServicePrincipal('tasks.apprunner.amazonaws.com'),
      roleName: 'InstanceRoleForApprunnerBedrock'
    })

    instanceRole.addToPolicy(
      new aws_iam.PolicyStatement({
        effect: Effect.ALLOW,
        resources: [
          'arn:aws:bedrock:us-east-1::foundation-model/anthropic.claude-v2',
          'arn:aws:bedrock:us-east-1::foundation-model/stability.stable-diffusion-xl-v1',
          'arn:aws:bedrock:us-east-1::foundation-model/amazon.titan-embed-text-v1'
        ],
        actions: [
          'bedrock:InvokeModel',
          'bedrock:InvokeModelWithResponseStream'
        ]
      })
    )

    instanceRole.addToPolicy(
      new aws_iam.PolicyStatement({
        effect: Effect.ALLOW,
        resources: [`arn:aws:s3:::${props.bucket}/*`],
        actions: ['s3:PutObject', 's3:GetObject']
      })
    )

    instanceRole.addToPolicy(
      new aws_iam.PolicyStatement({
        effect: Effect.ALLOW,
        resources: [props.aossCollectionArn],
        actions: ['aoss:APIAccessAll']
      })
    )

    const autoscaling = new aws_apprunner.CfnAutoScalingConfiguration(
      this,
      'AutoScalingForGoBedrockApp',
      {
        autoScalingConfigurationName: 'AutoScalingForGoBedrockApp',
        // min number instance
        minSize: 1,
        // max number instance
        maxSize: 10,
        // max concurrent request per instance
        maxConcurrency: 100
      }
    )

    const apprunner = new aws_apprunner.CfnService(
      this,
      'GoBedrockAppRunnerService',
      {
        serviceName: 'GoBedrockAppRunnerService',
        sourceConfiguration: {
          authenticationConfiguration: {
            accessRoleArn: buildRole.roleArn
          },

          autoDeploymentsEnabled: false,

          imageRepository: {
            imageIdentifier: props.ecr,
            imageRepositoryType: 'ECR',
            imageConfiguration: {
              port: '3000',
              runtimeEnvironmentVariables: [
                {
                  name: 'BUCKET',
                  value: 'demo'
                },
                {
                  name: 'HOSTNAME',
                  value: '0.0.0.0'
                },
                {
                  name: 'PORT',
                  value: '3000'
                }
              ]
              // startCommand: "",
            }
          }
        },

        instanceConfiguration: {
          cpu: '1 vCPU',
          memory: '2 GB',
          instanceRoleArn: instanceRole.roleArn
        },

        observabilityConfiguration: {
          observabilityEnabled: false
        },

        autoScalingConfigurationArn: autoscaling.ref
      }
    )

    apprunner.addDependency(autoscaling)
  }
}

Here is Dockerfile

# syntax=docker/dockerfile:1

# Build the application from source
FROM golang:1.21.5 AS build-stage

WORKDIR /app

COPY go.mod go.sum ./
RUN go mod download

COPY *.go ./

RUN CGO_ENABLED=0 GOOS=linux go build -o /entest

# Run the tests in the container
FROM build-stage AS run-test-stage

# Deploy the application binary into a lean image
FROM gcr.io/distroless/base-debian11 AS build-release-stage

WORKDIR /

COPY --from=build-stage /entest /entest
COPY *.html ./
COPY static ./static

EXPOSE 3000

USER nonroot:nonroot

ENTRYPOINT ["/entest"]

And build script

import os

# parameters
REGION = "us-east-1"
ACCOUNT = os.environ["ACCOUNT_ID"]

# delete all docker images
os.system("sudo docker system prune -a")

# build go-bedrock-app image
os.system("sudo docker build -t go-bedrock-app . ")

#  aws ecr login
os.system(f"aws ecr get-login-password --region {REGION} | sudo docker login --username AWS --password-stdin {ACCOUNT}.dkr.ecr.{REGION}.amazonaws.com")

# get image id
IMAGE_ID=os.popen("sudo docker images -q go-bedrock-app:latest").read()

# tag go-bedrock-app image
os.system(f"sudo docker tag {IMAGE_ID.strip()} {ACCOUNT}.dkr.ecr.{REGION}.amazonaws.com/go-bedrock-app:latest")

# create ecr repository
os.system(f"aws ecr create-repository --registry-id {ACCOUNT} --repository-name go-bedrock-app")

# push image to ecr
os.system(f"sudo docker push {ACCOUNT}.dkr.ecr.{REGION}.amazonaws.com/go-bedrock-app:latest")

# run locally to test
# os.system(f"sudo docker run -d -p 3001:3000 go-bedrock-app:latest")

Basic Prompt#

Let do a simple multiple conversation turns which mean client web send a list of messaages to bedrock like [user,bot,user, ...].

update client code to store messages
update server code to parese the messages and send request to bedrock

In frontend, we need a buffer to store chat history

// conversation turns
let messages = []

// get html component for model answer
const modelAnswer = document.getElementById('model-answer')

const callBedrockStream = async () => {
  // present model answer to frontend
  modelAnswer.innerText = ''

  // get user question
  const userQuestion = document.getElementById('text-input').value

  // push user question to messages
  messages.push({
    role: 'user',
    content: [{ type: 'text', text: userQuestion }]
  })

  if (userQuestion) {
    try {
      const response = await fetch('/bedrock-haiku', {
        method: 'POST',
        headers: {
          'Content-Type': 'application/json'
        },
        body: JSON.stringify({ messages: messages })
      })

      console.log(response)
      const reader = response.body.getReader()
      const decoder = new TextDecoder()

      while (true) {
        const { done, value } = await reader.read()
        if (done) {
          break
        }
        const text = decoder.decode(value)
        console.log(text)
        modelAnswer.innerText += text
      }

      // push model answer to converstion turn
      messages.push({
        role: 'assistant',
        content: [{ type: 'text', text: modelAnswer.innerText }]
      })
    } catch (error) {
      console.log(error)
    }
  } else {
    console.log('Please enter question ...')
  }
}

document.getElementById('submit').addEventListener('click', async event => {
  event.preventDefault()
  await callBedrockStream()
})

document
  .getElementById('text-input')
  .addEventListener('keydown', async event => {
    if (event.code === 'Enter') {
      await callBedrockStream()
    }
  })

Here is frondend code

haiku.html

<html>
  <head>
    <meta name="viewport" content="width=device-width" />
    <style>
      :root {
        box-sizing: border-box;
      }
      *,
      ::before,
      ::after {
        box-sizing: inherit;
      }

      body {
        /* background-color: antiquewhite; */
      }

      .container {
        width: 100%;
        max-width: 500px;
        margin: auto;
        /* background-color: antiquewhite; */
      }

      .button {
        background-color: #43a047;
        padding: 8px 20px;
        border-radius: 5px;
        border: none;
        cursor: pointer;
        position: absolute;
        transform: translateY(-50%);
        top: 50%;
        right: 10px;
        opacity: 0.8;
      }

      .button:hover {
        background-color: orange;
      }

      .text-input {
        padding: 10px 15px;
        width: 100%;
        outline: none;
        border: solid black 1px;
        background-color: #e0e0e0;
        box-shadow: 0 10px 15px -3px #e0e0e0;

        font-family: system-ui, -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto,
          Oxygen, Ubuntu, Cantarell, 'Open Sans', 'Helvetica Neue', sans-serif;
        font-size: medium;
        font-weight: 400;
        letter-spacing: normal;
        line-height: 25px;
      }

      .text-input:focus {
        border: solid #4caf50 1.5px;
        outline: none;
      }

      .container-input {
        position: relative;
      }

      .form {
        margin-top: 20px;
      }

      .text-model {
        /* color: #4caf50; */
        font-family: system-ui, -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto,
          Oxygen, Ubuntu, Cantarell, 'Open Sans', 'Helvetica Neue', sans-serif;
        font-size: medium;
        font-weight: 400;
        letter-spacing: normal;
        line-height: 25px;
      }
    </style>
  </head>
  <body>
    <div class="container">
      <form id="form" onkeydown="return event.key != 'Enter';" class="form">
        <div class="container-input">
          <input class="text-input" type="text" id="text-input" />
          <button id="submit" class="button">Ask</button>
        </div>
      </form>
      <div>
        <p id="model-answer" class="text-model"></p>
      </div>
    </div>
    <script>
      // conversation turns
      let messages = []

      // get html component for model answer
      const modelAnswer = document.getElementById('model-answer')

      const callBedrockStream = async () => {
        // present model answer to frontend
        modelAnswer.innerText = ''

        // get user question
        const userQuestion = document.getElementById('text-input').value

        // push user question to messages
        messages.push({
          role: 'user',
          content: [{ type: 'text', text: userQuestion }]
        })

        if (userQuestion) {
          try {
            const response = await fetch('/bedrock-haiku', {
              method: 'POST',
              headers: {
                'Content-Type': 'application/json'
              },
              body: JSON.stringify({ messages: messages })
            })

            console.log(response)
            const reader = response.body.getReader()
            const decoder = new TextDecoder()

            while (true) {
              const { done, value } = await reader.read()
              if (done) {
                break
              }
              const text = decoder.decode(value)
              console.log(text)
              modelAnswer.innerText += text
            }

            // push model answer to converstion turn
            messages.push({
              role: 'assistant',
              content: [{ type: 'text', text: modelAnswer.innerText }]
            })
          } catch (error) {
            console.log(error)
          }
        } else {
          console.log('Please enter question ...')
        }
      }

      document
        .getElementById('submit')
        .addEventListener('click', async event => {
          event.preventDefault()
          await callBedrockStream()
        })

      document
        .getElementById('text-input')
        .addEventListener('keydown', async event => {
          if (event.code === 'Enter') {
            await callBedrockStream()
          }
        })
    </script>
  </body>
</html>

And here is server code. First we need some structs to parse client request and form correct prompt for bedrock

// claude3 request data type
type Content struct {
	Type string `json:"type"`
	Text string `json:"text"`
}

type Message struct {
	Role    string    `json:"role"`
	Content []Content `json:"content"`
}

type RequestBodyClaude3 struct {
	MaxTokensToSample int       `json:"max_tokens"`
	Temperature       float64   `json:"temperature,omitempty"`
	AnthropicVersion  string    `json:"anthropic_version"`
	Messages          []Message `json:"messages"`
}

// frontend request data type
type FrontEndRequest struct {
	Messages []Message `json:"messages"`
}

// claude3 response data type
type Delta struct {
	Type string `json:"type"`
	Text string `json:"text"`
}

type ResponseClaude3 struct {
	Type  string `json:"type"`
	Index int    `json:"index"`
	Delta Delta  `json:"delta"`
}

Then here is the full code

bedrock.go

package main

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/aws/aws-sdk-go-v2/aws"
	"github.com/aws/aws-sdk-go-v2/service/bedrockruntime"
	"github.com/aws/aws-sdk-go-v2/service/bedrockruntime/types"
)

// claude3 request data type
type Content struct {
	Type string `json:"type"`
	Text string `json:"text"`
}

type Message struct {
	Role    string    `json:"role"`
	Content []Content `json:"content"`
}

type RequestBodyClaude3 struct {
	MaxTokensToSample int       `json:"max_tokens"`
	Temperature       float64   `json:"temperature,omitempty"`
	AnthropicVersion  string    `json:"anthropic_version"`
	Messages          []Message `json:"messages"`
}

// frontend request data type
type FrontEndRequest struct {
	Messages []Message `json:"messages"`
}

// claude3 response data type
type Delta struct {
	Type string `json:"type"`
	Text string `json:"text"`
}

type ResponseClaude3 struct {
	Type  string `json:"type"`
	Index int    `json:"index"`
	Delta Delta  `json:"delta"`
}

// claude2 data type
type Request struct {
	Prompt            string   `json:"prompt"`
	MaxTokensToSample int      `json:"max_tokens_to_sample"`
	Temperature       float64  `json:"temperature,omitempty"`
	TopP              float64  `json:"top_p,omitempty"`
	TopK              int      `json:"top_k,omitempty"`
	StopSequences     []string `json:"stop_sequences,omitempty"`
}

type Response struct {
	Completion string `json:"completion"`
}

type HelloHandler struct{}

type Query struct {
	Topic string `json:"topic"`
}

func HandleBedrockClaude2Chat(w http.ResponseWriter, r *http.Request) {

	const claudePromptFormat = "\n\nHuman: %s\n\nAssistant:"

	var query Query
	var message string

	// parse mesage from request
	error := json.NewDecoder(r.Body).Decode(&query)

	if error != nil {
		message = "how to learn japanese as quick as possible?"
		panic(error)
	}

	message = query.Topic

	fmt.Println(message)

	prompt := "" + fmt.Sprintf(claudePromptFormat, message)

	payload := Request{
		Prompt:            prompt,
		MaxTokensToSample: 2048,
	}

	payloadBytes, error := json.Marshal(payload)

	if error != nil {
		fmt.Fprintf(w, "ERROR")
		// return "", error
	}

	output, error := BedrockClient.InvokeModelWithResponseStream(
		context.Background(),
		&bedrockruntime.InvokeModelWithResponseStreamInput{
			Body:        payloadBytes,
			ModelId:     aws.String("anthropic.claude-v2"),
			ContentType: aws.String("application/json"),
		},
	)

	if error != nil {
		fmt.Fprintf(w, "ERROR")
		// return "", error
	}

	for event := range output.GetStream().Events() {
		switch v := event.(type) {
		case *types.ResponseStreamMemberChunk:

			//fmt.Println("payload", string(v.Value.Bytes))

			var resp Response
			err := json.NewDecoder(bytes.NewReader(v.Value.Bytes)).Decode(&resp)
			if err != nil {
				fmt.Fprintf(w, "ERROR")
				// return "", err
			}

			fmt.Println(resp.Completion)

			fmt.Fprintf(w, resp.Completion)
			if f, ok := w.(http.Flusher); ok {
				f.Flush()
			} else {
				fmt.Println("Damn, no flush")
			}

		case *types.UnknownUnionMember:
			fmt.Println("unknown tag:", v.Tag)

		default:
			fmt.Println("union is nil or unknown type")
		}
	}
}

func HandleBedrockClaude3HaikuChat(w http.ResponseWriter, r *http.Request) {

	// list of messages sent from frontend client
	var request FrontEndRequest

	// parse mesage from request
	error := json.NewDecoder(r.Body).Decode(&request)

	if error != nil {
		panic(error)
	}

	messages := request.Messages

	fmt.Println(messages)

	payload := RequestBodyClaude3{
		MaxTokensToSample: 2048,
		AnthropicVersion:  "bedrock-2023-05-31",
		Temperature:       0.9,
		Messages:          messages,
	}

	payloadBytes, error := json.Marshal(payload)

	if error != nil {
		fmt.Fprintf(w, "ERROR")
		// return "", error
	}

	output, error := BedrockClient.InvokeModelWithResponseStream(
		context.Background(),
		&bedrockruntime.InvokeModelWithResponseStreamInput{
			Body:        payloadBytes,
			ModelId:     aws.String("anthropic.claude-3-haiku-20240307-v1:0"),
			ContentType: aws.String("application/json"),
			Accept:      aws.String("application/json"),
		},
	)

	if error != nil {
		fmt.Fprintf(w, "ERROR")
		// return "", error
	}

	for event := range output.GetStream().Events() {
		switch v := event.(type) {
		case *types.ResponseStreamMemberChunk:

			//fmt.Println("payload", string(v.Value.Bytes))

			var resp ResponseClaude3
			err := json.NewDecoder(bytes.NewReader(v.Value.Bytes)).Decode(&resp)
			if err != nil {
				fmt.Fprintf(w, "ERROR")
				// return "", err
			}

			fmt.Println(resp.Delta.Text)

			fmt.Fprintf(w, resp.Delta.Text)
			if f, ok := w.(http.Flusher); ok {
				f.Flush()
			} else {
				fmt.Println("Damn, no flush")
			}

		case *types.UnknownUnionMember:
			fmt.Println("unknown tag:", v.Tag)

		default:
			fmt.Println("union is nil or unknown type")
		}
	}
}

func TestHaiku() {

	fmt.Println("Hello")

	payload := RequestBodyClaude3{
		MaxTokensToSample: 2048,
		AnthropicVersion:  "bedrock-2023-05-31",
		Temperature:       0.8,
		Messages: []Message{{
			Role: "user",
			Content: []Content{{
				Type: "text",
				Text: "How to cook chicken soup?",
			}},
		}, {
			Role: "assistant",
			Content: []Content{{
				Type: "text",
				Text: `Here is a basic recipe for cooking chicken soup:

				Ingredients:
				- 1 whole chicken or 3-4 lbs of chicken parts (breasts, thighs, drumsticks)
				- 8 cups of water or chicken broth
				- 2 large carrots, peeled and sliced
				- 2 celery stalks, sliced
				- 1 onion, diced
				- 2 cloves garlic, minced
				- 1 bay leaf
				- 1 tsp dried thyme
				- Salt and pepper to taste
				- Egg noodles or other pasta (optional)

				Instructions:

				1. Place the whole chicken or chicken parts in a large pot and cover with the water or broth. Bring to a boil over high heat.

				2. Once boiling, reduce heat to medium-low, cover and simmer for 45-60 minutes, until the chicken is cooked through.

				3. Remove the chicken from the pot and set aside. Skim any foam or fat that rises to the surface.

				4. Add the carrots, celery, onion, garlic, bay leaf and thyme to the pot. Simmer for 15-20 minutes until the vegetables are tender.

				5. Meanwhile, remove the meat from the chicken bones and shred or chop it into bite-size pieces.

				6. Add the cooked chicken back to the pot and season with salt and pepper to taste.

				7. If using, add the egg noodles or other pasta and cook for the time specified on the package.

				8. Serve the chicken soup hot. You can garnish with fresh parsley or dill if desired.

				The longer you simmer the soup, the more flavorful it will become. You can also add other vegetables like potatoes, peas, or corn. Adjust seasoning as needed.`,
			}},
		},
			{
				Role: "user",
				Content: []Content{{
					Type: "text",
					Text: "How to customize it for 3 years old girl?",
				}},
			},
		},
	}

	payloadBytes, error := json.Marshal(payload)

	if error != nil {
		fmt.Println(error)
	}

	output, error := BedrockClient.InvokeModelWithResponseStream(
		context.Background(),
		&bedrockruntime.InvokeModelWithResponseStreamInput{
			Body:        payloadBytes,
			ModelId:     aws.String("anthropic.claude-3-haiku-20240307-v1:0"),
			ContentType: aws.String("application/json"),
			Accept:      aws.String("application/json"),
		},
	)

	if error != nil {
		fmt.Println(error)
	}

	fmt.Println(output)

	for event := range output.GetStream().Events() {
		switch v := event.(type) {
		case *types.ResponseStreamMemberChunk:

			// fmt.Println("payload", string(v.Value.Bytes))

			// var resp map[string]interface{}
			var resp ResponseClaude3

			err := json.NewDecoder(bytes.NewReader(v.Value.Bytes)).Decode(&resp)
			if err != nil {
				fmt.Println(err)
			}

			fmt.Println(resp.Delta.Text)

		case *types.UnknownUnionMember:
			fmt.Println("unknown tag:", v.Tag)

		default:
			fmt.Println("union is nil or unknown type")
		}
	}
}

// func main() {

// 	TestHaiku()
// }

Image Analyser#

simple frontend to view image from local
invoke bedorck claude haiku to analyse the image

Here is a sample code to call haiku for image analyzing

func testHaikuImage() {

	// read image from local file
	imageData, error := ioutil.ReadFile("demo.jpeg")

	if error != nil {
		fmt.Println(error)
	}

	// encode image to base64
	base64Image := base64.StdEncoding.EncodeToString(imageData)

	source := map[string]interface{}{
		"type":       "base64",
		"media_type": "image/jpeg",
		"data":       base64Image,
	}

	messages := []map[string]interface{}{{
		"role":    "user",
		"content": []map[string]interface{}{{"type": "image", "source": source}, {"type": "text", "text": "what is in this image?"}},
	}}

	payload := map[string]interface{}{
		"max_tokens":        2048,
		"anthropic_version": "bedrock-2023-05-31",
		"temperature":       0.9,
		"messages":          messages,
	}

	// convert payload struct to bytes
	payloadBytes, error := json.Marshal(payload)

	if error != nil {
		fmt.Println(error)
		// fmt.Fprintf(w, "ERROR")
		// return "", error
	}

	// fmt.Println("invoke bedrock ...")

	// invoke bedrock claude3 haiku
	output, error := BedrockClient.InvokeModel(
		context.Background(),
		&bedrockruntime.InvokeModelInput{
			Body:        payloadBytes,
			ModelId:     aws.String("anthropic.claude-3-haiku-20240307-v1:0"),
			ContentType: aws.String("application/json"),
			Accept:      aws.String("application/json"),
		},
	)

	if error != nil {
		fmt.Println(error)
		// fmt.Fprintf(w, "ERROR")
		// return "", error
	}

	// response
	fmt.Println(string(output.Body))
}

Here is the frontend

image.html

<html>
  <head>
    <title>Image Prompt</title>
    <meta name="viewport" content="width=device-width" />
    <style>
      :root {
        box-sizing: border-box;
      }