AVB/bedrock-access-gateway
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

suppot all Claude models Cross-Region Inference (#65)

Browse Source
This commit is contained in:
heisenbergye
2024-10-29 14:43:31 +08:00
committed by GitHub
parent 9cc3ea8253
commit 5f7676608a
6 changed files with 163 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
src/api/models/bedrock.py
View File

@@ -35,6 +35,8 @@ from api.schema import (
EmbeddingsResponse,
EmbeddingsUsage,
Embedding,
)
from api.setting import DEBUG, AWS_REGION
@@ -197,6 +199,59 @@ class BedrockModel(BaseChatModel):
"tool_call": True,
"stream_tool_call": False,
},
# claude 3 Haiku cross-region inference profile
"us.anthropic.claude-3-haiku-20240307-v1:0": {
"system": True,
"multimodal": True,
"tool_call": True,
"stream_tool_call": True,
},
"eu.anthropic.claude-3-haiku-20240307-v1:0": {
"system": True,
"multimodal": True,
"tool_call": True,
"stream_tool_call": True,
},
# claude 3 Opus cross-region inference profile
"us.anthropic.claude-3-opus-20240229-v1:0": {
"system": True,
"multimodal": True,
"tool_call": True,
"stream_tool_call": True,
},
# claude 3 Sonnet cross-region inference profile
"us.anthropic.claude-3-sonnet-20240229-v1:0": {
"system": True,
"multimodal": True,
"tool_call": True,
"stream_tool_call": True,
},
"eu.anthropic.claude-3-sonnet-20240229-v1:0": {
"system": True,
"multimodal": True,
"tool_call": True,
"stream_tool_call": True,
},
# claude 3.5 Sonnet cross-region inference profile
"us.anthropic.claude-3-5-sonnet-20240620-v1:0": {
"system": True,
"multimodal": True,
"tool_call": True,
"stream_tool_call": True,
},
"eu.anthropic.claude-3-5-sonnet-20240620-v1:0": {
"system": True,
"multimodal": True,
"tool_call": True,
"stream_tool_call": True,
},
# claude 3.5 Sonnet v2 cross-region inference profile(Now only us-west-2)
"us.anthropic.claude-3-5-sonnet-20241022-v2:0": {
"system": True,
"multimodal": True,
"tool_call": True,
"stream_tool_call": True,
},
}
def list_models(self) -> list[str]:

5
src/requirements.txt
View File

@@ -5,5 +5,6 @@ mangum==0.17.0
tiktoken==0.6.0
requests==2.32.3
numpy==1.26.4
boto3==1.34.132
botocore==1.34.132
boto3==1.35.49
botocore==1.35.49