Aws CRT¶
Description:
- The AWS Common Runtime (CRT) libraries are a base library of the SDKs. The CRT is a modular family of independent packages, written in C. Each package provides good performance and minimal footprint for different required functionalities.
Resources:
- [Github] https://awslabs.github.io/aws-crt-python/
- [Github] https://github.com/awslabs/aws-crt-python
- [Docs] https://docs.aws.amazon.com/sdkref/latest/guide/common-runtime.html
- [Blog] https://aws.amazon.com/blogs/storage/improving-amazon-s3-throughput-for-the-aws-cli-and-boto3-with-the-aws-common-runtime/
Environment variables¶
This notebook reads its settings from environment variables. Set them before starting Jupyter:
S3_BUCKET: name of an S3 bucket you can accessAWS_REGION: optional, the region of that bucket (defaults tous-east-1)
Example 1 uploads a small file to ganymede-demo/file.txt in that bucket, so you need write access. Examples 2 and 3 download it again. AWS credentials come from the usual AWS configuration (environment variables, ~/.aws, or an instance role).
Example 1¶
Upload local file to s3 using CRT
In [ ]:
Copied!
import os
from boto3.s3.transfer import TransferConfig
from pathlib import Path
import boto3
tmp_path = "/tmp/ganymede_demo"
dir = Path(tmp_path)
dir.mkdir(parents=True, exist_ok=True)
dir.joinpath("file.txt").write_text("Hello from the AWS CRT")
config = TransferConfig(preferred_transfer_client='crt')
s3 = boto3.client('s3', region_name=os.environ.get("AWS_REGION", "us-east-1"))
s3.upload_file(f'{tmp_path}/file.txt', Bucket=os.environ["S3_BUCKET"], Key='ganymede-demo/file.txt', Config=config)
import os
from boto3.s3.transfer import TransferConfig
from pathlib import Path
import boto3
tmp_path = "/tmp/ganymede_demo"
dir = Path(tmp_path)
dir.mkdir(parents=True, exist_ok=True)
dir.joinpath("file.txt").write_text("Hello from the AWS CRT")
config = TransferConfig(preferred_transfer_client='crt')
s3 = boto3.client('s3', region_name=os.environ.get("AWS_REGION", "us-east-1"))
s3.upload_file(f'{tmp_path}/file.txt', Bucket=os.environ["S3_BUCKET"], Key='ganymede-demo/file.txt', Config=config)
Example 2¶
Download the file from s3 to local using CRT
In [ ]:
Copied!
import os
from boto3.s3.transfer import TransferConfig
import boto3
config = TransferConfig(preferred_transfer_client='crt')
s3 = boto3.client('s3', region_name=os.environ.get("AWS_REGION", "us-east-1"))
s3.download_file(
Bucket=os.environ["S3_BUCKET"],
Key='ganymede-demo/file.txt',
Filename='/tmp/ganymede_demo/file_copy.txt',
Config=config,
)
print(open('/tmp/ganymede_demo/file_copy.txt').read())
import os
from boto3.s3.transfer import TransferConfig
import boto3
config = TransferConfig(preferred_transfer_client='crt')
s3 = boto3.client('s3', region_name=os.environ.get("AWS_REGION", "us-east-1"))
s3.download_file(
Bucket=os.environ["S3_BUCKET"],
Key='ganymede-demo/file.txt',
Filename='/tmp/ganymede_demo/file_copy.txt',
Config=config,
)
print(open('/tmp/ganymede_demo/file_copy.txt').read())
Example 3¶
Use the low-level awscrt.s3 client directly to download the object into memory
In [ ]:
Copied!
import os
from awscrt.auth import AwsCredentialsProvider
from awscrt.http import HttpHeaders, HttpRequest
from awscrt.s3 import S3Client, S3RequestType, create_default_s3_signing_config
region = os.environ.get("AWS_REGION", "us-east-1")
bucket = os.environ["S3_BUCKET"]
key = "ganymede-demo/file.txt"
# Sign requests with credentials from the usual AWS configuration
credential_provider = AwsCredentialsProvider.new_default_chain()
signing_config = create_default_s3_signing_config(region=region, credential_provider=credential_provider)
client = S3Client(region=region, signing_config=signing_config)
headers = HttpHeaders([("host", f"{bucket}.s3.{region}.amazonaws.com")])
request = HttpRequest("GET", f"/{key}", headers)
body = bytearray()
def on_body(chunk, offset, **kwargs):
body.extend(chunk)
s3_request = client.make_request(type=S3RequestType.GET_OBJECT, request=request, on_body=on_body)
s3_request.finished_future.result()
print(f"Downloaded {len(body)} bytes from s3://{bucket}/{key}")
print(body.decode())
import os
from awscrt.auth import AwsCredentialsProvider
from awscrt.http import HttpHeaders, HttpRequest
from awscrt.s3 import S3Client, S3RequestType, create_default_s3_signing_config
region = os.environ.get("AWS_REGION", "us-east-1")
bucket = os.environ["S3_BUCKET"]
key = "ganymede-demo/file.txt"
# Sign requests with credentials from the usual AWS configuration
credential_provider = AwsCredentialsProvider.new_default_chain()
signing_config = create_default_s3_signing_config(region=region, credential_provider=credential_provider)
client = S3Client(region=region, signing_config=signing_config)
headers = HttpHeaders([("host", f"{bucket}.s3.{region}.amazonaws.com")])
request = HttpRequest("GET", f"/{key}", headers)
body = bytearray()
def on_body(chunk, offset, **kwargs):
body.extend(chunk)
s3_request = client.make_request(type=S3RequestType.GET_OBJECT, request=request, on_body=on_body)
s3_request.finished_future.result()
print(f"Downloaded {len(body)} bytes from s3://{bucket}/{key}")
print(body.decode())