RHCS4.0 Tier Transition — self service data movement policies

Introduction

Prerequisites

Configuration

radosgw-admin zonegroup get      "placement_targets": [
{
"name": "default-placement",
"tags": [],
"storage_classes": [
"STANDARD"
]
}
]
radosgw-admin zone get    "placement_pools": [
{
"key": "default-placement",
"val": {
"index_pool": "default.rgw.buckets.index",
"storage_classes": {
"STANDARD": {
"data_pool": "default.rgw.buckets.data"
}
},
"data_extra_pool": "default.rgw.buckets.non-ec",
"index_type": 0
}
}
]
radosgw-admin zonegroup placement add \
--rgw-zonegroup default \
--placement-id default-placement \
--storage-class STANDARD_IA
radosgw-admin zone placement add \
--rgw-zone default \
--placement-id default-placement \
--storage-class STANDARD_IA \
--data-pool default.rgw.STANDARD_IA.data \
--compression lz4
"storage_classes": {
"STANDARD": {
"data_pool": "default.rgw.buckets.data"
},
"STANDARD_IA": {
"data_pool": "default.rgw.STANDARD_IA.data",
"compression_type": "lz4"
}
}
radosgw-admin user create --uid=testie --display-name=testie --access-key=testie --secret=testie
export AWS_ACCESS_KEY_ID=testie
export AWS_SECRET_ACCESS_KEY=testie
aws s3api create-bucket --bucket testie --create-bucket-configuration LocationConstraint=default:default-placement --endpoint-url http://RGW_URLradosgw-admin bucket stats --bucket=testie
{
"bucket": "testie",
"tenant": "",
"zonegroup": "0a827883-1d31-4e68-9553-8076ec28cd63",
"placement_rule": "default-placement",
}
rgw lc debug interval = 60
systemctl restart ceph-radosgw@rgw.$HOSTNAME.service
# import modules
import boto3
import botocore
import datetime
# S3client class defines all needed attributes to ease s3 client connection via boto3 module.
# The class uses boto3 module as base reference.
class S3client: # this method initiates all needed s3 attributes for connection
def __init__(self, endpoint_url, aws_access_key_id, aws_secret_access_key, path_style):
self.endpoint_url = endpoint_url
self.aws_access_key_id = aws_access_key_id
self.aws_secret_access_key = aws_secret_access_key
self.path_style = path_style
# this method configures path style for s3 connection
def configure_path_style(self):
botocore.config.Config(s3=self.path_style)
# this method creates connectivity to a s3 resource such as Ceph RGW
def create_connection(self):
connection = boto3.client(
's3',
endpoint_url=self.endpoint_url,
aws_access_key_id=self.aws_access_key_id,
aws_secret_access_key=self.aws_secret_access_key
)
return connection
def main(): # creates s3 client instance of S3client class
s3client_instance = S3client(
'http://<RGW_URL',
'testie',
'testie',
{"addressing_style": "path"}
)
# configures path style for s3 connectivity
s3client_instance.configure_path_style()
# creates connection to s3 resource, in this case Ceph RGW
s3_connection = s3client_instance.create_connection()
# creates bucket with the specified name
bucket_name = 'testie'

# configures lifecycle configuration for the bucket
# creates a rule that moves all WMV objects after 1 day, and deletes them after 3 days
s3_connection.put_bucket_lifecycle_configuration(
Bucket=bucket_name,
LifecycleConfiguration=
{
"Rules": [
{
"Filter": {
"Prefix": ""
},
"Expiration": {
"Days": 3,
},
"ID": "string",
"Status": "Enabled",
"Transitions": [
{
"Days": 1,
"StorageClass": "STANDARD_IA"
}
]
}
]
}
)
# uploads 100 objects and tag them in the needed tags for expiration
for i in range(100):
object_key = 'WMV' + str(i)
s3_connection.upload_file('/etc/hosts', bucket_name, object_key)
s3_connection.put_object_tagging(
Bucket=bucket_name,
Key=object_key,
Tagging={'TagSet': [{'Key': 'suffix', 'Value': 'WMV'}]}
)
print("object: {0} uploaded".format(object_key))
if __name__ == "__main__":
main()
rados ls -p default.rgw.buckets.data
39664c5a-2b23-4a46-85c1-5bbfeeadfece.4179.1_WMV37
39664c5a-2b23-4a46-85c1-5bbfeeadfece.4179.1_WMV34
39664c5a-2b23-4a46-85c1-5bbfeeadfece.4179.1_WMV29
39664c5a-2b23-4a46-85c1-5bbfeeadfece.4179.1_WMV80
39664c5a-2b23-4a46-85c1-5bbfeeadfece.4179.1_WMV96
39664c5a-2b23-4a46-85c1-5bbfeeadfece.4179.1_WMV0
aws s3api head-object --bucket testie --key WMV50 --endpoint-url http://<RGW_URL>
{
"AcceptRanges": "bytes",
"Expiration": "expiry-date=\"Mon, 09 Mar 2020 15:34:17 GMT\", rule
-id=\"string\"",
"LastModified": "Fri, 06 Mar 2020 15:34:17 GMT",
"ContentLength": 227,
"ETag": "\"d9f12a3992e69b1bf6087ab5c3dda072\"",
"ContentType": "binary/octet-stream",
"Metadata": {},
"StorageClass": "STANDARD_IA"
}
rados ls -p default.rgw.STANDARD_IA.data
39664c5a-2b23-4a46-85c1-5bbfeeadfece.4179.1_WMV37
39664c5a-2b23-4a46-85c1-5bbfeeadfece.4179.1_WMV34
39664c5a-2b23-4a46-85c1-5bbfeeadfece.4179.1_WMV29
39664c5a-2b23-4a46-85c1-5bbfeeadfece.4179.1_WMV80
39664c5a-2b23-4a46-85c1-5bbfeeadfece.4179.1_WMV96
39664c5a-2b23-4a46-85c1-5bbfeeadfece.4179.1_WMV0
rados ls -p default.rgw.buckets.data | wc -l 
0
rados ls -p default.rgw.STANDARD_IA.data | wc -l
0

Conclusion

Solution Architect, Red Hat