Skip to main content

Ceph

一个 分布式对象存储 组件

install​

  1. 创建Ceph专用网络
sudo docker network create --driver bridge --subnet 172.20.0.0/16 ceph-network
  1. 拉取搭建用镜像
sudo docker pull ceph/daemon:latest-luminous
  1. 搭建mon节点
sudo docker run -d --name ceph-mon \
--network ceph-network --ip 172.20.0.10 \
-e CLUSTER=ceph -e WEIGHT=1.0 \
-e MON_IP=172.20.0.10 -e MON_NAME=ceph-mon \
-e CEPH_PUBLIC_NETWORK=172.20.0.0/16 \
-v /etc/ceph:/etc/ceph \
-v /var/lib/ceph/:/var/lib/ceph/ \
-v /var/log/ceph/:/var/log/ceph/ \
ceph/daemon:latest-luminous mon
  1. 搭建osd节点
sudo docker exec ceph-mon ceph auth get client.bootstrap-osd -o /var/lib/ceph/bootstrap-osd/ceph.keyring

修改配置文件以兼容etx4硬盘，在文件最后添加：

sudo vi /etc/ceph/ceph.conf
osd max object name len = 256
osd max object namespace len = 64

分别启动三个容器来模拟集群

sudo docker run -d --privileged=true --name ceph-osd-1 \
--network ceph-network --ip 172.20.0.11 \
-e CLUSTER=ceph -e WEIGHT=1.0 \
-e MON_NAME=ceph-mon -e MON_IP=172.20.0.10 \
-e OSD_TYPE=directory \
-v /etc/ceph:/etc/ceph \
-v /var/lib/ceph/:/var/lib/ceph/ \
-v /var/lib/ceph/osd/1:/var/lib/ceph/osd \
-v /etc/localtime:/etc/localtime:ro \
ceph/daemon:latest-luminous osd 

sudo docker run -d --privileged=true --name ceph-osd-2 \
--network ceph-network --ip 172.20.0.12 \
-e CLUSTER=ceph -e WEIGHT=1.0 \
-e MON_NAME=ceph-mon -e MON_IP=172.20.0.10 \
-e OSD_TYPE=directory \
-v /etc/ceph:/etc/ceph \
-v /var/lib/ceph/:/var/lib/ceph/ \
-v /var/lib/ceph/osd/2:/var/lib/ceph/osd \
-v /etc/localtime:/etc/localtime:ro \
ceph/daemon:latest-luminous osd 

sudo docker run -d --privileged=true --name ceph-osd-3 \
--network ceph-network --ip 172.20.0.13 \
-e CLUSTER=ceph -e WEIGHT=1.0 \
-e MON_NAME=ceph-mon -e MON_IP=172.20.0.10 \
-e OSD_TYPE=directory \
-v /etc/ceph:/etc/ceph \
-v /var/lib/ceph/:/var/lib/ceph/ \
-v /var/lib/ceph/osd/3:/var/lib/ceph/osd \
-v /etc/localtime:/etc/localtime:ro \
ceph/daemon:latest-luminous osd
  1. 搭建mgr节点
sudo docker run -d --privileged=true --name ceph-mgr \
--network ceph-network --ip 172.20.0.14 \
-e CLUSTER=ceph -p 7000:7000 \
--pid=container:ceph-mon \
-v /etc/ceph:/etc/ceph \
-v /var/lib/ceph/:/var/lib/ceph/ \
ceph/daemon:latest-luminous mgr

开启管理界面

sudo docker exec ceph-mgr ceph mgr module enable dashboard
  1. 搭建rgw节点
sudo docker exec ceph-mon ceph auth get client.bootstrap-rgw -o /var/lib/ceph/bootstrap-rgw/ceph.keyring 

sudo docker run -d --privileged=true --name ceph-rgw \
--network ceph-network --ip 172.20.0.15 \
-e CLUSTER=ceph -e RGW_NAME=ceph-rgw \
-p 7480:7480 \
-v /var/lib/ceph/:/var/lib/ceph/ \
-v /etc/ceph:/etc/ceph \
-v /etc/localtime:/etc/localtime:ro \
ceph/daemon:latest-luminous rgw
  1. 检查Ceph状态
sudo docker exec ceph-mon ceph -s
  1. 添加rgw用户
sudo docker exec ceph-rgw radosgw-admin user create --uid="admin" --display-name="admin_user"

会返回一个 这样的结果（记下这个 下面需要用到）

{
    "user_id": "admin",
    "display_name": "admin_user",
    "email": "",
    "suspended": 0,
    "max_buckets": 1000,
    "auid": 0,
    "subusers": [],
    "keys": [
        {
            "user": "admin",
            "access_key": "JQOJQ47HXEKAZEXWWDOZ",
            "secret_key": "APU0ya3hEbkVh3XRfjMlmLzq08RYMQCsUDtA8cE0"
        }
    ],
    "swift_keys": [],
    "caps": [],
    "op_mask": "read, write, delete",
    "default_placement": "",
    "placement_tags": [],
    "bucket_quota": {
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    },
    "user_quota": {
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    },
    "temp_url_keys": [],
    "type": "rgw"
}

client​

java​

1. 创建项目 引入依赖​

创建项目 略

引入依赖​

官方的 java 客户端已经很久没更新了
但 Ceph 兼容 S3 的 API

pom.xml
<dependency>
    <groupId>com.amazonaws</groupId>
    <artifactId>aws-java-sdk-s3</artifactId>
    <version>1.11.925</version>
</dependency>

2. 使用 S3 客户端连接​

// 配置协议
ClientConfiguration clientConfig = new ClientConfiguration();
clientConfig.setProtocol(Protocol.HTTP);

// 配置 认证
// 刚才记下来的 accessKey 和 secretKey
String accessKey = "JQOJQ47HXEKAZEXWWDOZ";
String secretKey = "APU0ya3hEbkVh3XRfjMlmLzq08RYMQCsUDtA8cE0";
AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey);

// build client
AmazonS3 client = AmazonS3Client.builder()
        .withClientConfiguration(clientConfig)
        .withCredentials(new AWSStaticCredentialsProvider(credentials))
        .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration("10.18.19.56:7480", null))
        .build();

创建 桶和对象​

Bucket bucket = client.createBucket("my-new-bucket");

client.putObject("my-new-bucket", "key1", "value1");

遍历 桶 和 获取对象​

ObjectListing objects = client.listObjects("my-new-bucket");
do {
    for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
        System.out.println(objectSummary.getKey() + "\t" +
                objectSummary.getSize() + "\t" +
                StringUtils.fromDate(objectSummary.getLastModified()) + "\t" +
                        objectSummary.toString());
    }
    objects = client.listNextBatchOfObjects(objects);
} while (objects.isTruncated());


S3ObjectInputStream key1 = client.getObject("my-new-bucket", "key1")
.getObjectContent();

System.out.println(new String(key1.readAllBytes()));

参考​