使用 Terraform 在 AWS 上安装 Rancher Cluster (1)

Submitted by Lizhe on Sat, 03/09/2019 - 20:48

 

1. 下载 terraform

20190309091652

2. 把terraform加入环境变量

简单点可以直接拷贝到 /usr/local/bin

3. 初始化 terraform

20190309092552

4. 使用 tf 文件初始化 Kubernetes 集群

[root@ranchertarget terraform_workspace]# terraform apply

20190309093310

 

[root@ranchertarget terraform_workspace]# cat rancher.tf 
provider "aws" {
  region     = "ap-northeast-1"
  access_key = "AKIAI4IQRLNCGFJVMJFQ"
  secret_key = "0T/4R6ANzjw9IVRzpOUW9gnLx+JCKtGt+SW2Wdbr"
}
variable "cluster-name" {
  default = "rancher-eks"
  type    = "string"
}
data "aws_availability_zones" "available" {}
resource "aws_vpc" "rancher" {
  cidr_block = "10.0.0.0/16"
  tags = "${
    map(
     "Name", "rancher-eks-node",
     "kubernetes.io/cluster/${var.cluster-name}", "shared",
    )
  }"
}
resource "aws_subnet" "rancher" {
  count = 2
  availability_zone = "${data.aws_availability_zones.available.names[count.index]}"
  cidr_block        = "10.0.${count.index}.0/24"
  vpc_id            = "${aws_vpc.rancher.id}"
  tags = "${
    map(
     "Name", "rancher-eks-node",
     "kubernetes.io/cluster/${var.cluster-name}", "shared",
    )
  }"
}
resource "aws_internet_gateway" "rancher" {
  vpc_id = "${aws_vpc.rancher.id}"
  tags = {
    Name = "rancher-eks"
  }
}
resource "aws_route_table" "rancher" {
  vpc_id = "${aws_vpc.rancher.id}"
  route {
    cidr_block = "0.0.0.0/0"
    gateway_id = "${aws_internet_gateway.rancher.id}"
  }
}
resource "aws_route_table_association" "rancher" {
  count = 2
  subnet_id      = "${aws_subnet.rancher.*.id[count.index]}"
  route_table_id = "${aws_route_table.rancher.id}"
}
resource "aws_iam_role" "rancher-cluster" {
  name = "rancher-eks-cluster"
  assume_role_policy = <<POLICY
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Effect": "Allow",
      "Principal": {
        "Service": "eks.amazonaws.com"
      },
      "Action": "sts:AssumeRole"
    }
  ]
}
POLICY
}
resource "aws_iam_role_policy_attachment" "rancher-cluster-AmazonEKSClusterPolicy" {
  policy_arn = "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy"
  role       = "${aws_iam_role.rancher-cluster.name}"
}
resource "aws_iam_role_policy_attachment" "rancher-cluster-AmazonEKSServicePolicy" {
  policy_arn = "arn:aws:iam::aws:policy/AmazonEKSServicePolicy"
  role       = "${aws_iam_role.rancher-cluster.name}"
}
resource "aws_security_group" "rancher-cluster" {
  name        = "rancher-eks-cluster"
  description = "Cluster communication with worker nodes"
  vpc_id      = "${aws_vpc.rancher.id}"
  egress {
    from_port   = 0
    to_port     = 0
    protocol    = "-1"
    cidr_blocks = ["0.0.0.0/0"]
  }
  tags = {
    Name = "rancher-eks"
  }
}
resource "aws_security_group_rule" "rancher-cluster-ingress-workstation-https" {
  cidr_blocks       = ["0.0.0.0/32"]
  description       = "Allow workstation to communicate with the cluster API Server"
  from_port         = 443
  protocol          = "tcp"
  security_group_id = "${aws_security_group.rancher-cluster.id}"
  to_port           = 443
  type              = "ingress"
}
resource "aws_eks_cluster" "rancher" {
  name            = "${var.cluster-name}"
  role_arn        = "${aws_iam_role.rancher-cluster.arn}"
  vpc_config {
    security_group_ids = ["${aws_security_group.rancher-cluster.id}"]
    subnet_ids         = ["${aws_subnet.rancher.*.id}"]
  }
  depends_on = [
    "aws_iam_role_policy_attachment.rancher-cluster-AmazonEKSClusterPolicy",
    "aws_iam_role_policy_attachment.rancher-cluster-AmazonEKSServicePolicy",
  ]
}
locals {
  kubeconfig = <<KUBECONFIG
apiVersion: v1
clusters:
- cluster:
    server: ${aws_eks_cluster.rancher.endpoint}
    certificate-authority-data: ${aws_eks_cluster.rancher.certificate_authority.0.data}
  name: kubernetes
contexts:
- context:
    cluster: kubernetes
    user: aws
  name: aws
current-context: aws
kind: Config
preferences: {}
users:
- name: aws
  user:
    exec:
      apiVersion: client.authentication.k8s.io/v1alpha1
      command: aws-iam-authenticator
      args:
        - "token"
        - "-i"
        - "${var.cluster-name}"
KUBECONFIG
}
output "kubeconfig" {
  value = "${local.kubeconfig}"
}
resource "aws_iam_role" "rancher-node" {
  name = "rancher-eks-node"
  assume_role_policy = <<POLICY
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Effect": "Allow",
      "Principal": {
        "Service": "ec2.amazonaws.com"
      },
      "Action": "sts:AssumeRole"
    }
  ]
}
POLICY
}
resource "aws_iam_role_policy_attachment" "rancher-node-AmazonEKSWorkerNodePolicy" {
  policy_arn = "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy"
  role       = "${aws_iam_role.rancher-node.name}"
}
resource "aws_iam_role_policy_attachment" "rancher-node-AmazonEKS_CNI_Policy" {
  policy_arn = "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy"
  role       = "${aws_iam_role.rancher-node.name}"
}
resource "aws_iam_role_policy_attachment" "rancher-node-AmazonEC2ContainerRegistryReadOnly" {
  policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly"
  role       = "${aws_iam_role.rancher-node.name}"
}
resource "aws_iam_instance_profile" "rancher-node" {
  name = "rancher-eks"
  role = "${aws_iam_role.rancher-node.name}"
}
resource "aws_security_group" "rancher-node" {
  name        = "rancher-eks-node"
  description = "Security group for all nodes in the cluster"
  vpc_id      = "${aws_vpc.rancher.id}"
  egress {
    from_port   = 0
    to_port     = 0
    protocol    = "-1"
    cidr_blocks = ["0.0.0.0/0"]
  }
  tags = "${
    map(
     "Name", "rancher-eks-node",
     "kubernetes.io/cluster/${var.cluster-name}", "owned",
    )
  }"
}
resource "aws_security_group_rule" "rancher-node-ingress-self" {
  description              = "Allow node to communicate with each other"
  from_port                = 0
  protocol                 = "-1"
  security_group_id        = "${aws_security_group.rancher-node.id}"
  source_security_group_id = "${aws_security_group.rancher-node.id}"
  to_port                  = 65535
  type                     = "ingress"
}
resource "aws_security_group_rule" "rancher-node-ingress-cluster" {
  description              = "Allow worker Kubelets and pods to receive communication from the cluster control plane"
  from_port                = 1025
  protocol                 = "tcp"
  security_group_id        = "${aws_security_group.rancher-node.id}"
  source_security_group_id = "${aws_security_group.rancher-cluster.id}"
  to_port                  = 65535
  type                     = "ingress"
}
resource "aws_security_group_rule" "rancher-cluster-ingress-node-https" {
  description              = "Allow pods to communicate with the cluster API Server"
  from_port                = 443
  protocol                 = "tcp"
  security_group_id        = "${aws_security_group.rancher-cluster.id}"
  source_security_group_id = "${aws_security_group.rancher-node.id}"
  to_port                  = 443
  type                     = "ingress"
}
data "aws_ami" "eks-worker" {
  filter {
    name   = "name"
    values = ["amazon-eks-node-${aws_eks_cluster.rancher.version}-v*"]
  }
  most_recent = true
  owners      = ["602401143452"] # Amazon EKS AMI Account ID
}
data "aws_region" "current" {}
locals {
  rancher-node-userdata = <<USERDATA
#!/bin/bash
set -o xtrace
/etc/eks/bootstrap.sh --apiserver-endpoint '${aws_eks_cluster.rancher.endpoint}' --b64-cluster-ca '${aws_eks_cluster.rancher.certificate_authority.0.data}' '${var.cluster-name}'
USERDATA
}
resource "aws_launch_configuration" "rancher" {
  associate_public_ip_address = true
  iam_instance_profile        = "${aws_iam_instance_profile.rancher-node.name}"
  image_id                    = "${data.aws_ami.eks-worker.id}"
  key_name                    = "rancher"
  instance_type               = "t2.small"
  name_prefix                 = "rancher-eks"
  security_groups             = ["${aws_security_group.rancher-node.id}"]
  user_data_base64            = "${base64encode(local.rancher-node-userdata)}"
  lifecycle {
    create_before_destroy = true
  }
}
resource "aws_autoscaling_group" "rancher" {
  desired_capacity     = 2
  launch_configuration = "${aws_launch_configuration.rancher.id}"
  max_size             = 2
  min_size             = 1
  name                 = "rancher-eks"
  vpc_zone_identifier  = ["${aws_subnet.rancher.*.id}"]
  tag {
    key                 = "Name"
    value               = "rancher-eks"
    propagate_at_launch = true
  }
  tag {
    key                 = "kubernetes.io/cluster/${var.cluster-name}"
    value               = "owned"
    propagate_at_launch = true
  }
}
locals {
  config_map_aws_auth = <<CONFIGMAPAWSAUTH
apiVersion: v1
kind: ConfigMap
metadata:
  name: aws-auth
  namespace: kube-system
data:
  mapRoles: |
    - rolearn: ${aws_iam_role.rancher-node.arn}
      username: system:node:{{EC2PrivateDNSName}}
      groups:
        - system:bootstrappers
        - system:nodes
CONFIGMAPAWSAUTH
}
output "config_map_aws_auth" {
  value = "${local.config_map_aws_auth}"
}

5. 安装 aws cli , aws-iam-authenticator 和 kubectl

安装 aws cli

curl -O https://bootstrap.pypa.io/get-pip.py

python get-pip.py --user

export PATH=~/.local/bin:$PATH

pip install awscli --upgrade --user

安装 aws-iam-authenticator , 并且一定要加入环境变量

wget https://amazon-eks.s3-us-west-2.amazonaws.com/1.10.3/2018-07-26/bin/linux/amd64/aws-iam-authenticator

安装kubectl

curl -o kubectl https://amazon-eks.s3-us-west-2.amazonaws.com/1.10.3/2018-07-26/bin/linux/amd64/kubectl

 

配置aws cli

[root@localhost ~]# aws configure

AWS Access Key ID [None]: xxxxxxxxx

AWS Secret Access Key [None]: xxxxxxxxxxxxxxxxxxxxxx

Default region name [None]: ap-northeast-1

Default output format [None]: json

[root@localhost ~]#

 

配置kubectl

aws eks update-kubeconfig --name rancher-eks

6. 等待运行结束之后, 拷贝以下内容到 auth.yaml文件

kubectl apply -f auth.yaml

20190309095057

 

apiVersion: v1
kind: ConfigMap
metadata:
  name: aws-auth
  namespace: kube-system
data:
  mapRoles: |
    - rolearn: arn:aws:iam::174441739162:role/rancher-eks-node
      username: system:node:{{EC2PrivateDNSName}}
      groups:
        - system:bootstrappers
        - system:nodes

 

7. 安装 helm

curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get | bash

helm completion bash > .helmrc && echo "source .helmrc" >> .bashrc

初始化 helm

helm init

可以看到, init 命令会在 cluster 上安装 tiller 服务

20190309101425

 

8. 安装rancher

添加仓库

helm repo add rancher-stable https://releases.rancher.com/server-charts/stable

给tiller添加权限

kubectl create serviceaccount --namespace kube-system tiller

kubectl create clusterrolebinding tiller-cluster-rule --clusterrole=cluster-admin --serviceaccount=kube-system:tiller

kubectl patch deploy --namespace kube-system tiller-deploy -p '{"spec":{"template":{"spec":{"serviceAccount":"tiller"}}}}'

安装 cert-manager

helm install stable/cert-manager  --name cert-manager  --namespace kube-system  --version v0.5.2

检查状态

kubectl -n kube-system rollout status deploy/cert-manager

9. 安装 ingress

查看一下创建的ec2 的ip  3.112.68.249, 18.182.56.253

helm install --name nginx-ingress --set "rabc.create=true,controller.service.externalIPs[0]=3.112.68.249,controller.service.externalIPs[1]=18.182.56.253" stable/nginx-ingress

创建成功之后会生成对应的 ELB DNS

20190309103036

20190309103357

10. 使用上面的 dns 地址安装 rancher

[root@ranchertarget terraform_workspace]# helm install rancher-stable/rancher  --name rancher  --namespace cattle-system --set hostname=ab2df1f92427711e9af8406b2c568707-1550919181.ap-northeast-1.elb.amazonaws.com

 

20190309103702

 

11 安装完成

20190309103907

20190309104240