heat_template_version: 2014-10-16

description: >
  This template will boot a Kubernetes cluster with one or more
  minions (as specified by the number_of_minions parameter, which
  defaults to 1).

parameters:

  ssh_key_name:
    type: string
    description: name of ssh key to be provisioned on our server

  external_network:
    type: string
    description: uuid/name of a network to use for floating ip addresses
    default: public

  fixed_network:
    type: string
    description: uuid/name of an existing network to use to provision machines
    default: ""

  fixed_subnet:
    type: string
    description: uuid/name of an existing subnet to use to provision machines
    default: ""

  server_image:
    type: string
    description: glance image used to boot the server

  master_flavor:
    type: string
    default: m1.small
    description: flavor to use when booting the server for master nodes

  minion_flavor:
    type: string
    default: m1.small
    description: flavor to use when booting the server for minions

  prometheus_monitoring:
    type: boolean
    default: false
    description: >
      whether or not to have the grafana-prometheus-cadvisor monitoring setup

  grafana_admin_passwd:
    type: string
    default: admin
    hidden: true
    description: >
      admin user password for the Grafana monitoring interface

  dns_nameserver:
    type: string
    description: address of a DNS nameserver reachable in your environment
    default: 8.8.8.8

  number_of_masters:
    type: number
    description: how many kubernetes masters to spawn
    default: 1

  number_of_minions:
    type: number
    description: how many kubernetes minions to spawn
    default: 1

  fixed_network_cidr:
    type: string
    description: network range for fixed ip network
    default: 10.0.0.0/24

  portal_network_cidr:
    type: string
    description: >
      address range used by kubernetes for service portals
    default: 10.254.0.0/16

  network_driver:
    type: string
    description: network driver to use for instantiating container networks
    default: flannel

  flannel_network_cidr:
    type: string
    description: network range for flannel overlay network
    default: 10.100.0.0/16

  flannel_network_subnetlen:
    type: number
    description: size of subnet assigned to each minion
    default: 24

  flannel_backend:
    type: string
    description: >
      specify the backend for flannel, default udp backend
    default: "udp"
    constraints:
      - allowed_values: ["udp", "vxlan", "host-gw"]

  system_pods_initial_delay:
    type: number
    description: >
      health check, time to wait for system pods (podmaster, scheduler) to boot
      (in seconds)
    default: 30

  system_pods_timeout:
    type: number
    description: >
      health check, timeout for system pods (podmaster, scheduler) to answer.
      (in seconds)
    default: 5

  admission_control_list:
    type: string
    description: >
      List of admission control plugins to activate
    default: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"

  kube_allow_priv:
    type: string
    description: >
      whether or not kubernetes should permit privileged containers.
    default: "true"
    constraints:
      - allowed_values: ["true", "false"]

  etcd_volume_size:
    type: number
    description: >
      size of the cinder volume for etcd storage
    default: 0

  docker_volume_size:
    type: number
    description: >
      size of a cinder volume to allocate to docker for container/image
      storage
    default: 0

  docker_volume_type:
    type: string
    description: >
      type of a cinder volume to allocate to docker for container/image
      storage

  docker_storage_driver:
    type: string
    description: docker storage driver name
    default: "devicemapper"

  cgroup_driver:
    type: string
    description: >
      cgroup driver name that kubelet should use, ideally the same as
      the docker cgroup driver.
    default: "systemd"

  wait_condition_timeout:
    type: number
    description: >
      timeout for the Wait Conditions
    default: 6000

  minions_to_remove:
    type: comma_delimited_list
    description: >
      List of minions to be removed when doing an update. Individual minion may
      be referenced several ways: (1) The resource name (e.g. ['1', '3']),
      (2) The private IP address ['10.0.0.4', '10.0.0.6']. Note: the list should
      be empty when doing an create.
    default: []

  discovery_url:
    type: string
    description: >
      Discovery URL used for bootstrapping the etcd cluster.

  registry_enabled:
    type: boolean
    description: >
      Indicates whether the docker registry is enabled.
    default: false

  registry_port:
    type: number
    description: port of registry service
    default: 5000

  swift_region:
    type: string
    description: region of swift service
    default: ""

  registry_container:
    type: string
    description: >
      name of swift container which docker registry stores images in
    default: "container"

  registry_insecure:
    type: boolean
    description: >
      indicates whether to skip TLS verification between registry and backend storage
    default: true

  registry_chunksize:
    type: number
    description: >
      size fo the data segments for the swift dynamic large objects
    default: 5242880

  volume_driver:
    type: string
    description: volume driver to use for container storage
    default: ""

  region_name:
    type: string
    description: A logically separate section of the cluster

  username:
    type: string
    description: >
      user account

  password:
    type: string
    description: >
      user password, not set in current implementation, only used to
      fill in for Kubernetes config file
    default:
      ChangeMe
    hidden: true

  loadbalancing_protocol:
    type: string
    description: >
      The protocol which is used for load balancing. If you want to change
      tls_disabled option to 'True', please change this to "HTTP".
    default: TCP
    constraints:
      - allowed_values: ["TCP", "HTTP"]

  tls_disabled:
    type: boolean
    description: whether or not to disable TLS
    default: False

  kube_dashboard_enabled:
    type: boolean
    description: whether or not to enable kubernetes dashboard
    default: True

  influx_grafana_dashboard_enabled:
    type: boolean
    description: Enable influxdb with grafana dashboard for data from heapster
    default: False

  verify_ca:
    type: boolean
    description: whether or not to validate certificate authority

  kubernetes_port:
    type: number
    description: >
      The port which are used by kube-apiserver to provide Kubernetes
      service.
    default: 6443

  cluster_uuid:
    type: string
    description: identifier for the cluster this template is generating

  magnum_url:
    type: string
    description: endpoint to retrieve TLS certs from

  http_proxy:
    type: string
    description: http proxy address for docker
    default: ""

  https_proxy:
    type: string
    description: https proxy address for docker
    default: ""

  no_proxy:
    type: string
    description: no proxies for docker
    default: ""

  trustee_domain_id:
    type: string
    description: domain id of the trustee

  trustee_user_id:
    type: string
    description: user id of the trustee

  trustee_username:
    type: string
    description: username of the trustee

  trustee_password:
    type: string
    description: password of the trustee
    hidden: true

  trust_id:
    type: string
    description: id of the trust which is used by the trustee
    hidden: true

  auth_url:
    type: string
    description: url for keystone

  kube_tag:
    type: string
    description: tag of the k8s containers used to provision the kubernetes cluster
    default: v1.9.3

  etcd_tag:
    type: string
    description: tag of the etcd system container
    default: v3.2.7

  flannel_tag:
    type: string
    description: tag of the flannel system containers
    default: v0.9.0

  kube_version:
    type: string
    description: version of kubernetes used for kubernetes cluster
    default: v1.9.3

  kube_dashboard_version:
    type: string
    description: version of kubernetes dashboard used for kubernetes cluster
    default: v1.8.3

  insecure_registry_url:
    type: string
    description: insecure registry url
    default: ""

  container_infra_prefix:
    type: string
    description: >
      prefix of container images used in the cluster, kubernetes components,
      kubernetes-dashboard, coredns etc
    constraints:
      - allowed_pattern: "^$|.*/"
    default: ""

  dns_service_ip:
    type: string
    description: >
      address used by Kubernetes DNS service
    default: 10.254.0.10

  dns_cluster_domain:
    type: string
    description: >
      domain name for cluster DNS
    default: "cluster.local"

  openstack_ca:
    type: string
    hidden: true
    description: The OpenStack CA certificate to install on the node.

  nodes_affinity_policy:
    type: string
    description: >
      affinity policy for nodes server group
    constraints:
      - allowed_values: ["affinity", "anti-affinity", "soft-affinity",
                         "soft-anti-affinity"]

  availability_zone:
    type: string
    description: >
      availability zone for master and nodes
    default: ""

  cert_manager_api:
    type: boolean
    description: true if the kubernetes cert api manager should be enabled
    default: false

  ca_key:
    type: string
    description: key of internal ca for the kube certificate api manager
    default: ""
    hidden: true

  calico_tag:
    type: string
    description: tag of the calico containers used to provision the calico node
    default: v2.6.7

  calico_cni_tag:
    type: string
    description: tag of the cni used to provision the calico node
    default: v1.11.2

  calico_kube_controllers_tag:
    type: string
    description: tag of the kube_controllers used to provision the calico node
    default: v1.0.3

  calico_ipv4pool:
    type: string
    description: Configure the IP pool from which Pod IPs will be chosen
    default: "192.168.0.0/16"

  pods_network_cidr:
    type: string
    description: Configure the IP pool/range from which pod IPs will be chosen

  ingress_controller:
    type: string
    description: >
      ingress controller backend to use
    default: ""

  ingress_controller_role:
    type: string
    description: >
      node role where the ingress controller backend should run
    default: "ingress"

  kubelet_options:
    type: string
    description: >
      additional options to be passed to the kubelet
    default: ""

  kubeapi_options:
    type: string
    description: >
      additional options to be passed to the api
    default: ""

  kubecontroller_options:
    type: string
    description: >
      additional options to be passed to the controller manager
    default: ""

  kubeproxy_options:
    type: string
    description: >
      additional options to be passed to the kube proxy
    default: ""

  kubescheduler_options:
    type: string
    description: >
      additional options to be passed to the scheduler
    default: ""

  octavia_enabled:
    type: boolean
    description: >
      whether or not to use Octavia for LoadBalancer type service.
    default: False

resources:

  ######################################################################
  #
  # network resources.  allocate a network and router for our server.
  # Important: the Load Balancer feature in Kubernetes requires that
  # the name for the fixed_network must be "private" for the
  # address lookup in Kubernetes to work properly
  #

  network:
    type: ../../common/templates/network.yaml
    properties:
      existing_network: {get_param: fixed_network}
      existing_subnet: {get_param: fixed_subnet}
      private_network_cidr: {get_param: fixed_network_cidr}
      dns_nameserver: {get_param: dns_nameserver}
      external_network: {get_param: external_network}
      private_network_name: private

  api_lb:
    type: ../../common/templates/lb.yaml
    properties:
      fixed_subnet: {get_attr: [network, fixed_subnet]}
      external_network: {get_param: external_network}
      protocol: {get_param: loadbalancing_protocol}
      port: {get_param: kubernetes_port}

  etcd_lb:
    type: ../../common/templates/lb.yaml
    properties:
      fixed_subnet: {get_attr: [network, fixed_subnet]}
      external_network: {get_param: external_network}
      protocol: {get_param: loadbalancing_protocol}
      port: 2379

  ######################################################################
  #
  # security groups.  we need to permit network traffic of various
  # sorts.
  #

  secgroup_kube_master:
    type: OS::Neutron::SecurityGroup
    properties:
      rules:
        - protocol: icmp
        - protocol: tcp
          port_range_min: 22
          port_range_max: 22
        - protocol: tcp
          port_range_min: 7080
          port_range_max: 7080
        - protocol: tcp
          port_range_min: 8080
          port_range_max: 8080
        - protocol: tcp
          port_range_min: 2379
          port_range_max: 2379
        - protocol: tcp
          port_range_min: 2380
          port_range_max: 2380
        - protocol: tcp
          port_range_min: 6443
          port_range_max: 6443
        - protocol: tcp
          port_range_min: 10250
          port_range_max: 10250
        - protocol: tcp
          port_range_min: 30000
          port_range_max: 32767

  secgroup_kube_minion:
    type: OS::Neutron::SecurityGroup
    properties:
      rules:
        - protocol: icmp
        - protocol: tcp
        - protocol: udp

  ######################################################################
  #
  # resources that expose the IPs of either the kube master or a given
  # LBaaS pool depending on whether LBaaS is enabled for the cluster.
  #

  api_address_lb_switch:
    type: Magnum::ApiGatewaySwitcher
    properties:
      pool_public_ip: {get_attr: [api_lb, floating_address]}
      pool_private_ip: {get_attr: [api_lb, address]}
      master_public_ip: {get_attr: [kube_masters, resource.0.kube_master_external_ip]}
      master_private_ip: {get_attr: [kube_masters, resource.0.kube_master_ip]}

  etcd_address_lb_switch:
    type: Magnum::ApiGatewaySwitcher
    properties:
      pool_private_ip: {get_attr: [etcd_lb, address]}
      master_private_ip: {get_attr: [kube_masters, resource.0.kube_master_ip]}

  ######################################################################
  #
  # resources that expose the IPs of either floating ip or a given
  # fixed ip depending on whether FloatingIP is enabled for the cluster.
  #

  api_address_floating_switch:
    type: Magnum::FloatingIPAddressSwitcher
    properties:
      public_ip: {get_attr: [api_address_lb_switch, public_ip]}
      private_ip: {get_attr: [api_address_lb_switch, private_ip]}

  ######################################################################
  #
  # resources that expose the server group for all nodes include master
  # and minions.
  #

  nodes_server_group:
    type: OS::Nova::ServerGroup
    properties:
      policies: [{get_param: nodes_affinity_policy}]

  ######################################################################
  #
  # kubernetes masters. This is a resource group that will create
  # <number_of_masters> masters.
  #

  kube_masters:
    type: OS::Heat::ResourceGroup
    depends_on:
      - network
    properties:
      count: {get_param: number_of_masters}
      resource_def:
        type: kubemaster.yaml
        properties:
          name:
            list_join:
              - '-'
              - [{ get_param: 'OS::stack_name' }, 'master', '%index%']
          master_index: '%index%'
          prometheus_monitoring: {get_param: prometheus_monitoring}
          grafana_admin_passwd: {get_param: grafana_admin_passwd}
          api_public_address: {get_attr: [api_lb, floating_address]}
          api_private_address: {get_attr: [api_lb, address]}
          ssh_key_name: {get_param: ssh_key_name}
          server_image: {get_param: server_image}
          master_flavor: {get_param: master_flavor}
          external_network: {get_param: external_network}
          kube_allow_priv: {get_param: kube_allow_priv}
          etcd_volume_size: {get_param: etcd_volume_size}
          docker_volume_size: {get_param: docker_volume_size}
          docker_volume_type: {get_param: docker_volume_type}
          docker_storage_driver: {get_param: docker_storage_driver}
          cgroup_driver: {get_param: cgroup_driver}
          wait_condition_timeout: {get_param: wait_condition_timeout}
          network_driver: {get_param: network_driver}
          flannel_network_cidr: {get_param: flannel_network_cidr}
          flannel_network_subnetlen: {get_param: flannel_network_subnetlen}
          flannel_backend: {get_param: flannel_backend}
          system_pods_initial_delay: {get_param: system_pods_initial_delay}
          system_pods_timeout: {get_param: system_pods_timeout}
          portal_network_cidr: {get_param: portal_network_cidr}
          admission_control_list: {get_param: admission_control_list}
          discovery_url: {get_param: discovery_url}
          cluster_uuid: {get_param: cluster_uuid}
          magnum_url: {get_param: magnum_url}
          volume_driver: {get_param: volume_driver}
          fixed_network: {get_attr: [network, fixed_network]}
          fixed_subnet: {get_attr: [network, fixed_subnet]}
          api_pool_id: {get_attr: [api_lb, pool_id]}
          etcd_pool_id: {get_attr: [etcd_lb, pool_id]}
          username: {get_param: username}
          password: {get_param: password}
          kubernetes_port: {get_param: kubernetes_port}
          tls_disabled: {get_param: tls_disabled}
          kube_dashboard_enabled: {get_param: kube_dashboard_enabled}
          influx_grafana_dashboard_enabled: {get_param: influx_grafana_dashboard_enabled}
          verify_ca: {get_param: verify_ca}
          secgroup_kube_master_id: {get_resource: secgroup_kube_master}
          http_proxy: {get_param: http_proxy}
          https_proxy: {get_param: https_proxy}
          no_proxy: {get_param: no_proxy}
          kube_tag: {get_param: kube_tag}
          kube_version: {get_param: kube_version}
          etcd_tag: {get_param: etcd_tag}
          flannel_tag: {get_param: flannel_tag}
          kube_dashboard_version: {get_param: kube_dashboard_version}
          trustee_user_id: {get_param: trustee_user_id}
          trustee_password: {get_param: trustee_password}
          trust_id: {get_param: trust_id}
          auth_url: {get_param: auth_url}
          insecure_registry_url: {get_param: insecure_registry_url}
          container_infra_prefix: {get_param: container_infra_prefix}
          etcd_lb_vip: {get_attr: [etcd_lb, address]}
          dns_service_ip: {get_param: dns_service_ip}
          dns_cluster_domain: {get_param: dns_cluster_domain}
          openstack_ca: {get_param: openstack_ca}
          nodes_server_group_id: {get_resource: nodes_server_group}
          availability_zone: {get_param: availability_zone}
          ca_key: {get_param: ca_key}
          cert_manager_api: {get_param: cert_manager_api}
          calico_tag: {get_param: calico_tag}
          calico_cni_tag: {get_param: calico_cni_tag}
          calico_kube_controllers_tag: {get_param: calico_kube_controllers_tag}
          calico_ipv4pool: {get_param: calico_ipv4pool}
          pods_network_cidr: {get_param: pods_network_cidr}
          ingress_controller: {get_param: ingress_controller}
          ingress_controller_role: {get_param: ingress_controller_role}
          kubelet_options: {get_param: kubelet_options}
          kubeapi_options: {get_param: kubeapi_options}
          kubeproxy_options: {get_param: kubeproxy_options}
          kubecontroller_options: {get_param: kubecontroller_options}
          kubescheduler_options: {get_param: kubescheduler_options}
          octavia_enabled: {get_param: octavia_enabled}

  ######################################################################
  #
  # kubernetes minions. This is an resource group that will initially
  # create <number_of_minions> minions, and needs to be manually scaled.
  #

  kube_minions:
    type: OS::Heat::ResourceGroup
    depends_on:
      - network
    properties:
      count: {get_param: number_of_minions}
      removal_policies: [{resource_list: {get_param: minions_to_remove}}]
      resource_def:
        type: kubeminion.yaml
        properties:
          name:
            list_join:
              - '-'
              - [{ get_param: 'OS::stack_name' }, 'minion', '%index%']
          prometheus_monitoring: {get_param: prometheus_monitoring}
          ssh_key_name: {get_param: ssh_key_name}
          server_image: {get_param: server_image}
          minion_flavor: {get_param: minion_flavor}
          fixed_network: {get_attr: [network, fixed_network]}
          fixed_subnet: {get_attr: [network, fixed_subnet]}
          network_driver: {get_param: network_driver}
          flannel_network_cidr: {get_param: flannel_network_cidr}
          kube_master_ip: {get_attr: [api_address_lb_switch, private_ip]}
          etcd_server_ip: {get_attr: [etcd_address_lb_switch, private_ip]}
          external_network: {get_param: external_network}
          kube_allow_priv: {get_param: kube_allow_priv}
          docker_volume_size: {get_param: docker_volume_size}
          docker_volume_type: {get_param: docker_volume_type}
          docker_storage_driver: {get_param: docker_storage_driver}
          cgroup_driver: {get_param: cgroup_driver}
          wait_condition_timeout: {get_param: wait_condition_timeout}
          registry_enabled: {get_param: registry_enabled}
          registry_port: {get_param: registry_port}
          swift_region: {get_param: swift_region}
          registry_container: {get_param: registry_container}
          registry_insecure: {get_param: registry_insecure}
          registry_chunksize: {get_param: registry_chunksize}
          cluster_uuid: {get_param: cluster_uuid}
          magnum_url: {get_param: magnum_url}
          volume_driver: {get_param: volume_driver}
          region_name: {get_param: region_name}
          auth_url: {get_param: auth_url}
          username: {get_param: username}
          password: {get_param: password}
          kubernetes_port: {get_param: kubernetes_port}
          tls_disabled: {get_param: tls_disabled}
          verify_ca: {get_param: verify_ca}
          secgroup_kube_minion_id: {get_resource: secgroup_kube_minion}
          http_proxy: {get_param: http_proxy}
          https_proxy: {get_param: https_proxy}
          no_proxy: {get_param: no_proxy}
          kube_tag: {get_param: kube_tag}
          kube_version: {get_param: kube_version}
          flannel_tag: {get_param: flannel_tag}
          trustee_user_id: {get_param: trustee_user_id}
          trustee_username: {get_param: trustee_username}
          trustee_password: {get_param: trustee_password}
          trustee_domain_id: {get_param: trustee_domain_id}
          trust_id: {get_param: trust_id}
          insecure_registry_url: {get_param: insecure_registry_url}
          container_infra_prefix: {get_param: container_infra_prefix}
          dns_service_ip: {get_param: dns_service_ip}
          dns_cluster_domain: {get_param: dns_cluster_domain}
          openstack_ca: {get_param: openstack_ca}
          nodes_server_group_id: {get_resource: nodes_server_group}
          availability_zone: {get_param: availability_zone}
          pods_network_cidr: {get_param: pods_network_cidr}
          kubelet_options: {get_param: kubelet_options}
          kubeproxy_options: {get_param: kubeproxy_options}
          octavia_enabled: {get_param: octavia_enabled}

outputs:

  api_address:
    value:
      str_replace:
        template: api_ip_address
        params:
          api_ip_address: {get_attr: [api_address_floating_switch, ip_address]}
    description: >
      This is the API endpoint of the Kubernetes cluster. Use this to access
      the Kubernetes API.

  registry_address:
    value:
      str_replace:
        template: localhost:port
        params:
          port: {get_param: registry_port}
    description:
      This is the url of docker registry server where you can store docker
      images.

  kube_masters_private:
    value: {get_attr: [kube_masters, kube_master_ip]}
    description: >
      This is a list of the "private" IP addresses of all the Kubernetes masters.

  kube_masters:
    value: {get_attr: [kube_masters, kube_master_external_ip]}
    description: >
      This is a list of the "public" IP addresses of all the Kubernetes masters.
      Use these IP addresses to log in to the Kubernetes masters via ssh.

  kube_minions_private:
    value: {get_attr: [kube_minions, kube_minion_ip]}
    description: >
      This is a list of the "private" IP addresses of all the Kubernetes minions.

  kube_minions:
    value: {get_attr: [kube_minions, kube_minion_external_ip]}
    description: >
      This is a list of the "public" IP addresses of all the Kubernetes minions.
      Use these IP addresses to log in to the Kubernetes minions via ssh.
