kubernetes用StatefulSet组建zookeeper集群

  运维

官方地址
yaml稍作修改:

apiVersion: v1
kind: Service
metadata:
  name: zk-hs
  labels:
    app: zk
spec:
  ports:
  - port: 2888
    name: server
  - port: 3888
    name: leader-election
  clusterIP: None
  selector:
    app: zk
---
apiVersion: v1
kind: Service
metadata:
  name: zk-cs
  labels:
    app: zk
spec:
  ports:
  - port: 2181
    name: client
  selector:
    app: zk
---
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
metadata:
  name: zk-pdb
spec:
  selector:
    matchLabels:
      app: zk
  maxUnavailable: 1
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: zk
spec:
  selector:
    matchLabels:
      app: zk
  serviceName: zk-hs
  replicas: 3
  updateStrategy:
    type: RollingUpdate
  podManagementPolicy: Parallel
  template:
    metadata:
      labels:
        app: zk
    spec:
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            - labelSelector:
                matchExpressions:
                  - key: "app"
                    operator: In
                    values:
                    - zk
              topologyKey: "kubernetes.io/hostname"
      containers:
      - name: kubernetes-zookeeper
        imagePullPolicy: Always
        #此处更新为国内源
        image: "mirrorgooglecontainers/kubernetes-zookeeper:1.0-3.4.10"
        resources:
          requests:
            memory: "1Gi"
            cpu: "0.5"
        ports:
        - containerPort: 2181
          name: client
        - containerPort: 2888
          name: server
        - containerPort: 3888
          name: leader-election
        command:
        - sh
        - -c
        - "start-zookeeper \
          --servers=3 \
          --data_dir=/var/lib/zookeeper/data \
          --data_log_dir=/var/lib/zookeeper/data/log \
          --conf_dir=/opt/zookeeper/conf \
          --client_port=2181 \
          --election_port=3888 \
          --server_port=2888 \
          --tick_time=2000 \
          --init_limit=10 \
          --sync_limit=5 \
          --heap=512M \
          --max_client_cnxns=60 \
          --snap_retain_count=3 \
          --purge_interval=12 \
          --max_session_timeout=40000 \
          --min_session_timeout=4000 \
          --log_level=INFO"
        readinessProbe:
          exec:
            command:
            - sh
            - -c
            - "zookeeper-ready 2181"
          initialDelaySeconds: 10
          timeoutSeconds: 5
        livenessProbe:
          exec:
            command:
            - sh
            - -c
            - "zookeeper-ready 2181"
          initialDelaySeconds: 10
          timeoutSeconds: 5
        volumeMounts:
        - name: datadir
          mountPath: /var/lib/zookeeper
      securityContext:
        runAsUser: 1000
        fsGroup: 1000
  volumeClaimTemplates:
  - metadata:
      name: datadir
    spec:
      accessModes: [ "ReadWriteOnce" ]
      resources:
        requests:
          storage: 10Gi
      #使用storageclass
      volumeName:
      storageClassName: nfs

镜像内启动脚本:

#!/usr/bin/env bash
# Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#
#Usage: start-zookeeper [OPTIONS]
# Starts a ZooKeeper server based on the supplied options.
#     --servers           The number of servers in the ensemble. The default
#                         value is 1.

#     --data_dir          The directory where the ZooKeeper process will store its
#                         snapshots. The default is /var/lib/zookeeper/data.

#     --data_log_dir      The directory where the ZooKeeper process will store its
#                         write ahead log. The default is
#                         /var/lib/zookeeper/data/log.

#     --conf_dir          The directoyr where the ZooKeeper process will store its
#                         configuration. The default is /opt/zookeeper/conf.

#     --client_port       The port on which the ZooKeeper process will listen for
#                         client requests. The default is 2181.

#     --election_port     The port on which the ZooKeeper process will perform
#                         leader election. The default is 3888.

#     --server_port       The port on which the ZooKeeper process will listen for
#                         requests from other servers in the ensemble. The
#                         default is 2888.

#     --tick_time         The length of a ZooKeeper tick in ms. The default is
#                         2000.

#     --init_limit        The number of Ticks that an ensemble member is allowed
#                         to perform leader election. The default is 10.

#     --sync_limit        The maximum session timeout that the ensemble will
#                         allows a client to request. The default is 5.

#     --heap              The maximum amount of heap to use. The format is the
#                         same as that used for the Xmx and Xms parameters to the
#                         JVM. e.g. --heap=2G. The default is 2G.

#     --max_client_cnxns  The maximum number of client connections that the
#                         ZooKeeper process will accept simultaneously. The
#                         default is 60.

#     --snap_retain_count The maximum number of snapshots the ZooKeeper process
#                         will retain if purge_interval is greater than 0. The
#                         default is 3.

#     --purge_interval    The number of hours the ZooKeeper process will wait
#                         between purging its old snapshots. If set to 0 old
#                         snapshots will never be purged. The default is 0.

#     --max_session_timeout The maximum time in milliseconds for a client session
#                         timeout. The default value is 2 * tick time.

#     --min_session_timeout The minimum time in milliseconds for a client session
#                         timeout. The default value is 20 * tick time.

#     --log_level         The log level for the zookeeeper server. Either FATAL,
#                         ERROR, WARN, INFO, DEBUG. The default is INFO.


USER=`whoami`
HOST=`hostname -s`
DOMAIN=`hostname -d`
LOG_LEVEL=INFO
DATA_DIR="/var/lib/zookeeper/data"
DATA_LOG_DIR="/var/lib/zookeeper/log"
LOG_DIR="/var/log/zookeeper"
CONF_DIR="/opt/zookeeper/conf"
CLIENT_PORT=2181
SERVER_PORT=2888
ELECTION_PORT=3888
TICK_TIME=2000
INIT_LIMIT=10
SYNC_LIMIT=5
HEAP=2G
MAX_CLIENT_CNXNS=60
SNAP_RETAIN_COUNT=3
PURGE_INTERVAL=0
SERVERS=1

function print_usage() {
echo "\
Usage: start-zookeeper [OPTIONS]
Starts a ZooKeeper server based on the supplied options.
    --servers           The number of servers in the ensemble. The default
                        value is 1.

    --data_dir          The directory where the ZooKeeper process will store its
                        snapshots. The default is /var/lib/zookeeper/data.

    --data_log_dir      The directory where the ZooKeeper process will store its
                        write ahead log. The default is
                        /var/lib/zookeeper/data/log.

    --conf_dir          The directoyr where the ZooKeeper process will store its
                        configuration. The default is /opt/zookeeper/conf.

    --client_port       The port on which the ZooKeeper process will listen for
                        client requests. The default is 2181.

    --election_port     The port on which the ZooKeeper process will perform
                        leader election. The default is 3888.

    --server_port       The port on which the ZooKeeper process will listen for
                        requests from other servers in the ensemble. The
                        default is 2888.

    --tick_time         The length of a ZooKeeper tick in ms. The default is
                        2000.

    --init_limit        The number of Ticks that an ensemble member is allowed
                        to perform leader election. The default is 10.

    --sync_limit        The maximum session timeout that the ensemble will
                        allows a client to request. The default is 5.

    --heap              The maximum amount of heap to use. The format is the
                        same as that used for the Xmx and Xms parameters to the
                        JVM. e.g. --heap=2G. The default is 2G.

    --max_client_cnxns  The maximum number of client connections that the
                        ZooKeeper process will accept simultaneously. The
                        default is 60.

    --snap_retain_count The maximum number of snapshots the ZooKeeper process
                        will retain if purge_interval is greater than 0. The
                        default is 3.

    --purge_interval    The number of hours the ZooKeeper process will wait
                        between purging its old snapshots. If set to 0 old
                        snapshots will never be purged. The default is 0.

    --max_session_timeout The maximum time in milliseconds for a client session
                        timeout. The default value is 2 * tick time.

    --min_session_timeout The minimum time in milliseconds for a client session
                        timeout. The default value is 20 * tick time.

    --log_level         The log level for the zookeeeper server. Either FATAL,
                        ERROR, WARN, INFO, DEBUG. The default is INFO.
"
}

function create_data_dirs() {
    if [ ! -d $DATA_DIR  ]; then
        mkdir -p $DATA_DIR
        chown -R $USER:$USER $DATA_DIR
    fi

    if [ ! -d $DATA_LOG_DIR  ]; then
        mkdir -p $DATA_LOG_DIR
        chown -R $USER:USER $DATA_LOG_DIR
    fi

    if [ ! -d $LOG_DIR  ]; then
        mkdir -p $LOG_DIR
        chown -R $USER:$USER $LOG_DIR
    fi
    if [ ! -f $ID_FILE ] && [ $SERVERS -gt 1 ]; then
        echo $MY_ID >> $ID_FILE
    fi
}

function print_servers() {
    for (( i=1; i<=$SERVERS; i++ ))
    do
        echo "server.$i=$NAME-$((i-1)).$DOMAIN:$SERVER_PORT:$ELECTION_PORT"
    done
}

function create_config() {
    rm -f $CONFIG_FILE
    echo "#This file was autogenerated DO NOT EDIT" >> $CONFIG_FILE
    echo "clientPort=$CLIENT_PORT" >> $CONFIG_FILE
    echo "dataDir=$DATA_DIR" >> $CONFIG_FILE
    echo "dataLogDir=$DATA_LOG_DIR" >> $CONFIG_FILE
    echo "tickTime=$TICK_TIME" >> $CONFIG_FILE
    echo "initLimit=$INIT_LIMIT" >> $CONFIG_FILE
    echo "syncLimit=$SYNC_LIMIT" >> $CONFIG_FILE
    echo "maxClientCnxns=$MAX_CLIENT_CNXNS" >> $CONFIG_FILE
    echo "minSessionTimeout=$MIN_SESSION_TIMEOUT" >> $CONFIG_FILE
    echo "maxSessionTimeout=$MAX_SESSION_TIMEOUT" >> $CONFIG_FILE
    echo "autopurge.snapRetainCount=$SNAP_RETAIN_COUNT" >> $CONFIG_FILE
    echo "autopurge.purgeInteval=$PURGE_INTERVAL" >> $CONFIG_FILE
     if [ $SERVERS -gt 1 ]; then
        print_servers >> $CONFIG_FILE
    fi
    cat $CONFIG_FILE >&2
}

function create_jvm_props() {
    rm -f $JAVA_ENV_FILE
    echo "ZOO_LOG_DIR=$LOG_DIR" >> $JAVA_ENV_FILE
    echo "JVMFLAGS=\"-Xmx$HEAP -Xms$HEAP\"" >> $JAVA_ENV_FILE
}

function create_log_props() {
    rm -f $LOGGER_PROPS_FILE
    echo "Creating ZooKeeper log4j configuration"
    echo "zookeeper.root.logger=CONSOLE" >> $LOGGER_PROPS_FILE
    echo "zookeeper.console.threshold="$LOG_LEVEL >> $LOGGER_PROPS_FILE
    echo "log4j.rootLogger=\${zookeeper.root.logger}" >> $LOGGER_PROPS_FILE
    echo "log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender" >> $LOGGER_PROPS_FILE
    echo "log4j.appender.CONSOLE.Threshold=\${zookeeper.console.threshold}" >> $LOGGER_PROPS_FILE
    echo "log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout" >> $LOGGER_PROPS_FILE
    echo "log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n" >> $LOGGER_PROPS_FILE
}

optspec=":hv-:"
while getopts "$optspec" optchar; do

    case "${optchar}" in
        -)
            case "${OPTARG}" in
                servers=*)
                    SERVERS=${OPTARG##*=}
                    ;;
                data_dir=*)
                    DATA_DIR=${OPTARG##*=}
                    ;;
                data_log_dir=*)
                    DATA_LOG_DIR=${OPTARG##*=}
                    ;;
                log_dir=*)
                    LOG_DIR=${OPTARG##*=}
                    ;;
                conf_dir=*)
                    CONF_DIR=${OPTARG##*=}
                    ;;
                client_port=*)
                    CLIENT_PORT=${OPTARG##*=}
                    ;;
                election_port=*)
                    ELECTION_PORT=${OPTARG##*=}
                    ;;
                server_port=*)
                    SERVER_PORT=${OPTARG##*=}
                    ;;
                tick_time=*)
                    TICK_TIME=${OPTARG##*=}
                    ;;
                init_limit=*)
                    INIT_LIMIT=${OPTARG##*=}
                    ;;
                sync_limit=*)
                    SYNC_LIMIT=${OPTARG##*=}
                    ;;
                heap=*)
                    HEAP=${OPTARG##*=}
                    ;;
                max_client_cnxns=*)
                    MAX_CLIENT_CNXNS=${OPTARG##*=}
                    ;;
                snap_retain_count=*)
                    SNAP_RETAIN_COUNT=${OPTARG##*=}
                    ;;
                purge_interval=*)
                    PURGE_INTERVAL=${OPTARG##*=}
                    ;;
                max_session_timeout=*)
                    MAX_SESSION_TIMEOUT=${OPTARG##*=}
                    ;;
                min_session_timeout=*)
                    MIN_SESSION_TIMEOUT=${OPTARG##*=}
                    ;;
                log_level=*)
                    LOG_LEVEL=${OPTARG##*=}
                    ;;
                *)
                    echo "Unknown option --${OPTARG}" >&2
                    exit 1
                    ;;
            esac;;
        h)
            print_usage
            exit
            ;;
        v)
            echo "Parsing option: '-${optchar}'" >&2
            ;;
        *)
            if [ "$OPTERR" != 1 ] || [ "${optspec:0:1}" = ":" ]; then
                echo "Non-option argument: '-${OPTARG}'" >&2
            fi
            ;;
    esac
done

MIN_SESSION_TIMEOUT=${MIN_SESSION_TIMEOUT:- $((TICK_TIME*2))}
MAX_SESSION_TIMEOUT=${MAX_SESSION_TIMEOUT:- $((TICK_TIME*20))}
ID_FILE="$DATA_DIR/myid"
CONFIG_FILE="$CONF_DIR/zoo.cfg"
LOGGER_PROPS_FILE="$CONF_DIR/log4j.properties"
JAVA_ENV_FILE="$CONF_DIR/java.env"
if [[ $HOST =~ (.*)-([0-9]+)$ ]]; then
    NAME=${BASH_REMATCH[1]}
    ORD=${BASH_REMATCH[2]}
else
    echo "Fialed to parse name and ordinal of Pod"
    exit 1
fi

MY_ID=$((ORD+1))

create_config && create_jvm_props && create_log_props && create_data_dirs && exec zkServer.sh start-foreground

通过主机名的数字来产生myid,生成的配置文件如下:

#This file was autogenerated DO NOT EDIT
clientPort=2181
dataDir=/var/lib/zookeeper/data
dataLogDir=/var/lib/zookeeper/data/log
tickTime=2000
initLimit=10
syncLimit=5
maxClientCnxns=60
minSessionTimeout=4000
maxSessionTimeout=40000
autopurge.snapRetainCount=3
autopurge.purgeInteval=12
server.1=zk-0.zk-hs.test6.svc.cluster.local:2888:3888
server.2=zk-1.zk-hs.test6.svc.cluster.local:2888:3888
server.3=zk-2.zk-hs.test6.svc.cluster.local:2888:3888

One Reply to “kubernetes用StatefulSet组建zookeeper集群”

  • 只有zk不需要返回主节点地址 servie随便转达到任何一个节点 都能处理:
    1.在Client向Follwer发出一个写的请求
    2.Follwer把请求发送给Leader
    3.Leader接收到以后开始发起投票并通知Follwer进行投票
    4.Follwer把投票结果发送给Leader
    5.Leader将结果汇总后如果需要写入,则开始写入同时把写入操作通知给
    Leader,然后commit;
    6.Follwer把请求结果返回给Client

LEAVE A COMMENT

发表评论前,请选对水果【Apple=苹果、Banana=香蕉、Pineapple=菠萝、Grapes=葡萄】 Grapes...