kubernetes命名空间无法删除问题

namespace删除后一直是Terminating状态:

[root@node01 ~]# kubectl get ns
NAME          STATUS        AGE
database      Active        13d
default       Active        15d
kube-public   Active        15d
kube-system   Active        15d
test          Terminating   14d
test1         Terminating   14d
test2         Terminating   14d
test3         Terminating   13d
test4         Active        11d
test5         Active        6d23h
test6         Active        6d23h
wordpress     Active        12d

[root@node01 ~]# kubectl delete ns test
Error from server (Conflict): Operation cannot be fulfilled on namespaces "test": The system is ensuring all content is removed from this namespace.  Upon completion, this namespace will automat
ically be purged by the system.

[root@node01 ~]# kubectl delete ns test --grace-period=0 --force
warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinit
ely.Error from server (Conflict): Operation cannot be fulfilled on namespaces "test": The system is ensuring all content is removed from this namespace.  Upon completi
on, this namespace will automatically be purged by the system.

#通常可以加--now 参数快速删除
[root@node01 ~]# kubectl delete ns test -now
Error from server (Conflict): Operation cannot be fulfilled on namespaces "test": The system is ensuring all content is removed from this namespace.  Upon completi
on, this namespace will automatically be purged by the system.

[root@node01 ~]# kubectl -n test get all
No resources found.

kubectl edit namespaces test

# Please edit the object below. Lines beginning with a '#' will be ignored,
# and an empty file will abort the edit. If an error occurs while saving this file will be
# reopened with the relevant failures.
#
apiVersion: v1
kind: Namespace
metadata:
  creationTimestamp: "2019-01-25T01:39:43Z"
  deletionTimestamp: "2019-01-25T08:25:46Z"
  name: test
  resourceVersion: "196448"
  selfLink: /api/v1/namespaces/test
  uid: 16927be3-2042-11e9-b729-000c2900f1ab
spec:
  finalizers:
  - kubernetes
status:
  phase: Terminating

[root@node01 ~]# kubectl get ns test -o json > test.json

{
    "apiVersion": "v1",
    "kind": "Namespace",
    "metadata": {
        "creationTimestamp": "2019-01-25T01:39:43Z",
        "deletionTimestamp": "2019-01-25T08:25:46Z",
        "name": "test",
        "resourceVersion": "196448",
        "selfLink": "/api/v1/namespaces/test",
        "uid": "16927be3-2042-11e9-b729-000c2900f1ab"
    },
    "spec": {
        "finalizers": [
            "kubernetes"
        ]
    },
    "status": {
        "phase": "Terminating"
    }
}

删除finalizers内容：

{
    "apiVersion": "v1",
    "kind": "Namespace",
    "metadata": {
        "creationTimestamp": "2019-01-25T01:39:43Z",
        "deletionTimestamp": "2019-01-25T08:25:46Z",
        "name": "test",
        "resourceVersion": "196448",
        "selfLink": "/api/v1/namespaces/test",
        "uid": "16927be3-2042-11e9-b729-000c2900f1ab"
    },
    "spec": {
        "finalizers": [
        ]
    },
    "status": {
        "phase": "Terminating"
    }
}

curl -H "Content-Type: application/json" -X PUT --data-binary @test.json http://node01:6443/api/v1/namespaces/test/finalize

pod有一段graceful termination时间
其实本质是kubelet上最终删除pod的逻辑失败或者阻塞了
restart APIserver后就没有了

有时候已经删除了，

还有原因就是etcd出现故障了，如果etcd 故障，也是好久删不掉的
解决方法：
1、检查 server 端 apiserver，controller-manager,shcduler,状态是否正常
检查被调度的那台主机的kubelet和kube-proxy状态
2、检查etcd

kubectl delete ns test -now

# Please edit the object below. Lines beginning with a '#' will be ignored,
# and an empty file will abort the edit. If an error occurs while saving this file will be
# reopened with the relevant failures.
#
apiVersion: v1
kind: Namespace
metadata:
  creationTimestamp: "2019-01-25T01:39:43Z"
  deletionTimestamp: "2019-01-25T08:25:46Z"
  name: test
  resourceVersion: "196448"
  selfLink: /api/v1/namespaces/test
  uid: 16927be3-2042-11e9-b729-000c2900f1ab
spec:
  finalizers:
  - []
status:
  phase: Terminating

操作etcd删除：

#删除default namespace下的pod名为pod-to-be-deleted-0
ETCDCTL_API=3 etcdctl del /registry/pods/default/pod-to-be-deleted-0
#删除需要删除的NAMESPACE
etcdctl del /registry/namespaces/NAMESPACENAME

用etcd删除

/usr/local/bin # etcdctl --version
etcdctl version: 3.2.24
API version: 2

/usr/local/bin # etcdctl rm /registry/namespaces/test
Error:  malformed HTTP response "\x15\x03\x01\x00\x02\x02"

2019-02-21 09:08:35.926278 W | etcdserver: read-only range request "key:\"/registry/volumeattachments\" range_end:\"/registry/volumeattachmentt\" count_only:true " with result "range_response_count:0 size:7" took too long (331.94364ms) to execute

2019-02-21 09:50:48.922883 W | etcdserver: failed to send out heartbeat on time (exceeded the 100ms timeout for 48.688027ms)
2019-02-21 09:50:48.922892 W | etcdserver: server is likely overloaded

etcd出现了性能问题，官方的解释是磁盘或cpu负载问题：
https://github.com/etcd-io/etcd/blob/master/Documentation/faq.md#what-does-the-etcd-warning-apply-entries-took-too-long-mean