暗黑模式
kubectl get
bash
kubectl get pods -n ess -o wide1
:::details Output
```
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
ess-deployment-markers-post-j86b8 0/1 Completed 0 78m 10.42.0.136 root-silo-2.localdomain <none> <none>
ess-deployment-markers-pre-nks4q 0/1 Completed 0 80m 10.42.0.128 root-silo-2.localdomain <none> <none>
ess-element-web-54694f79bf-h4lcg 1/1 Running 0 12h 10.42.0.219 root-silo-2.localdomain <none> <none>
ess-haproxy-5bdc6b59cc-gphlq 1/1 Running 0 90m 10.42.0.100 root-silo-2.localdomain <none> <none>
ess-init-secrets-vszrv 0/1 Completed 0 80m 10.42.0.129 root-silo-2.localdomain <none> <none>
ess-matrix-authentication-service-5f9d874cc8-pfcbq 1/1 Running 0 2d14h 10.42.0.211 root-silo-2.localdomain <none> <none>
ess-matrix-rtc-authorisation-service-69bb465879-2zdxn 1/1 Running 0 12h 10.42.0.234 root-silo-2.localdomain <none> <none>
ess-synapse-background-0 1/1 Running 0 80m 10.42.1.165 mtx-k3s-worker-1 <none> <none>
```
:::1
2
3
4
5
6
7
8
9
10
11
12
13
2
3
4
5
6
7
8
9
10
11
12
13
bash
kubectl get pods -n ess -o custom-columns=NAME:.metadata.name,TYPE:.metadata.ownerReferences[0].kind,COMPONENT:.metadata.labels."app\.kubernetes\.io/component"1
:::details Output
```
NAME TYPE NODE
ess-deployment-markers-post-j86b8 Job matrix-tools
ess-deployment-markers-pre-nks4q Job matrix-tools
ess-element-web-54694f79bf-h4lcg ReplicaSet matrix-client
ess-haproxy-5bdc6b59cc-gphlq ReplicaSet matrix-stack-ingress
ess-init-secrets-vszrv Job matrix-tools
ess-matrix-authentication-service-5f9d874cc8-pfcbq ReplicaSet matrix-authentication
ess-matrix-rtc-authorisation-service-69bb465879-2zdxn ReplicaSet matrix-rtc-authorisation-service
ess-synapse-background-0 StatefulSet matrix-server
ess-synapse-check-config-hook-pnvng Job matrix-server
ess-synapse-check-config-qlr8x Job matrix-server
```
:::1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
2
3
4
5
6
7
8
9
10
11
12
13
14
15
kubectl scale
bash
kubectl scale deployment/xxxx statefulset/yyyy --replicas=1 -n ess
kubectl scale statefulset yyyy --replicas=1 -n ess1
2
3
2
3
bash
kubectl scale statefulset -l app.kubernetes.io/component=matrix-server --replicas=0 -n ess1
ConfigMap
bash
kubectl -n ess exec -it ess-haproxy-5bdc6b59cc-gphlq -- cat /usr/local/etc/haproxy/haproxy.cfg
kubectl -n ess edit configmap ess-haproxy
# 注意
# 不能直接对 ReplicaSet 使用 rollout restart，会报错：
# error: replicasets.apps "ess-haproxy-xxxx" restarting is not supported
# 应该对 Deployment 使用 rollout restart，例如：
kubectl -n ess rollout restart deployment ess-haproxy ？？？？1
2
3
4
5
6
7
8
9
2
3
4
5
6
7
8
9
delete node and re-join server
bash
# on agent node
/usr/local/bin/k3s-agent-uninstall.sh
curl -sfL https://get.k3s.io | K3S_URL=https://x.x.x.x:6443 K3S_TOKEN=xxxx INSTALL_K3S_EXEC="--flannel-iface=eth1" sh -
# /var/lib/rancher/k3s/server/node-token
cat /etc/systemd/system/k3s-agent.service # 查看是否有 --flannel-iface=eth1
# new server
# install k3s from scratch ...
# synapse media created
# stop pods
# old server
# rsync -av --inplace --whole-file --progress /media/ user@target:/media/1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
HAproxy
bash
kubectl exec -n ess -it pod/ess-haproxy-7b4676595c-hz68f -- sh
echo "help" | nc 127.0.0.1 1999
echo "show info" | nc 127.0.0.1 1999 # 关注 Run_queue 队列是否积压任务
echo "show backend" | nc 127.0.0.1 1999
echo "show servers conn synapse-sliding-sync" | nc 127.0.0.1 1999
echo "show servers conn synapse-receipts-account" | nc 127.0.0.1 1999
echo "show servers conn synapse-initial-synchrotron" | nc 127.0.0.1 1999
echo "show servers conn synapse-media-repository" | nc 127.0.0.1 19991
2
3
4
5
6
7
8
9
2
3
4
5
6
7
8
9
bash
$ echo "show servers conn synapse-sliding-sync" | nc 127.0.0.1 1999
bkname/svname bkid/svid addr port - purge_delay used_cur used_max need_est unsafe_nb safe_nb idle_lim idle_cur idle_per_thr[8]
synapse-sliding-sync/sliding-sync1 16/1 10.42.2.192 8008 - 5000 69 71 71 0 2 -1 2 0 0 0 1 1 0 0 0
synapse-sliding-sync/sliding-sync2 16/2 10.42.5.184 8008 - 5000 97 100 105 0 7 -1 7 0 1 2 0 0 3 1 0
synapse-sliding-sync/sliding-sync4 16/4 10.42.0.176 8008 - 5000 62 66 78 0 16 -1 16 2 2 2 2 4 1 2 1
synapse-sliding-sync/sliding-sync5 16/5 10.42.0.174 8008 - 5000 41 42 43 0 2 -1 2 0 0 1 0 0 0 0 1
synapse-sliding-sync/sliding-sync10 16/10 10.42.7.184 8008 - 5000 77 78 84 0 7 -1 7 3 0 0 2 0 0 1 1
synapse-sliding-sync/sliding-sync11 16/11 10.42.2.189 8008 - 5000 96 98 110 0 14 -1 14 1 2 0 0 5 0 3 3
synapse-sliding-sync/sliding-sync13 16/13 10.42.7.177 8008 - 5000 136 139 147 0 11 -1 11 0 3 2 0 2 0 1 3
synapse-sliding-sync/sliding-sync15 16/15 10.42.5.190 8008 - 5000 80 80 84 0 6 -1 6 2 1 1 0 0 1 1 01
2
3
4
5
6
7
8
9
10
2
3
4
5
6
7
8
9
10
synapse metrics
bash
kubectl exec -it ess-synapse-main-0 -n ess -- curl localhost:9001/metrics | less
kubectl exec -it ess-synapse-main-0 -n ess -- curl localhost:9001/metrics > metrics.main-0.txt
kubectl exec -it ess-synapse-sliding-sync-0 -n ess -- curl localhost:9001/metrics > metrics.sliding-sync-0.txt1
2
3
2
3
synapse_http_server_responses_total{code="200",method="GET"} 488062.0
synapse_http_server_responses_total{code="200",method="POST"} 1.973239e+06
synapse_http_server_responses_total{code="401",method="POST"} 27838.0
synapse_http_server_responses_total{code="400",method="POST"} 106.0
synapse_http_server_responses_total{code="500",method="POST"} 1124.0
synapse_http_server_responses_total{code="503",method="POST"} 103.0
synapse_threadpool_working_threads{name="database-master"} 0.0
synapse_threadpool_total_threads{name="database-master"} 26.0
synapse_threadpool_min_threads{name="database-master"} 3.0
synapse_threadpool_max_threads{name="database-master"} 26.01
2
3
4
5
6
7
8
9
10
11
2
3
4
5
6
7
8
9
10
11
Disk Pressure
bash
kubectl describe node xxx1
bash
# 1. clear disk
# 2. restart k3s
systemctl restart k3s
# 3. upgrade
helm upgrade ....
# 4. delete unused pods
kubectl get pods -A | grep 'Init:ContainerStatusUnknown' | awk '{print "kubectl delete pod "$2" -n "$1}' | bash1
2
3
4
5
6
7
8
9
10
2
3
4
5
6
7
8
9
10
