【发布时间】：2020-03-01 16:30:12
【问题描述】：
当我在 kuberntes(v1.15.2) 中启动我的 elasticsearch 集群的 pod 时，pods 日志给出了这个错误：
OpenJDK 64-Bit Server VM warning: Option UseConcMarkSweepGC was deprecated in version 9.0 and will likely be removed in a future release.
[2020-03-01T05:43:27,304][INFO ][o.e.n.Node ] [es-cluster-2] initializing ...
[2020-03-01T05:43:27,484][WARN ][o.e.b.ElasticsearchUncaughtExceptionHandler] [es-cluster-2] uncaught exception in thread [main]
org.elasticsearch.bootstrap.StartupException: java.lang.IllegalStateException: failed to obtain node locks, tried [[/usr/share/elasticsearch/data/k8s-logs]] with lock id [0]; maybe these locations are not writable or multiple nodes were started without increasing [node.max_local_storage_nodes] (was [1])?
at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:140) ~[elasticsearch-6.4.3.jar:6.4.3]
at org.elasticsearch.bootstrap.Elasticsearch.execute(Elasticsearch.java:127) ~[elasticsearch-6.4.3.jar:6.4.3]
at org.elasticsearch.cli.EnvironmentAwareCommand.execute(EnvironmentAwareCommand.java:86) ~[elasticsearch-6.4.3.jar:6.4.3]
at org.elasticsearch.cli.Command.mainWithoutErrorHandling(Command.java:124) ~[elasticsearch-cli-6.4.3.jar:6.4.3]
at org.elasticsearch.cli.Command.main(Command.java:90) ~[elasticsearch-cli-6.4.3.jar:6.4.3]
at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:93) ~[elasticsearch-6.4.3.jar:6.4.3]
at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:86) ~[elasticsearch-6.4.3.jar:6.4.3]
Caused by: java.lang.IllegalStateException: failed to obtain node locks, tried [[/usr/share/elasticsearch/data/k8s-logs]] with lock id [0]; maybe these locations are not writable or multiple nodes were started without increasing [node.max_local_storage_nodes] (was [1])?
at org.elasticsearch.env.NodeEnvironment.<init>(NodeEnvironment.java:247) ~[elasticsearch-6.4.3.jar:6.4.3]
at org.elasticsearch.node.Node.<init>(Node.java:274) ~[elasticsearch-6.4.3.jar:6.4.3]
at org.elasticsearch.node.Node.<init>(Node.java:256) ~[elasticsearch-6.4.3.jar:6.4.3]
at org.elasticsearch.bootstrap.Bootstrap$5.<init>(Bootstrap.java:213) ~[elasticsearch-6.4.3.jar:6.4.3]
at org.elasticsearch.bootstrap.Bootstrap.setup(Bootstrap.java:213) ~[elasticsearch-6.4.3.jar:6.4.3]
at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:326) ~[elasticsearch-6.4.3.jar:6.4.3]
at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:136) ~[elasticsearch-6.4.3.jar:6.4.3]
... 6 more
Caused by: java.io.IOException: failed to obtain lock on /usr/share/elasticsearch/data/nodes/0
at org.elasticsearch.env.NodeEnvironment.<init>(NodeEnvironment.java:226) ~[elasticsearch-6.4.3.jar:6.4.3]
at org.elasticsearch.node.Node.<init>(Node.java:274) ~[elasticsearch-6.4.3.jar:6.4.3]
at org.elasticsearch.node.Node.<init>(Node.java:256) ~[elasticsearch-6.4.3.jar:6.4.3]
at org.elasticsearch.bootstrap.Bootstrap$5.<init>(Bootstrap.java:213) ~[elasticsearch-6.4.3.jar:6.4.3]
at org.elasticsearch.bootstrap.Bootstrap.setup(Bootstrap.java:213) ~[elasticsearch-6.4.3.jar:6.4.3]
at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:326) ~[elasticsearch-6.4.3.jar:6.4.3]
at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:136) ~[elasticsearch-6.4.3.jar:6.4.3]
... 6 more
Caused by: java.nio.file.AccessDeniedException: /usr/share/elasticsearch/data/nodes/0/node.lock
at sun.nio.fs.UnixException.translateToIOException(UnixException.java:90) ~[?:?]
at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:111) ~[?:?]
at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:116) ~[?:?]
at sun.nio.fs.UnixFileSystemProvider.newFileChannel(UnixFileSystemProvider.java:178) ~[?:?]
at java.nio.channels.FileChannel.open(FileChannel.java:292) ~[?:?]
at java.nio.channels.FileChannel.open(FileChannel.java:340) ~[?:?]
at org.apache.lucene.store.NativeFSLockFactory.obtainFSLock(NativeFSLockFactory.java:125) ~[lucene-core-7.4.0.jar:7.4.0 9060ac689c270b02143f375de0348b7f626adebc - jpountz - 2018-06-18 16:51:45]
我的 pod 是通过有状态集部署并挂载在 NFS 文件系统上的。这是我的有状态 yaml：
apiVersion: v1
items:
- apiVersion: apps/v1
kind: StatefulSet
metadata:
annotations:
kubectl.kubernetes.io/last-applied-configuration: |
{"apiVersion":"apps/v1","kind":"StatefulSet","metadata":{"annotations":{},"name":"es-cluster","namespace":"logging"},"spec":{"replicas":1,"selector":{"matchLabels":{"app":"elasticsearch"}},"serviceName":"elasticsearch","template":{"metadata":{"labels":{"app":"elasticsearch"}},"spec":{"containers":[{"env":[{"name":"cluster.name","value":"k8s-logs"},{"name":"node.name","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"discovery.zen.ping.unicast.hosts","value":"es-cluster-0.elasticsearch,es-cluster-1.elasticsearch,es-cluster-2.elasticsearch"},{"name":"discovery.zen.minimum_master_nodes","value":"2"},{"name":"ES_JAVA_OPTS","value":"-Xms512m -Xmx512m"}],"image":"registry.cn-hangzhou.aliyuncs.com/dabai_app_k8s/dabai_fat/elasticsearch-oss:6.4.3","name":"elasticsearch","ports":[{"containerPort":9200,"name":"rest","protocol":"TCP"},{"containerPort":9300,"name":"inter-node","protocol":"TCP"}],"resources":{"limits":{"cpu":"1000m"},"requests":{"cpu":"100m"}},"volumeMounts":[{"mountPath":"/usr/share/elasticsearch/data","name":"data"}]}],"imagePullSecrets":[{"name":"regcred"}],"initContainers":[{"command":["sh","-c","chown -R 1000:1000 /usr/share/elasticsearch/data"],"image":"docker.io/library/busybox:1.31.1","name":"fix-permissions","securityContext":{"privileged":true},"volumeMounts":[{"mountPath":"/usr/share/elasticsearch/data","name":"data"}]},{"command":["sysctl","-w","vm.max_map_count=262144"],"image":"docker.io/library/busybox:1.31.1","name":"increase-vm-max-map","securityContext":{"privileged":true}},{"command":["sh","-c","ulimit -n 65536"],"image":"docker.io/library/busybox:1.31.1","name":"increase-fd-ulimit","securityContext":{"privileged":true}}]}},"volumeClaimTemplates":[{"metadata":{"labels":{"app":"elasticsearch"},"name":"data"},"spec":{"accessModes":["ReadWriteOnce"],"resources":{"requests":{"storage":"5Gi"}},"storageClassName":"es-data-db"}}]}}
creationTimestamp: "2020-02-28T06:28:48Z"
generation: 13
labels:
app: elasticsearch
name: es-cluster
namespace: logging
resourceVersion: "17199681"
selfLink: /apis/apps/v1/namespaces/logging/statefulsets/es-cluster
uid: 06afbdc1-3ada-401e-8e62-74c6ce4aba97
spec:
podManagementPolicy: OrderedReady
replicas: 3
revisionHistoryLimit: 10
selector:
matchLabels:
app: elasticsearch
serviceName: elasticsearch
template:
metadata:
creationTimestamp: null
labels:
app: elasticsearch
spec:
containers:
- env:
- name: cluster.name
value: k8s-logs
- name: node.name
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.name
- name: discovery.zen.ping.unicast.hosts
value: es-cluster-0.elasticsearch,es-cluster-1.elasticsearch,es-cluster-2.elasticsearch
- name: discovery.zen.minimum_master_nodes
value: "2"
- name: ES_JAVA_OPTS
value: -Xms512m -Xmx512m
image: registry.cn-hangzhou.aliyuncs.com/dabai_app_k8s/dabai_fat/elasticsearch-oss:6.4.3
imagePullPolicy: IfNotPresent
name: elasticsearch
ports:
- containerPort: 9200
name: rest
protocol: TCP
- containerPort: 9300
name: inter-node
protocol: TCP
resources:
limits:
cpu: "1"
requests:
cpu: 100m
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /usr/share/elasticsearch/data
name: data
dnsPolicy: ClusterFirst
imagePullSecrets:
- name: regcred
initContainers:
- command:
- sh
- -c
- chown -R 1000:1000 /usr/share/elasticsearch/data
image: docker.io/library/busybox:1.31.1
imagePullPolicy: IfNotPresent
name: fix-permissions
resources: {}
securityContext:
privileged: true
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /usr/share/elasticsearch/data
name: data
- command:
- sysctl
- -w
- vm.max_map_count=262144
image: docker.io/library/busybox:1.31.1
imagePullPolicy: IfNotPresent
name: increase-vm-max-map
resources: {}
securityContext:
privileged: true
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
- command:
- sh
- -c
- ulimit -n 65536
image: docker.io/library/busybox:1.31.1
imagePullPolicy: IfNotPresent
name: increase-fd-ulimit
resources: {}
securityContext:
privileged: true
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
restartPolicy: Always
schedulerName: default-scheduler
securityContext: {}
terminationGracePeriodSeconds: 30
updateStrategy:
rollingUpdate:
partition: 0
type: RollingUpdate
volumeClaimTemplates:
- metadata:
creationTimestamp: null
labels:
app: elasticsearch
name: data
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 5Gi
storageClassName: es-data-db
volumeMode: Filesystem
status:
phase: Pending
status:
collisionCount: 0
currentReplicas: 3
currentRevision: es-cluster-76d4866565
observedGeneration: 13
readyReplicas: 2
replicas: 3
updateRevision: es-cluster-76d4866565
updatedReplicas: 3
kind: List
metadata:
resourceVersion: ""
selfLink: ""
我启动了 3 个 pod，每个 pod 都有自己的数据文件夹（通过 3 PV 和 3 PVC 挂载）。不可能让多节点读取和写入一个数据文件夹。所以怎么会发生这种情况，我应该更改 NFS文件夹权限？
【问题讨论】：
-
我在使用 elasticsearch 7.16.3 的 alpine3.11 上遇到此问题，当我使用
apk add openjdk8-jre时，它可以工作，但如果我尝试使用openjdk9-jre，它会因root cause: java.io.IOException: Mount point not found而失败。在 jdk 10、11 上也是如此。让它工作的唯一方法是从我的 Dockerfile 中删除所有COPY命令。我需要这些命令，但随后 es 启动。