Add OpenEBS v4.6.1 with images repointed to ghcr.io/wrktalk-tech - Loki and Alloy disabled
This commit is contained in:
@@ -0,0 +1,175 @@
|
||||
{{/* vim: set filetype=mustache: */}}
|
||||
{{/*
|
||||
Expand the name of the chart.
|
||||
*/}}
|
||||
{{- define "rawfile-localpv.name" -}}
|
||||
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Create a default fully qualified app name.
|
||||
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
|
||||
If release name contains chart name it will be used as a full name.
|
||||
*/}}
|
||||
{{- define "rawfile-localpv.fullname" -}}
|
||||
{{- if .Values.fullnameOverride }}
|
||||
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
|
||||
{{- else }}
|
||||
{{- $name := default .Chart.Name .Values.nameOverride }}
|
||||
{{- if contains $name .Release.Name }}
|
||||
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
|
||||
{{- else }}
|
||||
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Create chart name and version as used by the chart label.
|
||||
*/}}
|
||||
{{- define "rawfile-localpv.chart" -}}
|
||||
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Common labels
|
||||
*/}}
|
||||
{{- define "rawfile-localpv.labels" -}}
|
||||
helm.sh/chart: {{ include "rawfile-localpv.chart" . }}
|
||||
{{ include "rawfile-localpv.selectorLabels" . }}
|
||||
{{- if .Chart.AppVersion }}
|
||||
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
|
||||
{{- end }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Selector labels
|
||||
*/}}
|
||||
{{- define "rawfile-localpv.selectorLabels" -}}
|
||||
app.kubernetes.io/name: {{ include "rawfile-localpv.name" . }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Create the name of the service account to use
|
||||
*/}}
|
||||
{{- define "rawfile-localpv.serviceAccountName" -}}
|
||||
{{- if .Values.serviceAccount.create }}
|
||||
{{- default (include "rawfile-localpv.fullname" .) .Values.serviceAccount.name }}
|
||||
{{- else }}
|
||||
{{- default "default" .Values.serviceAccount.name }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Some helpers to handle image global information
|
||||
*/}}
|
||||
|
||||
{{- define "rawfile-localpv.controller-resources" -}}
|
||||
{{- toYaml (.Values.controller.resources) }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "rawfile-localpv.controller-external-resizer-resources" -}}
|
||||
{{- toYaml (.Values.controller.externalResizer.resources) }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "rawfile-localpv.api-server-resources" -}}
|
||||
{{- toYaml (.Values.controller.apiServer.resources) }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "rawfile-localpv.node-resources" -}}
|
||||
{{- toYaml (.Values.node.resources) }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "rawfile-localpv.node-driver-registrar-resources" -}}
|
||||
{{- toYaml (.Values.node.driverRegistrar.resources) }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "rawfile-localpv.node-external-provisioner-resources" -}}
|
||||
{{- toYaml (.Values.node.externalProvisioner.resources) }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "rawfile-localpv.node-external-snapshotter-resources" -}}
|
||||
{{- toYaml (.Values.node.externalSnapshotter.resources) }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "rawfile-localpv.node-snapshot-controller-resources" -}}
|
||||
{{- toYaml (.Values.node.snapshotController.resources) }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "rawfile-localpv.node-kubelet-path" -}}
|
||||
{{- printf "%s/" (.Values.node.kubeletPath | trimSuffix "/") -}}
|
||||
{{- end }}
|
||||
|
||||
{{- define "rawfile-localpv.metadata-dir-path" -}}
|
||||
{{- tpl .Values.node.metadataDirPath . }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "rawfile-localpv.pool-volumes" -}}
|
||||
{{- range $name, $pool := .Values.node.storagePools }}
|
||||
- name: pool-{{ $name }}
|
||||
hostPath:
|
||||
path: {{ tpl $pool.path . }}
|
||||
type: DirectoryOrCreate
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "rawfile-localpv.pool-volume-mounts" -}}
|
||||
{{- range $name, $pool := .Values.node.storagePools }}
|
||||
- name: pool-{{ $name }}
|
||||
mountPath: {{ tpl $pool.path . }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
|
||||
{{/*
|
||||
Creates the image URL ie registry/repository:tag
|
||||
*/}}
|
||||
{{- define "rawfile-localpv.common.image" -}}
|
||||
{{- $registryName := ((.global).imageRegistry) | default ((.imageRoot).registry) | default ((.csiSideCars).registry) | trimSuffix "/" -}}
|
||||
{{- $repositoryName := .imageRoot.repository -}}
|
||||
{{- $separator := ":" -}}
|
||||
{{- $chartVersion := .chartVersion -}}
|
||||
{{- $termination := .imageRoot.tag | default (printf "v%s" $chartVersion) | toString -}}
|
||||
{{- if $registryName }}
|
||||
{{- printf "%s/%s%s%s" $registryName $repositoryName $separator $termination -}}
|
||||
{{- else }}
|
||||
{{- printf "%s%s%s" $repositoryName $separator $termination -}}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Concatenates imagepullsecrets and handles different formats (example - secret or - name: secret)
|
||||
*/}}
|
||||
{{- define "rawfile-localpv.common.pullSecrets" -}}
|
||||
{{- $names := list -}}
|
||||
{{- with .Values.global.imagePullSecrets -}}
|
||||
{{- range . -}}
|
||||
{{- if kindIs "map" . }}
|
||||
{{- if and (hasKey . "name") (not (empty .name)) -}}
|
||||
{{ $names = append $names .name }}
|
||||
{{- end -}}
|
||||
{{- else if not (empty .) -}}
|
||||
{{ $names = append $names . -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- with .Values.imagePullSecrets -}}
|
||||
{{- range . }}
|
||||
{{- if kindIs "map" . -}}
|
||||
{{- if and (hasKey . "name") (not (empty .name)) -}}
|
||||
{{- $names = append $names .name }}
|
||||
{{- end -}}
|
||||
{{- else if not (empty .) -}}
|
||||
{{- $names = append $names . -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- $names = uniq $names -}}
|
||||
{{- if $names -}}
|
||||
{{- range $names }}
|
||||
- name: {{ . }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
@@ -0,0 +1,179 @@
|
||||
{{- $cap := .Values.capabilities | default dict }}
|
||||
{{- if or ($cap.resize.enabled) ($cap.apiServer.enabled) }}
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "rawfile-localpv.fullname" . }}-controller
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels: &selectorLabels
|
||||
{{- include "rawfile-localpv.selectorLabels" . | nindent 6 }}
|
||||
component: controller
|
||||
template:
|
||||
metadata:
|
||||
labels: *selectorLabels
|
||||
spec:
|
||||
serviceAccount: {{ include "rawfile-localpv.fullname" . }}-driver
|
||||
priorityClassName: {{ .Values.controller.priorityClassName }}
|
||||
tolerations:
|
||||
{{- .Values.controller.tolerations | toYaml | nindent 8 }}
|
||||
nodeSelector:
|
||||
{{- .Values.controller.nodeSelector | toYaml | nindent 8 }}
|
||||
affinity:
|
||||
{{- .Values.controller.affinity | toYaml | nindent 8 }}
|
||||
volumes:
|
||||
- name: socket-dir
|
||||
emptyDir: {}
|
||||
containers:
|
||||
{{- if $cap.apiServer.enabled }}
|
||||
- name: api-server
|
||||
image: {{ include "rawfile-localpv.common.image" (dict "imageRoot" .Values.image "global" .Values.global "chartVersion" .Chart.AppVersion) | quote }}
|
||||
imagePullPolicy: {{ default .Values.image.pullPolicy .Values.global.imagePullPolicy }}
|
||||
args:
|
||||
- api
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
resources:
|
||||
{{- include "rawfile-localpv.api-server-resources" . | nindent 12 }}
|
||||
env:
|
||||
- name: PROVISIONER_NAME
|
||||
value: "{{ .Values.provisionerName }}"
|
||||
- name: NAMESPACE
|
||||
value: {{ .Release.Namespace }}
|
||||
- name: LOG_LEVEL
|
||||
value: {{ .Values.logLevel }}
|
||||
- name: LOG_FORMAT
|
||||
value: {{ .Values.logFormat }}
|
||||
- name: GA_ENABLED
|
||||
{{- if kindIs "bool" .Values.global.analytics.enabled }}
|
||||
value: "{{ .Values.global.analytics.enabled }}"
|
||||
{{- else }}
|
||||
value: "{{ .Values.analytics.enabled }}"
|
||||
{{- end }}
|
||||
{{- if .Values.global.analytics.gaId }}
|
||||
- name: GA_ID
|
||||
value: {{ .Values.global.analytics.gaId | quote }}
|
||||
{{- else if .Values.analytics.gaId }}
|
||||
- name: GA_ID
|
||||
value: {{ .Values.analytics.gaId | quote }}
|
||||
{{- end }}
|
||||
{{- if .Values.global.analytics.gaKey }}
|
||||
- name: GA_KEY
|
||||
value: {{ .Values.global.analytics.gaKey | quote }}
|
||||
{{- else if .Values.analytics.gaKey }}
|
||||
- name: GA_KEY
|
||||
value: {{ .Values.analytics.gaKey | quote }}
|
||||
{{- end }}
|
||||
- name: NODE_DS
|
||||
value: {{ include "rawfile-localpv.fullname" . }}-node
|
||||
- name: INTERNAL_PORT
|
||||
value: {{ .Values.node.internalGRPC.port | toString | quote }}
|
||||
{{- if .Values.auth.enabled }}
|
||||
- name: INTERNAL_SIGNATURE
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
{{- if .Values.auth.secretName }}
|
||||
name: {{ .Values.auth.secretName }}
|
||||
{{- else }}
|
||||
name: {{ include "rawfile-localpv.fullname" . }}-secrets
|
||||
{{- end }}
|
||||
key: internal-signature
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if $cap.resize.enabled }}
|
||||
- name: csi-driver
|
||||
image: {{ include "rawfile-localpv.common.image" (dict "imageRoot" .Values.image "global" .Values.global "chartVersion" .Chart.AppVersion) | quote }}
|
||||
imagePullPolicy: {{ default .Values.image.pullPolicy .Values.global.imagePullPolicy }}
|
||||
args:
|
||||
- csi-driver
|
||||
- --enable-metrics=false
|
||||
env:
|
||||
- name: PROVISIONER_NAME
|
||||
value: "{{ .Values.provisionerName }}"
|
||||
- name: CSI_DRIVER__ENDPOINT
|
||||
value: unix:///csi/csi.sock
|
||||
- name: CSI_DRIVER__NODE_ID
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
apiVersion: v1
|
||||
fieldPath: spec.nodeName
|
||||
- name: NAMESPACE
|
||||
value: {{ .Release.Namespace }}
|
||||
- name: LOG_LEVEL
|
||||
value: {{ .Values.logLevel }}
|
||||
- name: LOG_FORMAT
|
||||
value: {{ .Values.logFormat }}
|
||||
- name: CSI_DRIVER__PLUGIN_TYPE
|
||||
value: controller
|
||||
- name: CSI_DRIVER__GRPC_WORKERS
|
||||
value: {{ .Values.controller.grpcWorkers | toString | quote }}
|
||||
- name: GA_ENABLED
|
||||
{{- if kindIs "bool" .Values.global.analytics.enabled }}
|
||||
value: "{{ .Values.global.analytics.enabled }}"
|
||||
{{- else }}
|
||||
value: "{{ .Values.analytics.enabled }}"
|
||||
{{- end }}
|
||||
{{- if .Values.global.analytics.gaId }}
|
||||
- name: GA_ID
|
||||
value: {{ .Values.global.analytics.gaId | quote }}
|
||||
{{- else if .Values.analytics.gaId }}
|
||||
- name: GA_ID
|
||||
value: {{ .Values.analytics.gaId | quote }}
|
||||
{{- end }}
|
||||
{{- if .Values.global.analytics.gaKey }}
|
||||
- name: GA_KEY
|
||||
value: {{ .Values.global.analytics.gaKey | quote }}
|
||||
{{- else if .Values.analytics.gaKey }}
|
||||
- name: GA_KEY
|
||||
value: {{ .Values.analytics.gaKey | quote }}
|
||||
{{- end }}
|
||||
{{- if .Values.global.analytics.gaDnsNameservers }}
|
||||
- name: GA_DNS
|
||||
value: {{ .Values.global.analytics.gaDnsNameservers | quote }}
|
||||
{{- else if .Values.analytics.gaDnsNameservers }}
|
||||
- name: GA_DNS
|
||||
value: {{ .Values.analytics.gaDnsNameservers | quote }}
|
||||
{{- end }}
|
||||
- name: INTERNAL_PORT
|
||||
value: {{ .Values.node.internalGRPC.port | toString | quote }}
|
||||
- name: NODE_DS
|
||||
value: {{ include "rawfile-localpv.fullname" . }}-node
|
||||
{{- if .Values.auth.enabled }}
|
||||
- name: INTERNAL_SIGNATURE
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
{{- if .Values.auth.secretName }}
|
||||
name: {{ .Values.auth.secretName }}
|
||||
{{- else }}
|
||||
name: {{ include "rawfile-localpv.fullname" . }}-secrets
|
||||
{{- end }}
|
||||
key: internal-signature
|
||||
{{- end }}
|
||||
- name: CSI_DRIVER__CAPABILITIES__SNAPSHOTS__ENABLED
|
||||
value: "{{ .Values.capabilities.snapshots.enabled }}"
|
||||
- name: CSI_DRIVER__CAPABILITIES__RESIZE__ENABLED
|
||||
value: "{{ .Values.capabilities.resize.enabled }}"
|
||||
volumeMounts:
|
||||
- name: socket-dir
|
||||
mountPath: /csi
|
||||
resources:
|
||||
{{- include "rawfile-localpv.controller-resources" . | nindent 12 }}
|
||||
- name: external-resizer
|
||||
image: {{ include "rawfile-localpv.common.image" (dict "imageRoot" .Values.controller.externalResizer.image "csiSideCars" .Values.csiSideCars.image "global" .Values.global) | quote }}
|
||||
imagePullPolicy: {{ .Values.global.imagePullPolicy | default .Values.controller.externalResizer.image.pullPolicy | default .Values.csiSideCars.image.pullPolicy }}
|
||||
args:
|
||||
- "--csi-address=$(ADDRESS)"
|
||||
- "--handle-volume-inuse-error=false"
|
||||
- "--timeout=30s"
|
||||
- "--automaxprocs=true"
|
||||
env:
|
||||
- name: ADDRESS
|
||||
value: /csi/csi.sock
|
||||
volumeMounts:
|
||||
- name: socket-dir
|
||||
mountPath: /csi
|
||||
resources:
|
||||
{{- include "rawfile-localpv.controller-external-resizer-resources" . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,22 @@
|
||||
{{- $cap := .Values.capabilities | default dict }}
|
||||
{{- $api := $cap.apiServer | default dict }}
|
||||
{{- if $api.enabled }}
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "rawfile-localpv.fullname" . }}-controller
|
||||
labels:
|
||||
{{- include "rawfile-localpv.labels" . | nindent 4 }}
|
||||
component: controller
|
||||
spec:
|
||||
type: ClusterIP
|
||||
{{- if $api.enabled }}
|
||||
ports:
|
||||
- port: 8080
|
||||
targetPort: 8080
|
||||
name: api-server
|
||||
{{- end }}
|
||||
selector:
|
||||
{{- include "rawfile-localpv.selectorLabels" . | nindent 4 }}
|
||||
component: controller
|
||||
{{- end }}
|
||||
@@ -0,0 +1,11 @@
|
||||
apiVersion: storage.k8s.io/v1
|
||||
kind: CSIDriver
|
||||
metadata:
|
||||
name: {{ .Values.provisionerName }}
|
||||
spec:
|
||||
attachRequired: false
|
||||
podInfoOnMount: true
|
||||
fsGroupPolicy: File
|
||||
storageCapacity: true
|
||||
volumeLifecycleModes:
|
||||
- Persistent
|
||||
@@ -0,0 +1,282 @@
|
||||
apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
name: {{ include "rawfile-localpv.fullname" . }}-node
|
||||
spec:
|
||||
updateStrategy:
|
||||
rollingUpdate:
|
||||
maxUnavailable: "100%"
|
||||
selector:
|
||||
matchLabels: &selectorLabels
|
||||
{{- include "rawfile-localpv.selectorLabels" . | nindent 6 }}
|
||||
component: node
|
||||
template:
|
||||
metadata:
|
||||
labels: *selectorLabels
|
||||
{{- with .Values.node.podAnnotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
serviceAccount: {{ include "rawfile-localpv.fullname" . }}-driver
|
||||
priorityClassName: {{ .Values.node.priorityClassName }}
|
||||
{{- if .Values.node.hostNetwork }}
|
||||
hostNetwork: true
|
||||
{{- end }}
|
||||
tolerations:
|
||||
{{- .Values.node.tolerations | toYaml | nindent 8 }}
|
||||
nodeSelector:
|
||||
{{- .Values.node.nodeSelector | toYaml | nindent 8 }}
|
||||
affinity:
|
||||
{{- .Values.node.affinity | toYaml | nindent 8 }}
|
||||
volumes:
|
||||
- name: registration-dir
|
||||
hostPath:
|
||||
path: {{ include "rawfile-localpv.node-kubelet-path" . }}plugins_registry
|
||||
type: Directory
|
||||
- name: socket-dir
|
||||
hostPath:
|
||||
path: {{ include "rawfile-localpv.node-kubelet-path" . }}plugins/rawfile-localpv
|
||||
type: DirectoryOrCreate
|
||||
- name: mountpoint-dir
|
||||
hostPath:
|
||||
path: {{ include "rawfile-localpv.node-kubelet-path" . }}
|
||||
type: DirectoryOrCreate
|
||||
- name: metadata-dir
|
||||
hostPath:
|
||||
path: {{ include "rawfile-localpv.metadata-dir-path" . }}
|
||||
type: DirectoryOrCreate
|
||||
- name: device
|
||||
hostPath:
|
||||
path: /dev
|
||||
type: Directory
|
||||
{{- include "rawfile-localpv.pool-volumes" . | nindent 8 }}
|
||||
containers:
|
||||
- name: csi-driver
|
||||
image: {{ include "rawfile-localpv.common.image" (dict "imageRoot" .Values.image "global" .Values.global "chartVersion" .Chart.AppVersion) | quote }}
|
||||
imagePullPolicy: {{ default .Values.image.pullPolicy .Values.global.imagePullPolicy }}
|
||||
securityContext:
|
||||
privileged: true
|
||||
env:
|
||||
- name: PROVISIONER_NAME
|
||||
value: "{{ .Values.provisionerName }}"
|
||||
- name: CSI_DRIVER__ENDPOINT
|
||||
value: unix:///csi/csi.sock
|
||||
- name: CSI_DRIVER__NODE_ID
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
apiVersion: v1
|
||||
fieldPath: spec.nodeName
|
||||
- name: CSI_DRIVER__ENABLE_METRICS
|
||||
value: {{ .Values.metrics.enabled | toString | quote }}
|
||||
- name: CSI_DRIVER__METRICS_PORT
|
||||
value: {{ .Values.metrics.port | toString | quote }}
|
||||
- name: CSI_DRIVER__METADATA_DIR
|
||||
value: {{ include "rawfile-localpv.metadata-dir-path" . }}
|
||||
- name: NAMESPACE
|
||||
value: {{ .Release.Namespace }}
|
||||
- name: LOG_LEVEL
|
||||
value: {{ .Values.logLevel }}
|
||||
- name: LOG_FORMAT
|
||||
value: {{ .Values.logFormat }}
|
||||
- name: CSI_DRIVER__PLUGIN_TYPE
|
||||
value: node
|
||||
- name: CSI_DRIVER__GRPC_WORKERS
|
||||
value: {{ .Values.node.grpcWorkers | toString | quote }}
|
||||
- name: GA_ENABLED
|
||||
{{- if kindIs "bool" .Values.global.analytics.enabled }}
|
||||
value: "{{ .Values.global.analytics.enabled }}"
|
||||
{{- else }}
|
||||
value: "{{ .Values.analytics.enabled }}"
|
||||
{{- end }}
|
||||
{{- if .Values.global.analytics.gaId }}
|
||||
- name: GA_ID
|
||||
value: {{ .Values.global.analytics.gaId | quote }}
|
||||
{{- else if .Values.analytics.gaId }}
|
||||
- name: GA_ID
|
||||
value: {{ .Values.analytics.gaId | quote }}
|
||||
{{- end }}
|
||||
{{- if .Values.global.analytics.gaKey }}
|
||||
- name: GA_KEY
|
||||
value: {{ .Values.global.analytics.gaKey | quote }}
|
||||
{{- else if .Values.analytics.gaKey }}
|
||||
- name: GA_KEY
|
||||
value: {{ .Values.analytics.gaKey | quote }}
|
||||
{{- end }}
|
||||
{{- if .Values.global.analytics.gaDnsNameservers }}
|
||||
- name: GA_DNS
|
||||
value: {{ .Values.global.analytics.gaDnsNameservers | quote }}
|
||||
{{- else if .Values.analytics.gaDnsNameservers }}
|
||||
- name: GA_DNS
|
||||
value: {{ .Values.analytics.gaDnsNameservers | quote }}
|
||||
{{- end }}
|
||||
- name: CSI_DRIVER__INTERNAL_IP
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
apiVersion: v1
|
||||
fieldPath: status.podIP
|
||||
- name: INTERNAL_PORT
|
||||
value: {{ .Values.node.internalGRPC.port | toString | quote }}
|
||||
- name: CSI_DRIVER__INTERNAL_GRPC_WORKERS
|
||||
value: {{ .Values.node.internalGRPC.workers | toString | quote }}
|
||||
- name: NODE_DS
|
||||
value: {{ include "rawfile-localpv.fullname" . }}-node
|
||||
{{- if .Values.auth.enabled }}
|
||||
- name: INTERNAL_SIGNATURE
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
{{- if .Values.auth.secretName }}
|
||||
name: {{ .Values.auth.secretName }}
|
||||
{{- else }}
|
||||
name: {{ include "rawfile-localpv.fullname" . }}-secrets
|
||||
{{- end }}
|
||||
key: internal-signature
|
||||
{{- end }}
|
||||
- name: CSI_DRIVER__DEFAULT_FS
|
||||
value: {{ .Values.node.defaultFs }}
|
||||
- name: CSI_DRIVER__DEFAULT_POOL
|
||||
{{- if .Values.node.defaultPool }}
|
||||
value: {{ .Values.node.defaultPool }}
|
||||
{{- else if not .Values.node.storagePools }}
|
||||
value: "data-dir"
|
||||
{{- end }}
|
||||
- name: CSI_DRIVER__STORAGE_POOLS
|
||||
value: |
|
||||
{
|
||||
{{- $keys := keys .Values.node.storagePools | sortAlpha }}
|
||||
{{- range $i, $name := $keys }}
|
||||
"{{ $name }}": {
|
||||
"path": "{{ tpl (index $.Values.node.storagePools $name).path . }}"
|
||||
{{- with (index $.Values.node.storagePools $name).reservedCapacity }}
|
||||
, "reserved_capacity": "{{ . | toString }}"
|
||||
{{- end }}
|
||||
{{- with (index $.Values.node.storagePools $name).reservedCapacityMode }}
|
||||
, "reserved_capacity_mode": "{{ . | toString }}"
|
||||
{{- end }}
|
||||
{{- with (index $.Values.node.storagePools $name) }}
|
||||
{{- if hasKey . "accountVolumesFreeSpace" }}
|
||||
, "account_volumes_free_space": {{ .accountVolumesFreeSpace }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
}{{ if lt (add1 $i) (len $keys) }},{{ end }}
|
||||
{{- end }}
|
||||
}
|
||||
- name: CSI_DRIVER__CAPABILITIES__SNAPSHOTS__ENABLED
|
||||
value: "{{ .Values.capabilities.snapshots.enabled }}"
|
||||
- name: CSI_DRIVER__CAPABILITIES__RESIZE__ENABLED
|
||||
value: "{{ .Values.capabilities.resize.enabled }}"
|
||||
ports:
|
||||
- name: metrics
|
||||
containerPort: {{ .Values.metrics.port }}
|
||||
volumeMounts:
|
||||
- name: socket-dir
|
||||
mountPath: /csi
|
||||
- name: mountpoint-dir
|
||||
mountPath: {{ include "rawfile-localpv.node-kubelet-path" . }}
|
||||
mountPropagation: "Bidirectional"
|
||||
- name: metadata-dir
|
||||
mountPath: {{ include "rawfile-localpv.metadata-dir-path" . }}
|
||||
- name: device
|
||||
mountPath: /dev
|
||||
{{- include "rawfile-localpv.pool-volume-mounts" . | nindent 12 }}
|
||||
resources:
|
||||
{{- include "rawfile-localpv.controller-resources" . | nindent 12 }}
|
||||
- name: node-driver-registrar
|
||||
image: {{ include "rawfile-localpv.common.image" (dict "imageRoot" .Values.node.driverRegistrar.image "csiSideCars" .Values.csiSideCars.image "global" .Values.global) | quote }}
|
||||
imagePullPolicy: {{ .Values.global.imagePullPolicy | default .Values.node.driverRegistrar.image.pullPolicy | default .Values.csiSideCars.image.pullPolicy }}
|
||||
args:
|
||||
- --csi-address=$(ADDRESS)
|
||||
- --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
|
||||
- --health-port={{ .Values.node.driverRegistrar.healthzPort }}
|
||||
- --automaxprocs=true
|
||||
env:
|
||||
- name: ADDRESS
|
||||
value: /csi/csi.sock
|
||||
- name: DRIVER_REG_SOCK_PATH
|
||||
value: {{ include "rawfile-localpv.node-kubelet-path" . }}plugins/rawfile-localpv/csi.sock
|
||||
ports:
|
||||
- containerPort: {{ .Values.node.driverRegistrar.healthzPort }}
|
||||
name: healthz
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /healthz
|
||||
port: healthz
|
||||
initialDelaySeconds: 5
|
||||
timeoutSeconds: 5
|
||||
volumeMounts:
|
||||
- name: socket-dir
|
||||
mountPath: /csi
|
||||
- name: registration-dir
|
||||
mountPath: /registration
|
||||
resources:
|
||||
{{- include "rawfile-localpv.node-driver-registrar-resources" . | nindent 12 }}
|
||||
- name: external-provisioner
|
||||
image: {{ include "rawfile-localpv.common.image" (dict "imageRoot" .Values.node.externalProvisioner.image "csiSideCars" .Values.csiSideCars.image "global" .Values.global) | quote }}
|
||||
imagePullPolicy: {{ .Values.global.imagePullPolicy | default .Values.node.externalProvisioner.image.pullPolicy | default .Values.csiSideCars.image.pullPolicy }}
|
||||
args:
|
||||
- "--csi-address=$(ADDRESS)"
|
||||
- "--feature-gates=Topology=true"
|
||||
- "--strict-topology"
|
||||
- "--immediate-topology=false"
|
||||
- "--timeout=120s"
|
||||
- "--enable-capacity=true"
|
||||
- "--capacity-for-immediate-binding=true"
|
||||
- "--capacity-ownerref-level=1" # DaemonSet
|
||||
- "--node-deployment=true"
|
||||
- "--extra-create-metadata=true"
|
||||
- "--capacity-poll-interval={{ .Values.node.externalProvisioner.capacityPollInterval }}"
|
||||
- "--automaxprocs=true"
|
||||
env:
|
||||
- name: ADDRESS
|
||||
value: /csi/csi.sock
|
||||
- name: NODE_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
- name: NAMESPACE
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.namespace
|
||||
- name: POD_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.name
|
||||
volumeMounts:
|
||||
- name: socket-dir
|
||||
mountPath: /csi
|
||||
resources:
|
||||
{{- include "rawfile-localpv.node-external-provisioner-resources" . | nindent 12 }}
|
||||
{{- if .Values.capabilities.snapshots.enabled }}
|
||||
- name: external-snapshotter
|
||||
image: {{ include "rawfile-localpv.common.image" (dict "imageRoot" .Values.node.externalSnapshotter.image "csiSideCars" .Values.csiSideCars.image "global" .Values.global) | quote }}
|
||||
imagePullPolicy: {{ .Values.global.imagePullPolicy | default .Values.node.externalSnapshotter.image.pullPolicy | default .Values.csiSideCars.image.pullPolicy }}
|
||||
args:
|
||||
- "--csi-address=$(ADDRESS)"
|
||||
- "--node-deployment=true"
|
||||
- "--extra-create-metadata=true"
|
||||
- "--automaxprocs=true"
|
||||
env:
|
||||
- name: ADDRESS
|
||||
value: /csi/csi.sock
|
||||
- name: NODE_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
volumeMounts:
|
||||
- name: socket-dir
|
||||
mountPath: /csi
|
||||
resources:
|
||||
{{- include "rawfile-localpv.node-external-snapshotter-resources" . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if and .Values.capabilities.snapshots.enabled .Values.node.snapshotController.enabled }}
|
||||
- name: snapshot-controller
|
||||
args:
|
||||
- "--v=2"
|
||||
- "--enable-distributed-snapshotting=true"
|
||||
# runs as Daemonset, but only one should be active per cluster
|
||||
- "--leader-election=true"
|
||||
resources:
|
||||
{{- include "rawfile-localpv.node-snapshot-controller-resources" . | nindent 12 }}
|
||||
image: {{ include "rawfile-localpv.common.image" (dict "imageRoot" .Values.node.snapshotController.image "csiSideCars" .Values.csiSideCars.image "global" .Values.global) | quote }}
|
||||
imagePullPolicy: {{ .Values.global.imagePullPolicy | default .Values.node.snapshotController.image.pullPolicy | default .Values.csiSideCars.image.pullPolicy }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,20 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "rawfile-localpv.fullname" . }}-node
|
||||
labels:
|
||||
{{- include "rawfile-localpv.labels" . | nindent 4 }}
|
||||
component: node
|
||||
spec:
|
||||
type: ClusterIP
|
||||
ports:
|
||||
- name: metrics
|
||||
port: {{ .Values.metrics.port }}
|
||||
targetPort: metrics
|
||||
protocol: TCP
|
||||
- name: internal
|
||||
port: {{ .Values.node.internalGRPC.port }}
|
||||
protocol: TCP
|
||||
selector:
|
||||
{{- include "rawfile-localpv.selectorLabels" . | nindent 4 }}
|
||||
component: node
|
||||
@@ -0,0 +1,25 @@
|
||||
{{- if .Values.metrics.enabled }}
|
||||
{{- if .Values.metrics.serviceMonitor.enabled }}
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: {{ include "rawfile-localpv.fullname" . }}-node
|
||||
labels:
|
||||
{{- include "rawfile-localpv.labels" . | nindent 4 }}
|
||||
spec:
|
||||
endpoints:
|
||||
- port: metrics
|
||||
path: /metrics
|
||||
{{- with .Values.metrics.serviceMonitor.interval }}
|
||||
interval: {{ . }}
|
||||
{{- end }}
|
||||
jobLabel: "helm.sh/chart"
|
||||
namespaceSelector:
|
||||
matchNames:
|
||||
- {{ .Release.Namespace }}
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "rawfile-localpv.selectorLabels" . | nindent 6 }}
|
||||
component: node
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,203 @@
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: {{ include "rawfile-localpv.fullname" . }}-driver
|
||||
{{- if or .Values.global.imagePullSecrets .Values.imagePullSecrets }}
|
||||
imagePullSecrets:
|
||||
{{- include "rawfile-localpv.common.pullSecrets" . }}
|
||||
{{- end }}
|
||||
---
|
||||
kind: ClusterRole
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: {{ include "rawfile-localpv.fullname" . }}-provisioner
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["secrets"]
|
||||
verbs: ["get", "list"]
|
||||
- apiGroups: [""]
|
||||
resources: ["persistentvolumes"]
|
||||
verbs: ["get", "list", "watch", "create", "delete"]
|
||||
- apiGroups: [""]
|
||||
resources: ["persistentvolumeclaims"]
|
||||
verbs: ["get", "list", "watch", "update"]
|
||||
- apiGroups: ["storage.k8s.io"]
|
||||
resources: ["storageclasses"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["events"]
|
||||
verbs: ["list", "watch", "create", "update", "patch"]
|
||||
- apiGroups: ["snapshot.storage.k8s.io"]
|
||||
resources: ["volumesnapshots"]
|
||||
verbs: ["get", "list"]
|
||||
- apiGroups: ["snapshot.storage.k8s.io"]
|
||||
resources: ["volumesnapshotcontents"]
|
||||
verbs: ["get", "list"]
|
||||
- apiGroups: ["storage.k8s.io"]
|
||||
resources: ["csinodes"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["nodes"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: ["storage.k8s.io"]
|
||||
resources: ["volumeattachments"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: ["storage.k8s.io"]
|
||||
resources: ["csistoragecapacities"]
|
||||
verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
|
||||
- apiGroups: [""]
|
||||
resources: ["pods", "pods/log"]
|
||||
verbs: ["get"]
|
||||
- apiGroups: ["apps"]
|
||||
resources: ["daemonsets"]
|
||||
verbs: ["get"]
|
||||
---
|
||||
kind: ClusterRoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: {{ include "rawfile-localpv.fullname" . }}-provisioner
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "rawfile-localpv.fullname" . }}-driver
|
||||
namespace: {{ .Release.Namespace }}
|
||||
roleRef:
|
||||
kind: ClusterRole
|
||||
name: {{ include "rawfile-localpv.fullname" . }}-provisioner
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
---
|
||||
kind: ClusterRole
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: {{ include "rawfile-localpv.fullname" . }}-broker
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["persistentvolumes"]
|
||||
verbs: ["get"]
|
||||
- apiGroups: [""]
|
||||
resources: ["pods"]
|
||||
verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
|
||||
---
|
||||
kind: ClusterRoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: {{ include "rawfile-localpv.fullname" . }}-broker
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "rawfile-localpv.fullname" . }}-driver
|
||||
namespace: {{ .Release.Namespace }}
|
||||
roleRef:
|
||||
kind: ClusterRole
|
||||
name: {{ include "rawfile-localpv.fullname" . }}-broker
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
---
|
||||
kind: ClusterRole
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: {{ include "rawfile-localpv.fullname" . }}-resizer
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["secrets"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["persistentvolumes"]
|
||||
verbs: ["get", "list", "watch", "patch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["persistentvolumeclaims"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["pods"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["persistentvolumeclaims/status"]
|
||||
verbs: ["patch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["events"]
|
||||
verbs: ["list", "watch", "create", "update", "patch"]
|
||||
---
|
||||
kind: ClusterRoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: {{ include "rawfile-localpv.fullname" . }}-resizer
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "rawfile-localpv.fullname" . }}-driver
|
||||
namespace: {{ .Release.Namespace }}
|
||||
roleRef:
|
||||
kind: ClusterRole
|
||||
name: {{ include "rawfile-localpv.fullname" . }}-resizer
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
---
|
||||
kind: ClusterRole
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: {{ include "rawfile-localpv.fullname" . }}-snapshotter
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["events"]
|
||||
verbs: ["list", "watch", "create", "update", "patch"]
|
||||
- apiGroups: ["snapshot.storage.k8s.io"]
|
||||
resources: ["volumesnapshotclasses"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: ["snapshot.storage.k8s.io"]
|
||||
resources: ["volumesnapshotcontents"]
|
||||
verbs: ["create", "get", "list", "watch", "update", "delete", "patch"]
|
||||
- apiGroups: ["snapshot.storage.k8s.io"]
|
||||
resources: ["volumesnapshotcontents/status"]
|
||||
verbs: ["update", "patch"]
|
||||
- apiGroups: ["snapshot.storage.k8s.io"]
|
||||
resources: ["volumesnapshots"]
|
||||
verbs: ["get", "list", "watch", "update", "patch", "delete"]
|
||||
- apiGroups: ["snapshot.storage.k8s.io"]
|
||||
resources: ["volumesnapshots/status"]
|
||||
verbs: ["update", "patch"]
|
||||
# Leader election for the snapshot-controller.
|
||||
- apiGroups: ["coordination.k8s.io"]
|
||||
resources: ["leases"]
|
||||
verbs: ["get", "watch", "list", "delete", "update", "create"]
|
||||
---
|
||||
kind: ClusterRoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: {{ include "rawfile-localpv.fullname" . }}-snapshotter
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "rawfile-localpv.fullname" . }}-driver
|
||||
namespace: {{ .Release.Namespace }}
|
||||
roleRef:
|
||||
kind: ClusterRole
|
||||
name: {{ include "rawfile-localpv.fullname" . }}-snapshotter
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
---
|
||||
{{ $analytics := false }}
|
||||
{{- if kindIs "bool" .Values.global.analytics.enabled }}
|
||||
{{- $analytics = .Values.global.analytics.enabled }}
|
||||
{{- else }}
|
||||
{{- $analytics = .Values.analytics.enabled }}
|
||||
{{- end }}
|
||||
{{- if $analytics }}
|
||||
kind: ClusterRole
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: {{ include "rawfile-localpv.fullname" . }}-analytics
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["namespaces"]
|
||||
verbs: ["get"]
|
||||
- apiGroups: [""]
|
||||
resources: ["configmaps"]
|
||||
verbs: ["create", "get", "update", "patch"]
|
||||
---
|
||||
kind: ClusterRoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: {{ include "rawfile-localpv.fullname" . }}-analytics
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "rawfile-localpv.fullname" . }}-driver
|
||||
namespace: {{ .Release.Namespace }}
|
||||
roleRef:
|
||||
kind: ClusterRole
|
||||
name: {{ include "rawfile-localpv.fullname" . }}-analytics
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
---
|
||||
{{- end }}
|
||||
@@ -0,0 +1,21 @@
|
||||
{{- if and .Values.auth.enabled (not .Values.auth.secretName) }}
|
||||
{{- $secret_name := printf "%s-secrets" (include "rawfile-localpv.fullname" .) }}
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
labels:
|
||||
{{- include "rawfile-localpv.labels" . | nindent 4 }}
|
||||
name: {{ $secret_name }}
|
||||
type: Opaque
|
||||
data:
|
||||
{{- $old_sec := lookup "v1" "Secret" .Release.Namespace $secret_name }}
|
||||
{{- if not .Values.auth.token }}
|
||||
{{- if or (not $old_sec) (not $old_sec.data) }}
|
||||
internal-signature: {{ randAlphaNum 32 | b64enc }}
|
||||
{{- else }}
|
||||
internal-signature: {{ index $old_sec.data "internal-signature" }}
|
||||
{{- end }}
|
||||
{{- else }}
|
||||
internal-signature: {{ .Values.auth.token }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,16 @@
|
||||
{{- $vals := .Values }}
|
||||
{{- range $class := .Values.snapshotClasses }}
|
||||
{{- if $class.enabled }}
|
||||
apiVersion: snapshot.storage.k8s.io/v1
|
||||
kind: VolumeSnapshotClass
|
||||
metadata:
|
||||
name: {{ $class.name }}
|
||||
annotations:
|
||||
"helm.sh/hook": post-install,post-upgrade
|
||||
"helm.sh/hook-weight": "-5"
|
||||
{{- if $class.isDefault }}snapshot.storage.kubernetes.io/is-default-class: "true"{{ end }}
|
||||
driver: {{ $vals.provisionerName }}
|
||||
deletionPolicy: {{ $class.deletionPolicy }}
|
||||
---
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,29 @@
|
||||
{{- $vals := .Values }}
|
||||
{{- range $class := .Values.storageClasses }}
|
||||
{{- if $class.enabled }}
|
||||
apiVersion: storage.k8s.io/v1
|
||||
kind: StorageClass
|
||||
metadata:
|
||||
name: {{ $class.name }}
|
||||
annotations:
|
||||
"helm.sh/hook": post-install,post-upgrade
|
||||
"helm.sh/hook-weight": "-5"
|
||||
{{ if $class.isDefault }}storageclass.kubernetes.io/is-default-class: "true"{{ end }}
|
||||
provisioner: {{ $vals.provisionerName }}
|
||||
reclaimPolicy: {{ $class.reclaimPolicy }}
|
||||
volumeBindingMode: {{ $class.volumeBindingMode }}
|
||||
allowVolumeExpansion: {{ $class.allowVolumeExpansion }}
|
||||
mountOptions:
|
||||
{{- toYaml ($class.mountOptions | default (list)) | nindent 2 }}
|
||||
parameters:
|
||||
csi.storage.k8s.io/fstype: {{ $class.fsType | default "ext4" }}
|
||||
thinProvision: {{ $class.thinProvision | default "false" | toString | quote }}
|
||||
formatOptions: {{ ($class.formatOptions | default (list)) | join " " | quote }}
|
||||
copyOnWrite: {{ $class.copyOnWrite | toString | quote }}
|
||||
freezeFs: {{ $class.freezeFs | default "false" | toString | quote }}
|
||||
{{- with $class.storagePool }}
|
||||
storagePool: {{ . }}
|
||||
{{- end }}
|
||||
---
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
Reference in New Issue
Block a user