Skip to content

Commit b6d569d

Browse files
yaroslavborbatyaroslavborbat
committed
impl
Signed-off-by: Yaroslav Borbat <yaroslav.borbat@flant.com>
1 parent 7c5fc9e commit b6d569d

2 files changed

Lines changed: 22 additions & 67 deletions

File tree

‎templates/_hostnetwork_ports.tpl‎

Lines changed: 16 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -10,40 +10,38 @@ Port map:
1010
1111
virt-handler (kube-api-rewriter runs as its sidecar):
1212
4135-4199 virt-handler: live-migration tunnels (KubeVirt migration range).
13-
4100 virt-handler: Healthz port (--port flag).
13+
4100 virt-handler: healthz and Prometheus metrics (--port flag), kube-rbac-proxy implemented natively.
1414
4101 virt-handler: Console server port (--console-server-port flag).
15-
4102 virt-handler: Prometheus metrics (--metrics-port flag).
16-
4103 kube-api-rewriter sidecar: Prometheus metrics (MONITORING_BIND_ADDRESS), bound to pod IP.
15+
4102 kube-api-rewriter sidecar: Prometheus metrics (MONITORING_BIND_ADDRESS), bound to pod IP.
1716
liveness and readiness probes (/proxy/healthz, /proxy/readyz).
18-
4104 kube-api-rewriter sidecar: pprof (PPROF_BIND_ADDRESS), bound to pod IP, debug mode only.
19-
4105 kube-api-rewriter sidecar: Kubernetes API proxy (CLIENT_PROXY_PORT),
17+
4103 kube-api-rewriter sidecar: pprof (PPROF_BIND_ADDRESS), bound to pod IP, debug mode only.
18+
4104 kube-api-rewriter sidecar: Kubernetes API proxy (CLIENT_PROXY_PORT),
2019
virt-handler connects here instead of the real API server.
2120
2221
vm-route-forge:
23-
4106 vm-route-forge: liveness and readiness probes (HEALTH_PROBE_BIND_ADDRESS).
24-
4107 vm-route-forge: pprof (PPROF_BIND_ADDRESS), debug mode only.
22+
4105 vm-route-forge: liveness and readiness probes (HEALTH_PROBE_BIND_ADDRESS).
23+
4106 vm-route-forge: pprof (PPROF_BIND_ADDRESS), debug mode only.
2524
2625
virtualization-dra:
27-
4108 virtualization-dra: gRPC liveness and readiness probes.
26+
4107 virtualization-dra: gRPC liveness and readiness probes.
2827
4280 virtualization-dra: USB/IP daemon (--usbipd-port flag).
2928
*/ -}}
3029

3130
{{- /* virt-handler */ -}}
32-
{{- define "virt_handler.migration_port_first" -}}4100{{- end -}}
33-
{{- define "virt_handler.migration_port_last" -}}4148{{- end -}}
31+
{{- define "virt_handler.migration_port_first" -}}4135{{- end -}}
32+
{{- define "virt_handler.migration_port_last" -}}4199{{- end -}}
3433

3534
{{- define "virt_handler.port" -}}4100{{- end -}}
3635
{{- define "virt_handler.console_server_port" -}}4101{{- end -}}
37-
{{- define "virt_handler.metrics_port" -}}4102{{- end -}}
38-
{{- define "virt_handler.rewriter_healthz_port" -}}4103{{- end -}}
39-
{{- define "virt_handler.rewriter_monitoring_port" -}}4103{{- end -}}
40-
{{- define "virt_handler.rewriter_pprof_port" -}}4104{{- end -}}
41-
{{- define "virt_handler.rewriter_proxy_port" -}}4105{{- end -}}
36+
{{- define "virt_handler.rewriter_healthz_port" -}}4102{{- end -}}
37+
{{- define "virt_handler.rewriter_monitoring_port" -}}4102{{- end -}}
38+
{{- define "virt_handler.rewriter_pprof_port" -}}4103{{- end -}}
39+
{{- define "virt_handler.rewriter_proxy_port" -}}4104{{- end -}}
4240

4341
{{- /* vm-route-forge */ -}}
44-
{{- define "vm_route_forge.health_port" -}}4106{{- end -}}
45-
{{- define "vm_route_forge.pprof_port" -}}4157{{- end -}}
42+
{{- define "vm_route_forge.health_port" -}}4105{{- end -}}
43+
{{- define "vm_route_forge.pprof_port" -}}4106{{- end -}}
4644

4745
{{- /* virtualization-dra */ -}}
48-
{{- define "virtualization_dra.health_port" -}}4108{{- end -}}
46+
{{- define "virtualization_dra.health_port" -}}4107{{- end -}}
4947
{{- define "virtualization_dra.usbipd_port" -}}4280{{- end -}}

‎templates/kubevirt/kubevirt.yaml‎

Lines changed: 6 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -75,21 +75,6 @@ spec:
7575
virtualMachineOptions:
7676
disableSerialConsoleLog: {}
7777
customizeComponents:
78-
flags:
79-
{{- if ne "delve/virt-api" ($delve | dig "debug" "component" "<missing>") }}
80-
api:
81-
metrics-listen: 127.0.0.1
82-
metrics-port: "8080"
83-
{{- end }}
84-
{{- if ne "delve/virt-controller" ($delve | dig "debug" "component" "<missing>") }}
85-
controller:
86-
metrics-listen: 127.0.0.1
87-
metrics-port: "8080"
88-
{{- end }}
89-
{{- if ne "delve/virt-handler" ($delve | dig "debug" "component" "<missing>") }}
90-
handler:
91-
metrics-port: {{ include "virt_handler.metrics_port" . | quote }}
92-
{{- end }}
9378
patches:
9479
# Add node placement settings for virt-api, virt-controller, virt-operator, virt-handler.
9580
- resourceType: Deployment
@@ -210,33 +195,6 @@ spec:
210195
patch: {{ include "kube_api_rewriter.pod_spec_strategic_patch_json" (list . "virt-handler" $virtHandlerRewriterSettings) }}
211196
type: strategic
212197

213-
# Add kube-api-rewriter sidecar containers to virt-controller, virt-api, virt-handler.
214-
{{- $kubeRbacProxySettings := dict }}
215-
{{- $_ := set $kubeRbacProxySettings "runAsUserNobody" true }}
216-
{{- $_ := set $kubeRbacProxySettings "ignorePaths" "/proxy/healthz,/proxy/readyz" }}
217-
{{- $_ := set $kubeRbacProxySettings "upstreams" (list
218-
(dict "upstream" "http://127.0.0.1:9090/metrics" "path" "/proxy/metrics" "name" "kube-api-rewriter")
219-
(dict "upstream" "http://127.0.0.1:8080/metrics" "path" "/metrics" "name" "virt-controller")
220-
(dict "upstream" "http://127.0.0.1:9090/healthz" "path" "/proxy/healthz" "name" "kube-api-rewriter")
221-
(dict "upstream" "http://127.0.0.1:9090/readyz" "path" "/proxy/readyz" "name" "kube-api-rewriter")
222-
) }}
223-
- resourceName: virt-controller
224-
resourceType: Deployment
225-
patch: {{ include "kube_rbac_proxy.pod_spec_strategic_patch_json" (tuple . $kubeRbacProxySettings) }}
226-
type: strategic
227-
228-
{{- $_ := set $kubeRbacProxySettings "ignorePaths" "/proxy/healthz,/proxy/readyz" }}
229-
{{- $_ := set $kubeRbacProxySettings "upstreams" (list
230-
(dict "upstream" "http://127.0.0.1:9090/metrics" "path" "/proxy/metrics" "name" "kube-api-rewriter")
231-
(dict "upstream" "http://127.0.0.1:8080/metrics" "path" "/metrics" "name" "virt-api")
232-
(dict "upstream" "http://127.0.0.1:9090/healthz" "path" "/proxy/healthz" "name" "kube-api-rewriter")
233-
(dict "upstream" "http://127.0.0.1:9090/readyz" "path" "/proxy/readyz" "name" "kube-api-rewriter")
234-
) }}
235-
- resourceName: virt-api
236-
resourceType: Deployment
237-
patch: {{ include "kube_rbac_proxy.pod_spec_strategic_patch_json" (tuple . $kubeRbacProxySettings) }}
238-
type: strategic
239-
240198
# Add rewriter proxy container port to Services used by webhook configurations.
241199
# First need to set name for existing port to make strategic patch works later.
242200
- resourceName: virt-api
@@ -331,10 +289,10 @@ spec:
331289
resourceName: virt-handler
332290
patch: {{ include "pod_spec_priority_class_name_patch" $priorityClassName }}
333291
type: strategic
334-
# Patch service for https-metrics
292+
# Patch service to target the main virt-handler port
335293
- resourceType: Service
336294
resourceName: kubevirt-prometheus-metrics
337-
patch: '[{"op": "replace", "path": "/spec/ports/0/targetPort", "value": "https-metrics"}]'
295+
patch: '[{"op": "replace", "path": "/spec/ports/0/targetPort", "value": "virt-handler"}]'
338296
type: json
339297

340298
# Additional environment variables for virt-controller.
@@ -357,11 +315,10 @@ env:
357315
patch: '{"spec":{"template":{"metadata":{"labels":{"security.deckhouse.io/security-policy-exception": "virt-handler-ds"}}}}}'
358316
type: strategic
359317

360-
# Expose virt-handler ports: health API (--port), console server (--console-server-port),
361-
# and HTTPS metrics so kubevirt-prometheus-metrics Service can resolve targetPort: https-metrics.
318+
# Expose virt-handler ports: health API (--port) and console server (--console-server-port).
362319
- resourceName: virt-handler
363320
resourceType: DaemonSet
364-
patch: '{"spec":{"template":{"spec":{"containers":[{"name":"virt-handler","ports":[{"containerPort":{{ include "virt_handler.port" . | int }},"name":"virt-handler","protocol":"TCP"},{"containerPort":{{ include "virt_handler.console_server_port" . | int }},"name":"console","protocol":"TCP"},{"containerPort":{{ include "virt_handler.metrics_port" . | int }},"name":"https-metrics","protocol":"TCP"}]}]}}}}'
321+
patch: '{"spec":{"template":{"spec":{"containers":[{"name":"virt-handler","ports":[{"containerPort":{{ include "virt_handler.port" . | int }},"name":"virt-handler","protocol":"TCP"},{"containerPort":{{ include "virt_handler.console_server_port" . | int }},"name":"console","protocol":"TCP"}]}]}}}}'
365322
type: strategic
366323

367324
# Rewrite virt-api args, replacing the default ports baked into the image.
@@ -375,13 +332,13 @@ env:
375332
# This is required because customizeComponents.flags only appends flags and cannot replace existing ones.
376333
- resourceName: virt-handler
377334
resourceType: DaemonSet
378-
patch: '{"spec":{"template":{"spec":{"containers":[{"name":"virt-handler","args":["--port","{{ include "virt_handler.port" . }}","--hostname-override","$(NODE_NAME)","--pod-ip-address","$(MY_POD_IP)","--max-metric-requests","3","--console-server-port","{{ include "virt_handler.console_server_port" . }}","--metrics-port","{{ include "virt_handler.metrics_port" . }}","--migration-port-range-enabled","true","--migration-port-range-first","{{ include "virt_handler.migration_port_first" . }}","--migration-port-range-last","{{ include "virt_handler.migration_port_last" . }}","--graceful-shutdown-seconds","315","-v","2"]}]}}}}'
335+
patch: '{"spec":{"template":{"spec":{"containers":[{"name":"virt-handler","args":["--port","{{ include "virt_handler.port" . }}","--hostname-override","$(NODE_NAME)","--pod-ip-address","$(MY_POD_IP)","--max-metric-requests","3","--console-server-port","{{ include "virt_handler.console_server_port" . }}","--migration-port-range-enabled","true","--migration-port-range-first","{{ include "virt_handler.migration_port_first" . }}","--migration-port-range-last","{{ include "virt_handler.migration_port_last" . }}","--graceful-shutdown-seconds","315","-v","2"]}]}}}}'
379336
type: strategic
380337

381338
# Override virt-handler liveness and readiness probes to use the new host-network port.
382339
- resourceName: virt-handler
383340
resourceType: DaemonSet
384-
patch: '{"spec":{"template":{"spec":{"containers":[{"name":"virt-handler","livenessProbe":{"httpGet":{"path":"/healthz","port":{{ include "virt_handler.port" . | int }},"scheme":"HTTP"},"failureThreshold":3,"initialDelaySeconds":15,"periodSeconds":45,"successThreshold":1,"timeoutSeconds":10},"readinessProbe":{"httpGet":{"path":"/healthz","port":{{ include "virt_handler.port" . | int }},"scheme":"HTTP"},"failureThreshold":3,"initialDelaySeconds":15,"periodSeconds":20,"successThreshold":1,"timeoutSeconds":10}}]}}}}'
341+
patch: '{"spec":{"template":{"spec":{"containers":[{"name":"virt-handler","livenessProbe":{"httpGet":{"path":"/healthz","port":{{ include "virt_handler.port" . | int }},"scheme":"HTTPS"},"failureThreshold":3,"initialDelaySeconds":15,"periodSeconds":45,"successThreshold":1,"timeoutSeconds":10},"readinessProbe":{"httpGet":{"path":"/healthz","port":{{ include "virt_handler.port" . | int }},"scheme":"HTTPS"},"failureThreshold":3,"initialDelaySeconds":15,"periodSeconds":20,"successThreshold":1,"timeoutSeconds":10}}]}}}}'
385342
type: strategic
386343

387344
# Change host path for directory with capabilities xml files. We have custom qemu with different

0 commit comments

Comments
 (0)