diff --git a/docs/how-to/create-active-active-nfs.rst b/docs/how-to/create-active-active-nfs.rst new file mode 100644 index 00000000..0cf56340 --- /dev/null +++ b/docs/how-to/create-active-active-nfs.rst @@ -0,0 +1,87 @@ +.. _howto_active_active_nfs: + +How to create an active-active NFS Ganesha service +=================================================== + +This guide explains how to create a highly available, active-active NFS Ganesha service using the MicroCeph ingress feature. The ingress service uses keepalived and haproxy to provide a virtual IP (VIP) that load balances traffic across multiple NFS Ganesha instances. You can also use this feature to provide ingress for other services. + +Prerequisites +------------- + +- A running MicroCeph cluster with at least two nodes. +- The ``nfs`` feature enabled on at least two nodes, each belonging to the same NFS cluster. + +1. Enable NFS Ganesha services +------------------------------- + +First, enable the NFS Ganesha service on two or more nodes. Make sure to use the same ``--cluster-id`` for each instance to group them into a single NFS cluster. + +On the first node: + +.. code-block:: bash + + microceph enable nfs --cluster-id nfs-ha --target-node node1 + +On the second node: + +.. code-block:: bash + + microceph enable nfs --cluster-id nfs-ha --target-node node2 + +This will create two NFS Ganesha instances that are part of the ``nfs-ha`` cluster. + +2. Enable the ingress service +----------------------------- + +Next, enable the ingress service. This will create a VIP that floats between the nodes where the ingress service is enabled and load balances traffic to the target service instances. + +.. code-block:: bash + + microceph enable ingress --service-id ingress-nfs-ha \ + --vip-address 192.168.1.100 \ + --vip-interface eth0 \ + --target nfs.nfs-ha \ + --target-node node1 + +Repeat the same command for any other node where you want to run the ingress service (typically the same nodes as your target service). + +.. code-block:: bash + + microceph enable ingress --service-id ingress-nfs-ha \ + --vip-address 192.168.1.100 \ + --vip-interface eth0 \ + --target nfs.nfs-ha \ + --target-node node2 + +- ``--service-id``: A unique name for this ingress service instance. +- ``--vip-address``: The virtual IP address that clients will connect to. +- ``--vip-interface``: The network interface on which the VIP will be active. +- ``--target``: The service to provide ingress for, in the format ``.``. In this case, we are targeting the ``nfs`` service with the ID ``nfs-ha``. + +MicroCeph will automatically generate and manage the necessary VRRP password and router ID for the keepalived configuration. + +Creating multiple ingress services +---------------------------------- + +You can run the ``enable ingress`` command multiple times with different ``--service-id`` values to create multiple, independent ingress services. This is useful for providing VIPs for different services or different clusters of the same service. + +3. Verify the setup +------------------- + +You should now be able to mount the NFS share using the virtual IP address: + +.. code-block:: bash + + mount -t nfs -o port=2049 192.168.1.100:/ /mnt/nfs + +4. Disable the ingress service +------------------------------ + +To disable an ingress service, use the ``disable ingress`` command with the service ID: + +.. code-block:: bash + + microceph disable ingress --service-id ingress-nfs-ha --target-node node1 + microceph disable ingress --service-id ingress-nfs-ha --target-node node2 + +This will remove the configuration for this specific ingress service and reload the ingress service. If no other ingress services are configured, the `keepalived` and `haproxy` daemons will be stopped. diff --git a/docs/how-to/index.rst b/docs/how-to/index.rst index edc2b831..ef973fc8 100644 --- a/docs/how-to/index.rst +++ b/docs/how-to/index.rst @@ -30,6 +30,7 @@ configuration of metrics, alerts and other service instances. enable-metrics enable-alerts enable-service-instances + create-active-active-nfs Interacting with your cluster ----------------------------- diff --git a/microceph/api/servers.go b/microceph/api/servers.go index 002072f5..3889fef2 100644 --- a/microceph/api/servers.go +++ b/microceph/api/servers.go @@ -24,6 +24,7 @@ var Servers = map[string]rest.Server{ mgrServiceCmd, monServiceCmd, nfsServiceCmd, + ingressServiceCmd, poolsOpCmd, rgwServiceCmd, rbdMirroServiceCmd, diff --git a/microceph/api/services.go b/microceph/api/services.go index 4e66ebda..bdc0cae3 100644 --- a/microceph/api/services.go +++ b/microceph/api/services.go @@ -75,6 +75,12 @@ var rgwServiceCmd = rest.Endpoint{ Put: rest.EndpointAction{Handler: cmdEnableServicePut, ProxyTarget: true}, Delete: rest.EndpointAction{Handler: cmdRGWServiceDelete, ProxyTarget: true}, } + +var ingressServiceCmd = rest.Endpoint{ + Path: "services/ingress", + Put: rest.EndpointAction{Handler: cmdEnableServicePut, ProxyTarget: true}, + Delete: rest.EndpointAction{Handler: cmdIngressDeleteService, ProxyTarget: true}, +} var rbdMirroServiceCmd = rest.Endpoint{ Path: "services/rbd-mirror", Put: rest.EndpointAction{Handler: cmdEnableServicePut, ProxyTarget: true}, @@ -161,6 +167,30 @@ func cmdRestartServicePost(s state.State, r *http.Request) response.Response { return response.EmptySyncResponse } +// cmdIngressDeleteService handles the ingress service deletion. +func cmdIngressDeleteService(s state.State, r *http.Request) response.Response { + var svc types.IngressService + + err := json.NewDecoder(r.Body).Decode(&svc) + if err != nil { + logger.Errorf("failed decoding disable service request: %v", err) + return response.InternalError(err) + } + + if !types.IngressServiceIDRegex.MatchString(svc.ClusterID) { + err := fmt.Errorf("expected service_id to be valid (regex: '%s')", types.IngressServiceIDRegex.String()) + return response.SmartError(err) + } + + err = ceph.DisableIngress(r.Context(), interfaces.CephState{State: s}, svc.ClusterID) + if err != nil { + logger.Errorf("Failed disabling ingress: %v", err) + return response.SmartError(err) + } + + return response.EmptySyncResponse +} + // cmdDeleteService handles service deletion. func cmdDeleteService(s state.State, r *http.Request) response.Response { which := path.Base(r.URL.Path) diff --git a/microceph/api/types/service_ingress.go b/microceph/api/types/service_ingress.go new file mode 100644 index 00000000..ee29833c --- /dev/null +++ b/microceph/api/types/service_ingress.go @@ -0,0 +1,40 @@ +package types + +import ( + "fmt" + "net" + "regexp" + "strings" +) + +// IngressServiceIDRegex is the regular expression that a valid Ingress ServiceID must match. +var IngressServiceIDRegex = regexp.MustCompile(`^[a-zA-Z0-9.\-_]+$`) + +// IngressServicePlacement represents the configuration for an ingress service. +type IngressServicePlacement struct { + ServiceID string `json:"service_id"` + VIPAddress string `json:"vip_address"` + VIPInterface string `json:"vip_interface"` + Target string `json:"target"` +} + +// Validate checks if the IngressServicePlacement has valid fields. +func (isp *IngressServicePlacement) Validate() error { + if !IngressServiceIDRegex.MatchString(isp.ServiceID) { + return fmt.Errorf("expected service_id to be valid (regex: '%s')", IngressServiceIDRegex.String()) + } + if net.ParseIP(isp.VIPAddress) == nil { + return fmt.Errorf("vip_address '%s' could not be parsed", isp.VIPAddress) + } + if isp.VIPInterface == "" { + return fmt.Errorf("vip_interface must be provided") + } + if isp.Target == "" { + return fmt.Errorf("target must be provided") + } + parts := strings.Split(isp.Target, ".") + if len(parts) != 2 { + return fmt.Errorf("target must be in the format .") + } + return nil +} diff --git a/microceph/api/types/services.go b/microceph/api/types/services.go index 9da85cd6..e3b947b7 100644 --- a/microceph/api/types/services.go +++ b/microceph/api/types/services.go @@ -31,6 +31,11 @@ type NFSService struct { ClusterID string `json:"cluster_id" yaml:"cluster_id"` } +// IngressService represents an ingress service that is running on a given node. +type IngressService struct { + ClusterID string `json:"cluster_id"` +} + // NFSClusterIDRegex is a regex for acceptable ClusterIDs. var NFSClusterIDRegex = regexp.MustCompile(`^[\w][\w.-]{1,61}[\w]$`) diff --git a/microceph/ceph/ingress.go b/microceph/ceph/ingress.go new file mode 100644 index 00000000..358e583c --- /dev/null +++ b/microceph/ceph/ingress.go @@ -0,0 +1,255 @@ +package ceph + +import ( + "context" + "crypto/rand" + "database/sql" + "encoding/json" + "fmt" + "math/big" + "net/http" + "os" + "path/filepath" + "strings" + "text/template" + + "github.com/canonical/lxd/shared/api" + "github.com/canonical/lxd/shared/revert" + + "github.com/canonical/microceph/microceph/constants" + "github.com/canonical/microceph/microceph/database" + "github.com/canonical/microceph/microceph/interfaces" + "github.com/canonical/microceph/microceph/logger" +) + +const ( + keepalivedConfigTemplate = ` +{{range .}} +vrrp_instance VI_{{.Placement.ServiceID}} { + state MASTER + interface {{.Placement.VIPInterface}} + virtual_router_id {{.VRRPRouterID}} + priority 101 + advert_int 1 + authentication { + auth_type PASS + auth_pass {{.VRRPPassword}} + } + virtual_ipaddress { + {{.Placement.VIPAddress}} + } +} +{{end}} +` + haproxyConfigTemplate = ` +global + log /dev/log local0 + log /dev/log local1 notice + chroot /var/lib/haproxy + stats socket /run/haproxy/admin.sock mode 660 level admin expose-fd listeners + stats timeout 30s + user haproxy + group haproxy + daemon + +defaults + log global + mode tcp + option tcplog + option dontlognull + timeout connect 5000 + timeout client 50000 + timeout server 50000 + +frontend main + bind *:2049 + mode tcp + {{range .}} + acl is_{{.Placement.ServiceID}} dst {{.Placement.VIPAddress}} + use_backend backend_{{.Placement.ServiceID}} if is_{{.Placement.ServiceID}} + {{end}} + +{{range .}} +backend backend_{{.Placement.ServiceID}} + mode tcp + balance roundrobin + {{range .BackendServers}} + server {{.Name}} {{.Address}} check + {{end}} +{{end}} +` +) + +type backendServer struct { + Name string + Address string +} + +type ingressTemplateData struct { + Placement *database.IngressServiceGroupConfig + BackendServers []backendServer +} + +// EnableIngress enables or updates the ingress service on the host. +func EnableIngress(ctx context.Context, s interfaces.StateInterface, isp *IngressServicePlacement) error { + logger.Debugf("Enabling ingress service with ServiceID '%s'", isp.ServiceID) + + // Check if service group already exists and generate/fetch VRRP params + err := s.ClusterState().Database().Transaction(ctx, func(ctx context.Context, tx *sql.Tx) error { + group, err := database.GetServiceGroup(ctx, tx, "ingress", isp.ServiceID) + if err != nil && !api.StatusErrorCheck(err, http.StatusNotFound) { + return fmt.Errorf("failed to get service group: %w", err) + } + + if group != nil { + // Group exists, this is an update or re-enablement. + // We don't need to do anything here, the config will be regenerated later. + } else { + // Group doesn't exist, generate new VRRP params + password, err := generateRandomString(16) + if err != nil { + return fmt.Errorf("failed to generate VRRP password: %w", err) + } + isp.vrrpPassword = password + + routerID, err := rand.Int(rand.Reader, big.NewInt(254)) + if err != nil { + return fmt.Errorf("failed to generate VRRP router ID: %w", err) + } + isp.vrrpRouterID = int(routerID.Int64()) + 1 // 1-255 + } + return nil + }) + if err != nil { + return err + } + + return regenerateIngressConfigs(ctx, s) +} + +// DisableIngress disables an ingress service instance on the host. +func DisableIngress(ctx context.Context, s interfaces.StateInterface, serviceID string) error { + logger.Debugf("Disabling ingress service with ServiceID '%s'", serviceID) + + // Remove database records. + err := database.GroupedServicesQuery.RemoveForHost(ctx, s, "ingress", serviceID) + if err != nil { + return err + } + + return regenerateIngressConfigs(ctx, s) +} + +func regenerateIngressConfigs(ctx context.Context, s interfaces.StateInterface) error { + // Get all ingress service groups + ingressGroups, err := database.GroupedServicesQuery.GetGroupedServices(ctx, s, database.GroupedServiceFilter{Service: "ingress"}) + if err != nil { + return fmt.Errorf("failed to get ingress service groups: %w", err) + } + + if len(ingressGroups) == 0 { + // No more ingress services, stop the service and remove configs + logger.Debugf("No more ingress services, stopping service.") + err := snapStop("ingress", true) + if err != nil { + return err + } + pathConsts := constants.GetPathConst() + ingressConfDir := filepath.Join(pathConsts.ConfPath, "ingress") + return os.RemoveAll(ingressConfDir) + } + + var allTemplateData []ingressTemplateData + + for _, group := range ingressGroups { + var config database.IngressServiceGroupConfig + err := json.Unmarshal([]byte(group.Config), &config) + if err != nil { + return fmt.Errorf("failed to unmarshal ingress group config for %s: %w", group.GroupID, err) + } + + parts := strings.Split(config.Target, ".") + targetService := parts[0] + targetGroupID := parts[1] + + allServices, err := database.GroupedServicesQuery.GetGroupedServices(ctx, s, database.GroupedServiceFilter{Service: &targetService, GroupID: &targetGroupID}) + if err != nil { + return fmt.Errorf("failed to get target service endpoints for %s: %w", config.Target, err) + } + + var backendServers []backendServer + for _, ts := range allServices { + var nfsInfo database.NFSServiceInfo + err := json.Unmarshal([]byte(ts.Info), &nfsInfo) + if err != nil { + return fmt.Errorf("failed to unmarshal service info for member %s: %w", ts.Member, err) + } + member, err := s.ClusterState().GetMember(ts.Member) + if err != nil { + return fmt.Errorf("failed to get member %s: %w", ts.Member, err) + } + backendServers = append(backendServers, backendServer{Name: ts.Member, Address: fmt.Sprintf("%s:%d", member.Address, nfsInfo.BindPort)}) + } + + allTemplateData = append(allTemplateData, ingressTemplateData{ + Placement: &config, + BackendServers: backendServers, + }) + } + + pathConsts := constants.GetPathConst() + ingressConfDir := filepath.Join(pathConsts.ConfPath, "ingress") + err = os.MkdirAll(ingressConfDir, 0755) + if err != nil && !os.IsExist(err) { + return err + } + + // Create keepalived.conf + keepalivedConfPath := filepath.Join(ingressConfDir, "keepalived.conf") + err = writeTemplate(keepalivedConfPath, keepalivedConfigTemplate, allTemplateData) + if err != nil { + return fmt.Errorf("failed to write keepalived config: %w", err) + } + + // Create haproxy.cfg + haproxyConfPath := filepath.Join(ingressConfDir, "haproxy.cfg") + err = writeTemplate(haproxyConfPath, haproxyConfigTemplate, allTemplateData) + if err != nil { + return fmt.Errorf("failed to write haproxy config: %w", err) + } + + // Start or reload the ingress service. + err = snapCheckActive("ingress") + if err == nil { + return snapRestart("ingress", true) + } + return snapStart("ingress", true) +} + +func writeTemplate(path, tmpl string, data any) error { + t, err := template.New(filepath.Base(path)).Parse(tmpl) + if err != nil { + return err + } + + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + + return t.Execute(f, data) +} + +func generateRandomString(n int) (string, error) { + const letters = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + ret := make([]byte, n) + for i := 0; i < n; i++ { + num, err := rand.Int(rand.Reader, big.NewInt(int64(len(letters)))) + if err != nil { + return "", err + } + ret[i] = letters[num.Int64()] + } + return string(ret), nil +} diff --git a/microceph/ceph/service_placement_ingress.go b/microceph/ceph/service_placement_ingress.go new file mode 100644 index 00000000..a6a72e2b --- /dev/null +++ b/microceph/ceph/service_placement_ingress.go @@ -0,0 +1,66 @@ +package ceph + +import ( + "context" + "encoding/json" + "fmt" + "net" + + "github.com/canonical/microceph/microceph/api/types" + "github.com/canonical/microceph/microceph/database" + "github.com/canonical/microceph/microceph/interfaces" +) + +type IngressServicePlacement struct { + types.IngressServicePlacement + + // These fields will be populated by EnableIngress + vrrpPassword string + vrrpRouterID int +} + +func (isp *IngressServicePlacement) PopulateParams(s interfaces.StateInterface, payload string) error { + err := json.Unmarshal([]byte(payload), &isp.IngressServicePlacement) + if err != nil { + return err + } + return isp.Validate() +} + +func (isp *IngressServicePlacement) HospitalityCheck(s interfaces.StateInterface) error { + ifaces, err := net.Interfaces() + if err != nil { + return fmt.Errorf("failed to get network interfaces: %w", err) + } + + for _, i := range ifaces { + if i.Name == isp.VIPInterface { + return genericHospitalityCheck("ingress") + } + } + + return fmt.Errorf("network interface '%s' not found", isp.VIPInterface) +} + +// ServiceInit will call EnableIngress, which will generate or fetch VRRP params +// and store them in the isp struct for DbUpdate to use. +func (isp *IngressServicePlacement) ServiceInit(ctx context.Context, s interfaces.StateInterface) error { + return EnableIngress(ctx, s, isp) +} + +func (isp *IngressServicePlacement) PostPlacementCheck(s interfaces.StateInterface) error { + return genericPostPlacementCheck("ingress") +} + +func (isp *IngressServicePlacement) DbUpdate(ctx context.Context, s interfaces.StateInterface) error { + groupConfig := database.IngressServiceGroupConfig{ + VIPAddress: isp.VIPAddress, + VIPInterface: isp.VIPInterface, + Target: isp.Target, + VRRPPassword: isp.vrrpPassword, + VRRPRouterID: isp.vrrpRouterID, + } + serviceInfo := database.IngressServiceInfo{} + + return database.GroupedServicesQuery.AddNew(ctx, s, "ingress", isp.ServiceID, groupConfig, serviceInfo) +} diff --git a/microceph/ceph/services_placement.go b/microceph/ceph/services_placement.go index d0620283..b994519d 100644 --- a/microceph/ceph/services_placement.go +++ b/microceph/ceph/services_placement.go @@ -30,6 +30,7 @@ func GetServicePlacementTable() map[string](PlacementIntf) { "mgr": &GenericServicePlacement{"mgr"}, "mds": &GenericServicePlacement{"mds"}, "nfs": &NFSServicePlacement{}, + "ingress": &IngressServicePlacement{}, "rgw": &RgwServicePlacement{}, "rbd-mirror": &ClientServicePlacement{"rbd-mirror"}, "cephfs-mirror": &ClientServicePlacement{"cephfs-mirror"}, diff --git a/microceph/client/services.go b/microceph/client/services.go index 1a3253c6..8fd7f6d4 100644 --- a/microceph/client/services.go +++ b/microceph/client/services.go @@ -45,6 +45,22 @@ func DeleteService(ctx context.Context, c *client.Client, target string, service return nil } +// DeleteIngressService requests MicroCeph to deconfigure the ingress service on a given target node. +func DeleteIngressService(ctx context.Context, c *client.Client, target string, svc *types.IngressService) error { + queryCtx, cancel := context.WithTimeout(ctx, time.Second*120) + defer cancel() + + // Send this request to target. + c = c.UseTarget(target) + + err := c.Query(queryCtx, "DELETE", types.ExtendedPathPrefix, api.NewURL().Path("services", "ingress"), svc, nil) + if err != nil { + return fmt.Errorf("failed deleting ingress service: %w", err) + } + + return nil +} + // DeleteNFSService requests MicroCeph to deconfigure the NFS service on a given target node. func DeleteNFSService(ctx context.Context, c *client.Client, target string, svc *types.NFSService) error { queryCtx, cancel := context.WithTimeout(ctx, time.Second*120) diff --git a/microceph/cmd/microceph/disable.go b/microceph/cmd/microceph/disable.go index 4f30c584..31ccc5b1 100644 --- a/microceph/cmd/microceph/disable.go +++ b/microceph/cmd/microceph/disable.go @@ -26,6 +26,10 @@ func (c *cmdDisable) Command() *cobra.Command { disableCephfsMirror := cmdDisableCephFSMirror{common: c.common} cmd.AddCommand(disableCephfsMirror.Command()) + // Disable ingress + disableIngressCmd := cmdDisableIngress{common: c.common} + cmd.AddCommand(disableIngressCmd.Command()) + // Workaround for subcommand usage errors. See: https://github.com/spf13/cobra/issues/706 cmd.Args = cobra.NoArgs cmd.Run = func(cmd *cobra.Command, args []string) { _ = cmd.Usage() } diff --git a/microceph/cmd/microceph/disable_ingress.go b/microceph/cmd/microceph/disable_ingress.go new file mode 100644 index 00000000..b6b47fa4 --- /dev/null +++ b/microceph/cmd/microceph/disable_ingress.go @@ -0,0 +1,54 @@ +package main + +import ( + "context" + "fmt" + + "github.com/canonical/microcluster/v2/microcluster" + "github.com/spf13/cobra" + + "github.com/canonical/microceph/microceph/api/types" + "github.com/canonical/microceph/microceph/client" +) + +type cmdDisableIngress struct { + common *CmdControl + flagServiceID string + flagTargetNode string +} + +func (c *cmdDisableIngress) Command() *cobra.Command { + cmd := &cobra.Command{ + Use: "ingress --service-id [--target-node ]", + Short: "Disable the ingress service on a target node", + RunE: c.Run, + } + cmd.PersistentFlags().StringVar(&c.flagServiceID, "service-id", "", "Ingress Service ID") + cmd.PersistentFlags().StringVar(&c.flagTargetNode, "target-node", "", "Node to disable ingress on (default: this server)") + return cmd +} + +// Run handles the disable ingress command. +func (c *cmdDisableIngress) Run(cmd *cobra.Command, args []string) error { + if !types.IngressServiceIDRegex.MatchString(c.flagServiceID) { + return fmt.Errorf("please provide a valid service ID using the `--service-id` flag") + } + + m, err := microcluster.App(microcluster.Args{StateDir: c.common.FlagStateDir}) + if err != nil { + return err + } + + cli, err := m.LocalClient() + if err != nil { + return err + } + + svc := &types.IngressService{ClusterID: c.flagServiceID} + err = client.DeleteIngressService(context.Background(), cli, c.flagTargetNode, svc) + if err != nil { + return err + } + + return nil +} diff --git a/microceph/cmd/microceph/enable.go b/microceph/cmd/microceph/enable.go index 074ad19a..93cef7eb 100644 --- a/microceph/cmd/microceph/enable.go +++ b/microceph/cmd/microceph/enable.go @@ -31,6 +31,10 @@ func (c *cmdEnable) Command() *cobra.Command { cmd.AddCommand(enableRbdMirrorCmd.Command()) cmd.AddCommand(enableFsMirrorCmd.Command()) + // Enable Ingress + enableIngressCmd := cmdEnableIngress{common: c.common} + cmd.AddCommand(enableIngressCmd.Command()) + // Workaround for subcommand usage errors. See: https://github.com/spf13/cobra/issues/706 cmd.Args = cobra.NoArgs cmd.Run = func(cmd *cobra.Command, args []string) { _ = cmd.Usage() } diff --git a/microceph/cmd/microceph/enable_ingress.go b/microceph/cmd/microceph/enable_ingress.go new file mode 100644 index 00000000..7bd79ffc --- /dev/null +++ b/microceph/cmd/microceph/enable_ingress.go @@ -0,0 +1,74 @@ +package main + +import ( + "context" + "encoding/json" + + "github.com/canonical/microcluster/v2/microcluster" + "github.com/spf13/cobra" + + "github.com/canonical/microceph/microceph/api/types" + "github.com/canonical/microceph/microceph/client" +) + +type cmdEnableIngress struct { + common *CmdControl + wait bool + flagServiceID string + flagVIPAddress string + flagVIPInterface string + flagTarget string + flagTargetNode string +} + +func (c *cmdEnableIngress) Command() *cobra.Command { + cmd := &cobra.Command{ + Use: "ingress --service-id --vip-address --vip-interface --target ", + Short: "Enable the ingress service on a target node", + RunE: c.Run, + } + cmd.PersistentFlags().StringVar(&c.flagServiceID, "service-id", "", "Ingress Service ID") + cmd.PersistentFlags().StringVar(&c.flagVIPAddress, "vip-address", "", "Virtual IP address") + cmd.PersistentFlags().StringVar(&c.flagVIPInterface, "vip-interface", "", "Network interface for the VIP") + cmd.PersistentFlags().StringVar(&c.flagTarget, "target", "", "The service to provide ingress for (e.g. nfs.my-nfs-cluster)") + cmd.PersistentFlags().StringVar(&c.flagTargetNode, "target-node", "", "Node to enable ingress on (default: this server)") + cmd.Flags().BoolVar(&c.wait, "wait", true, "Wait for ingress service to be up") + return cmd +} + +// Run handles the enable ingress command. +func (c *cmdEnableIngress) Run(cmd *cobra.Command, args []string) error { + obj := types.IngressServicePlacement{ + ServiceID: c.flagServiceID, + VIPAddress: c.flagVIPAddress, + VIPInterface: c.flagVIPInterface, + Target: c.flagTarget, + } + + if err := obj.Validate(); err != nil { + return err + } + + jsp, err := json.Marshal(obj) + if err != nil { + return err + } + + req := &types.EnableService{ + Name: "ingress", + Wait: c.wait, + Payload: string(jsp[:]), + } + + m, err := microcluster.App(microcluster.Args{StateDir: c.common.FlagStateDir}) + if err != nil { + return err + } + + cli, err := m.LocalClient() + if err != nil { + return err + } + + return client.SendServicePlacementReq(context.Background(), cli, req, c.flagTargetNode) +} diff --git a/microceph/database/grouped_service.go b/microceph/database/grouped_service.go index 6b443de6..4d90a702 100644 --- a/microceph/database/grouped_service.go +++ b/microceph/database/grouped_service.go @@ -28,6 +28,7 @@ type GroupedService struct { GroupID string `db:"primary=yes&sql=service_groups.group_id"` Member string `db:"primary=yes&join=core_cluster_members.name&joinon=grouped_services.member_id"` Info string + Config string `db:"join=service_groups.config&joinon=grouped_services.service_group_id"` } // GroupedServiceFilter is a required struct for use with lxd-generate. It is used for filtering fields on database fetches. @@ -42,3 +43,7 @@ type NFSServiceInfo struct { BindAddress string `json:"bind_address"` BindPort uint `json:"bind_port"` } + +// IngressServiceInfo is a struct containing GroupedService information for the ingress service. +type IngressServiceInfo struct { +} diff --git a/microceph/database/service_group.go b/microceph/database/service_group.go index a2c215e7..ba2cf197 100644 --- a/microceph/database/service_group.go +++ b/microceph/database/service_group.go @@ -37,3 +37,12 @@ type ServiceGroupFilter struct { type NFSServiceGroupConfig struct { V4MinVersion uint `json:"v4_min_version"` } + +// IngressServiceGroupConfig is a struct containing a ServiceGroup's configuration for the ingress service. +type IngressServiceGroupConfig struct { + VIPAddress string `json:"vip_address"` + VIPInterface string `json:"vip_interface"` + Target string `json:"target"` + VRRPPassword string `json:"vrrp_password"` + VRRPRouterID int `json:"vrrp_router_id"` +} diff --git a/microceph/snapcraft/commands/ingress.reload b/microceph/snapcraft/commands/ingress.reload new file mode 100755 index 00000000..afff48b6 --- /dev/null +++ b/microceph/snapcraft/commands/ingress.reload @@ -0,0 +1,13 @@ +#!/bin/bash + +HAPROXY_PID_FILE="${SNAP_DATA}/run/ingress/haproxy.pid" +KEEPALIVED_PID_FILE="${SNAP_DATA}/run/ingress/keepalived.pid" +HAPROXY_CONF="${SNAP_DATA}/conf/ingress/haproxy.cfg" + +if [ -f "${HAPROXY_PID_FILE}" ]; then + haproxy -f "${HAPROXY_CONF}" -p "${HAPROXY_PID_FILE}" -sf $(cat ${HAPROXY_PID_FILE}) +fi + +if [ -f "${KEEPALIVED_PID_FILE}" ]; then + kill -HUP $(cat ${KEEPALIVED_PID_FILE}) +fi diff --git a/microceph/snapcraft/commands/ingress.start b/microceph/snapcraft/commands/ingress.start new file mode 100755 index 00000000..271cf3b3 --- /dev/null +++ b/microceph/snapcraft/commands/ingress.start @@ -0,0 +1,37 @@ +#!/bin/bash + +. "${SNAP}/commands/common" + +limits + +wait_for_config + +KEEPALIVED_CONF="${SNAP_DATA}/conf/ingress/keepalived.conf" +HAPROXY_CONF="${SNAP_DATA}/conf/ingress/haproxy.cfg" +KEEPALIVED_PID_FILE="${SNAP_DATA}/run/ingress/keepalived.pid" +HAPROXY_PID_FILE="${SNAP_DATA}/run/ingress/haproxy.pid" + +mkdir -p "${SNAP_DATA}/run/ingress" +mkdir -p "${SNAP_COMMON}/logs/ingress" + +# Start haproxy in the background +if [ -f "${HAPROXY_CONF}" ]; then + haproxy -f "${HAPROXY_CONF}" -p "${HAPROXY_PID_FILE}" & + HAPROXY_PID=$! +else + # On first start, config might not be there yet. + # Keepalived will fail and snapd will restart us. + # This is not ideal, but it's a race condition that's hard to solve here. + sleep 1 +fi + +# Clean up haproxy on exit +trap "kill ${HAPROXY_PID} && rm -f ${KEEPALIVED_PID_FILE}" SIGINT SIGTERM EXIT + +# Start keepalived in the foreground +if [ -f "${KEEPALIVED_CONF}" ]; then + exec keepalived --dont-fork --log-console --use-pid=${KEEPALIVED_PID_FILE} --config-file=${KEEPALIVED_CONF} +else + # Same as above, config might not be ready on first start. + sleep 1 +fi diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 97d04530..4e781840 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -110,6 +110,18 @@ apps: - network - network-bind - process-control + ingress: + command: commands/ingress.start + reload-command: commands/ingress.reload + daemon: simple + install-mode: disable + after: + - daemon + plugs: + - network + - network-bind + - network-control + - process-control osd: command: commands/osd.start reload-command: commands/osd.reload @@ -431,6 +443,12 @@ parts: - lib/*/liburcu-bp.so* - lib/*/libwbclient.so* + ingress-deps: + plugin: nil + stage-packages: + - keepalived + - haproxy + strip: after: - ceph diff --git a/snapcraft/commands/ingress.reload b/snapcraft/commands/ingress.reload new file mode 100644 index 00000000..afff48b6 --- /dev/null +++ b/snapcraft/commands/ingress.reload @@ -0,0 +1,13 @@ +#!/bin/bash + +HAPROXY_PID_FILE="${SNAP_DATA}/run/ingress/haproxy.pid" +KEEPALIVED_PID_FILE="${SNAP_DATA}/run/ingress/keepalived.pid" +HAPROXY_CONF="${SNAP_DATA}/conf/ingress/haproxy.cfg" + +if [ -f "${HAPROXY_PID_FILE}" ]; then + haproxy -f "${HAPROXY_CONF}" -p "${HAPROXY_PID_FILE}" -sf $(cat ${HAPROXY_PID_FILE}) +fi + +if [ -f "${KEEPALIVED_PID_FILE}" ]; then + kill -HUP $(cat ${KEEPALIVED_PID_FILE}) +fi diff --git a/snapcraft/commands/ingress.start b/snapcraft/commands/ingress.start new file mode 100644 index 00000000..271cf3b3 --- /dev/null +++ b/snapcraft/commands/ingress.start @@ -0,0 +1,37 @@ +#!/bin/bash + +. "${SNAP}/commands/common" + +limits + +wait_for_config + +KEEPALIVED_CONF="${SNAP_DATA}/conf/ingress/keepalived.conf" +HAPROXY_CONF="${SNAP_DATA}/conf/ingress/haproxy.cfg" +KEEPALIVED_PID_FILE="${SNAP_DATA}/run/ingress/keepalived.pid" +HAPROXY_PID_FILE="${SNAP_DATA}/run/ingress/haproxy.pid" + +mkdir -p "${SNAP_DATA}/run/ingress" +mkdir -p "${SNAP_COMMON}/logs/ingress" + +# Start haproxy in the background +if [ -f "${HAPROXY_CONF}" ]; then + haproxy -f "${HAPROXY_CONF}" -p "${HAPROXY_PID_FILE}" & + HAPROXY_PID=$! +else + # On first start, config might not be there yet. + # Keepalived will fail and snapd will restart us. + # This is not ideal, but it's a race condition that's hard to solve here. + sleep 1 +fi + +# Clean up haproxy on exit +trap "kill ${HAPROXY_PID} && rm -f ${KEEPALIVED_PID_FILE}" SIGINT SIGTERM EXIT + +# Start keepalived in the foreground +if [ -f "${KEEPALIVED_CONF}" ]; then + exec keepalived --dont-fork --log-console --use-pid=${KEEPALIVED_PID_FILE} --config-file=${KEEPALIVED_CONF} +else + # Same as above, config might not be ready on first start. + sleep 1 +fi