diff --git a/cns/NetworkContainerContract.go b/cns/NetworkContainerContract.go index 394f871f09..c93187a0e2 100644 --- a/cns/NetworkContainerContract.go +++ b/cns/NetworkContainerContract.go @@ -9,6 +9,7 @@ import ( "github.com/Azure/azure-container-networking/cns/types" "github.com/Azure/azure-container-networking/crd/nodenetworkconfig/api/v1alpha" + "github.com/Azure/azure-container-networking/network/policy" "github.com/google/uuid" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" @@ -503,6 +504,8 @@ type PodIpInfo struct { Routes []Route // PnpId is set for backend interfaces, Pnp Id identifies VF. Plug and play id(pnp) is also called as PCI ID PnPID string + // Default Deny ACL's to configure on HNS endpoints for Swiftv2 window nodes + EndpointPolicies []policy.Policy } type HostIPInfo struct { diff --git a/cns/middlewares/k8sSwiftV2.go b/cns/middlewares/k8sSwiftV2.go index a11290c205..20e514c13a 100644 --- a/cns/middlewares/k8sSwiftV2.go +++ b/cns/middlewares/k8sSwiftV2.go @@ -40,7 +40,9 @@ var _ cns.IPConfigsHandlerMiddleware = (*K8sSWIFTv2Middleware)(nil) // and release IP configs handlers. func (k *K8sSWIFTv2Middleware) IPConfigsRequestHandlerWrapper(defaultHandler, failureHandler cns.IPConfigsHandlerFunc) cns.IPConfigsHandlerFunc { return func(ctx context.Context, req cns.IPConfigsRequest) (*cns.IPConfigsResponse, error) { - podInfo, respCode, message := k.validateIPConfigsRequest(ctx, &req) + podInfo, respCode, message, defaultDenyACLbool := k.GetPodInfoForIPConfigsRequest(ctx, &req) + + logger.Printf("defaultDenyACLbool value is: %v", defaultDenyACLbool) if respCode != types.Success { return &cns.IPConfigsResponse{ @@ -55,6 +57,20 @@ func (k *K8sSWIFTv2Middleware) IPConfigsRequestHandlerWrapper(defaultHandler, fa if !req.SecondaryInterfacesExist { return ipConfigsResp, err } + + // ipConfigsResp has infra IP configs -> if defaultDenyACLbool is enabled, add the default deny acl's pn the infra IP configs + for i := range ipConfigsResp.PodIPInfo { + ipInfo := &ipConfigsResp.PodIPInfo[i] + // there will be no pod connectivity to and from those pods + if defaultDenyACLbool && ipInfo.NICType == cns.InfraNIC { + err = addDefaultDenyACL(ipInfo) + if err != nil { + logger.Errorf("failed to add default deny acl's for pod %v with err %v", podInfo.Name(), err) + } + break + } + } + // If the pod is v2, get the infra IP configs from the handler first and then add the SWIFTv2 IP config defer func() { // Release the default IP config if there is an error @@ -100,21 +116,23 @@ func (k *K8sSWIFTv2Middleware) IPConfigsRequestHandlerWrapper(defaultHandler, fa } } -// validateIPConfigsRequest validates if pod is multitenant by checking the pod labels, used in SWIFT V2 AKS scenario. +// GetPodInfoForIPConfigsRequest validates if pod is multitenant by checking the pod labels, used in SWIFT V2 AKS scenario. // nolint -func (k *K8sSWIFTv2Middleware) validateIPConfigsRequest(ctx context.Context, req *cns.IPConfigsRequest) (podInfo cns.PodInfo, respCode types.ResponseCode, message string) { +func (k *K8sSWIFTv2Middleware) GetPodInfoForIPConfigsRequest(ctx context.Context, req *cns.IPConfigsRequest) (podInfo cns.PodInfo, respCode types.ResponseCode, message string, defaultDenyACL bool) { + defaultDenyACLbool := false + // Retrieve the pod from the cluster podInfo, err := cns.UnmarshalPodInfo(req.OrchestratorContext) if err != nil { errBuf := errors.Wrapf(err, "failed to unmarshalling pod info from ipconfigs request %+v", req) - return nil, types.UnexpectedError, errBuf.Error() + return nil, types.UnexpectedError, errBuf.Error(), defaultDenyACLbool } logger.Printf("[SWIFTv2Middleware] validate ipconfigs request for pod %s", podInfo.Name()) podNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} pod := v1.Pod{} if err := k.Cli.Get(ctx, podNamespacedName, &pod); err != nil { errBuf := errors.Wrapf(err, "failed to get pod %+v", podNamespacedName) - return nil, types.UnexpectedError, errBuf.Error() + return nil, types.UnexpectedError, errBuf.Error(), defaultDenyACLbool } // check the pod labels for Swift V2, set the request's SecondaryInterfaceSet flag to true and check if its MTPNC CRD is ready @@ -126,12 +144,16 @@ func (k *K8sSWIFTv2Middleware) validateIPConfigsRequest(ctx context.Context, req mtpnc := v1alpha1.MultitenantPodNetworkConfig{} mtpncNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} if err := k.Cli.Get(ctx, mtpncNamespacedName, &mtpnc); err != nil { - return nil, types.UnexpectedError, fmt.Errorf("failed to get pod's mtpnc from cache : %w", err).Error() + return nil, types.UnexpectedError, fmt.Errorf("failed to get pod's mtpnc from cache : %w", err).Error(), defaultDenyACLbool } // Check if the MTPNC CRD is ready. If one of the fields is empty, return error if !mtpnc.IsReady() { - return nil, types.UnexpectedError, errMTPNCNotReady.Error() + return nil, types.UnexpectedError, errMTPNCNotReady.Error(), defaultDenyACLbool } + + // copying defaultDenyACL bool from mtpnc + defaultDenyACLbool = mtpnc.Status.DefaultDenyACL + // If primary Ip is set in status field, it indicates the presence of secondary interfaces if mtpnc.Status.PrimaryIP != "" { req.SecondaryInterfacesExist = true @@ -140,7 +162,7 @@ func (k *K8sSWIFTv2Middleware) validateIPConfigsRequest(ctx context.Context, req for _, interfaceInfo := range interfaceInfos { if interfaceInfo.DeviceType == v1alpha1.DeviceTypeInfiniBandNIC { if interfaceInfo.MacAddress == "" || interfaceInfo.NCID == "" { - return nil, types.UnexpectedError, errMTPNCNotReady.Error() + return nil, types.UnexpectedError, errMTPNCNotReady.Error(), defaultDenyACLbool } req.BackendInterfaceExist = true req.BackendInterfaceMacAddresses = append(req.BackendInterfaceMacAddresses, interfaceInfo.MacAddress) @@ -154,7 +176,7 @@ func (k *K8sSWIFTv2Middleware) validateIPConfigsRequest(ctx context.Context, req logger.Printf("[SWIFTv2Middleware] pod %s has secondary interface : %v", podInfo.Name(), req.SecondaryInterfacesExist) logger.Printf("[SWIFTv2Middleware] pod %s has backend interface : %v", podInfo.Name(), req.BackendInterfaceExist) // retrieve podinfo from orchestrator context - return podInfo, types.Success, "" + return podInfo, types.Success, "", defaultDenyACLbool } // getIPConfig returns the pod's SWIFT V2 IP configuration. diff --git a/cns/middlewares/k8sSwiftV2_linux.go b/cns/middlewares/k8sSwiftV2_linux.go index e9a93de0e2..99b8ae7846 100644 --- a/cns/middlewares/k8sSwiftV2_linux.go +++ b/cns/middlewares/k8sSwiftV2_linux.go @@ -103,3 +103,7 @@ func (k *K8sSWIFTv2Middleware) assignSubnetPrefixLengthFields(_ *cns.PodIpInfo, } func (k *K8sSWIFTv2Middleware) addDefaultRoute(*cns.PodIpInfo, string) {} + +func addDefaultDenyACL(_ *cns.PodIpInfo) error { + return nil +} diff --git a/cns/middlewares/k8sSwiftV2_linux_test.go b/cns/middlewares/k8sSwiftV2_linux_test.go index 76be6b2149..8bd0990728 100644 --- a/cns/middlewares/k8sSwiftV2_linux_test.go +++ b/cns/middlewares/k8sSwiftV2_linux_test.go @@ -144,7 +144,7 @@ func TestValidateMultitenantIPConfigsRequestSuccess(t *testing.T) { happyReq.OrchestratorContext = b happyReq.SecondaryInterfacesExist = false - _, respCode, err := middleware.validateIPConfigsRequest(context.TODO(), happyReq) + _, respCode, err, _ := middleware.GetPodInfoForIPConfigsRequest(context.TODO(), happyReq) assert.Equal(t, err, "") assert.Equal(t, respCode, types.Success) assert.Equal(t, happyReq.SecondaryInterfacesExist, true) @@ -158,7 +158,7 @@ func TestValidateMultitenantIPConfigsRequestSuccess(t *testing.T) { happyReq2.OrchestratorContext = b happyReq2.SecondaryInterfacesExist = false - _, respCode, err = middleware.validateIPConfigsRequest(context.TODO(), happyReq2) + _, respCode, err, _ = middleware.GetPodInfoForIPConfigsRequest(context.TODO(), happyReq2) assert.Equal(t, err, "") assert.Equal(t, respCode, types.Success) assert.Equal(t, happyReq.SecondaryInterfacesExist, true) @@ -172,7 +172,7 @@ func TestValidateMultitenantIPConfigsRequestSuccess(t *testing.T) { happyReq3.OrchestratorContext = b happyReq3.SecondaryInterfacesExist = false - _, respCode, err = middleware.validateIPConfigsRequest(context.TODO(), happyReq3) + _, respCode, err, _ = middleware.GetPodInfoForIPConfigsRequest(context.TODO(), happyReq3) assert.Equal(t, err, "") assert.Equal(t, respCode, types.Success) assert.Equal(t, happyReq3.SecondaryInterfacesExist, false) @@ -188,7 +188,7 @@ func TestValidateMultitenantIPConfigsRequestFailure(t *testing.T) { InfraContainerID: testPod1Info.InfraContainerID(), } failReq.OrchestratorContext = []byte("invalid") - _, respCode, _ := middleware.validateIPConfigsRequest(context.TODO(), failReq) + _, respCode, _, _ := middleware.GetPodInfoForIPConfigsRequest(context.TODO(), failReq) assert.Equal(t, respCode, types.UnexpectedError) // Pod doesn't exist in cache test @@ -198,19 +198,19 @@ func TestValidateMultitenantIPConfigsRequestFailure(t *testing.T) { } b, _ := testPod2Info.OrchestratorContext() failReq.OrchestratorContext = b - _, respCode, _ = middleware.validateIPConfigsRequest(context.TODO(), failReq) + _, respCode, _, _ = middleware.GetPodInfoForIPConfigsRequest(context.TODO(), failReq) assert.Equal(t, respCode, types.UnexpectedError) // Failed to get MTPNC b, _ = testPod3Info.OrchestratorContext() failReq.OrchestratorContext = b - _, respCode, _ = middleware.validateIPConfigsRequest(context.TODO(), failReq) + _, respCode, _, _ = middleware.GetPodInfoForIPConfigsRequest(context.TODO(), failReq) assert.Equal(t, respCode, types.UnexpectedError) // MTPNC not ready b, _ = testPod4Info.OrchestratorContext() failReq.OrchestratorContext = b - _, respCode, _ = middleware.validateIPConfigsRequest(context.TODO(), failReq) + _, respCode, _, _ = middleware.GetPodInfoForIPConfigsRequest(context.TODO(), failReq) assert.Equal(t, respCode, types.UnexpectedError) } diff --git a/cns/middlewares/k8sSwiftV2_windows.go b/cns/middlewares/k8sSwiftV2_windows.go index 2be2fbd1df..1f200ac259 100644 --- a/cns/middlewares/k8sSwiftV2_windows.go +++ b/cns/middlewares/k8sSwiftV2_windows.go @@ -1,9 +1,13 @@ package middlewares import ( + "encoding/json" + "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/cns/middlewares/utils" "github.com/Azure/azure-container-networking/crd/multitenancy/api/v1alpha1" + "github.com/Azure/azure-container-networking/network/policy" + "github.com/Microsoft/hcsshim/hcn" "github.com/pkg/errors" ) @@ -58,3 +62,56 @@ func (k *K8sSWIFTv2Middleware) addDefaultRoute(podIPInfo *cns.PodIpInfo, gwIP st } podIPInfo.Routes = append(podIPInfo.Routes, route) } + +// append the default deny acl's to the list defaultDenyACL field in podIpInfo +func addDefaultDenyACL(podIPInfo *cns.PodIpInfo) error { + blockEgressACL, err := getDefaultDenyACLPolicy(hcn.DirectionTypeOut) + if err != nil { + return errors.Wrap(err, "Failed to create default deny ACL policy egress") + } + + blockIngressACL, err := getDefaultDenyACLPolicy(hcn.DirectionTypeIn) + if err != nil { + return errors.Wrap(err, "Failed to create default deny ACL policy ingress") + } + + additionalArgs := []policy.Policy{ + { + Type: policy.ACLPolicy, + Data: blockEgressACL, + }, + { + Type: policy.ACLPolicy, + Data: blockIngressACL, + }, + } + + podIPInfo.EndpointPolicies = append(podIPInfo.EndpointPolicies, additionalArgs...) + + return nil +} + +// create the default deny acl's that need to be added to the list defaultDenyACL field in podIpInfo +func getDefaultDenyACLPolicy(direction hcn.DirectionType) ([]byte, error) { + type DefaultDenyACL struct { + Type string `json:"Type"` + Action hcn.ActionType `json:"Action"` + Direction hcn.DirectionType `json:"Direction"` + Priority int `json:"Priority"` + } + + denyACL := DefaultDenyACL{ + Type: "ACL", // policy type is ACL + Action: hcn.ActionTypeBlock, + Direction: direction, + Priority: 10_000, // default deny priority will be 10_000 + } + + denyACLJSON, err := json.Marshal(denyACL) + + if err != nil { + return nil, errors.Wrap(err, "error marshalling default deny policy to json") + } + + return denyACLJSON, nil +} diff --git a/cns/middlewares/k8sSwiftV2_windows_test.go b/cns/middlewares/k8sSwiftV2_windows_test.go index dab24685f9..fb6ed62978 100644 --- a/cns/middlewares/k8sSwiftV2_windows_test.go +++ b/cns/middlewares/k8sSwiftV2_windows_test.go @@ -1,12 +1,15 @@ package middlewares import ( + "encoding/json" "reflect" "testing" "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/cns/middlewares/mock" "github.com/Azure/azure-container-networking/crd/multitenancy/api/v1alpha1" + "github.com/Azure/azure-container-networking/network/policy" + "github.com/stretchr/testify/require" "gotest.tools/v3/assert" ) @@ -100,3 +103,73 @@ func TestAddDefaultRoute(t *testing.T) { t.Errorf("got '%+v', expected '%+v'", ipInfo.Routes, expectedRoutes) } } + +func TestAddDefaultDenyACL(t *testing.T) { + valueOut := []byte(`{ + "Type": "ACL", + "Action": "Block", + "Direction": "Out", + "Priority": 10000 + }`) + + valueIn := []byte(`{ + "Type": "ACL", + "Action": "Block", + "Direction": "In", + "Priority": 10000 + }`) + + expectedDefaultDenyACL := []policy.Policy{ + { + Type: policy.ACLPolicy, + Data: valueOut, + }, + { + Type: policy.ACLPolicy, + Data: valueIn, + }, + } + + podIPInfo := cns.PodIpInfo{ + PodIPConfig: cns.IPSubnet{ + IPAddress: "20.240.1.242", + PrefixLength: 32, + }, + NICType: cns.DelegatedVMNIC, + MacAddress: "12:34:56:78:9a:bc", + } + + err := addDefaultDenyACL(&podIPInfo) + assert.Equal(t, err, nil) + + // Normalize both slices so there is no extra spacing, new lines, etc + normalizedExpected := normalizeKVPairs(t, expectedDefaultDenyACL) + normalizedActual := normalizeKVPairs(t, podIPInfo.EndpointPolicies) + if !reflect.DeepEqual(normalizedExpected, normalizedActual) { + t.Errorf("got '%+v', expected '%+v'", podIPInfo.EndpointPolicies, expectedDefaultDenyACL) + } +} + +// normalizeKVPairs normalizes the JSON values in the KV pairs by unmarshaling them into a map, then marshaling them back to compact JSON to remove any extra space, new lines, etc +func normalizeKVPairs(t *testing.T, policies []policy.Policy) []policy.Policy { + normalized := make([]policy.Policy, len(policies)) + + for i, kv := range policies { + var unmarshaledValue map[string]interface{} + // Unmarshal the Value into a map + err := json.Unmarshal(kv.Data, &unmarshaledValue) + require.NoError(t, err, "Failed to unmarshal JSON value") + + // Marshal it back to compact JSON + normalizedValue, err := json.Marshal(unmarshaledValue) + require.NoError(t, err, "Failed to re-marshal JSON value") + + // Replace Value with the normalized compact JSON + normalized[i] = policy.Policy{ + Type: policy.ACLPolicy, + Data: normalizedValue, + } + } + + return normalized +}