Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 6 additions & 5 deletions api/v1alpha1/mount_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,10 @@ limitations under the License.
package v1alpha1

type CSIMountConfig struct {
MountID string `json:"mountID,omitempty"` // mount id
PvName string `json:"pvName"` // persistent volume name for mounting
MountPath string `json:"mountPath"` // mount target in container to mount the persistent volume
SubPath string `json:"subPath,omitempty"` // sub path address in persistent volume
ReadOnly bool `json:"readOnly,omitempty"` // whether to mount the persistent volume as read-only
MountID string `json:"mountID,omitempty"` // mount id
PvName string `json:"pvName"` // persistent volume name for mounting
MountPath string `json:"mountPath"` // mount target in container to mount the persistent volume
SubPath string `json:"subPath,omitempty"` // sub path address in persistent volume
ReadOnly bool `json:"readOnly,omitempty"` // whether to mount the persistent volume as read-only
Attributes map[string]string `json:"attributes,omitempty"` // generic key-value attributes for storage authentication and provider-specific configuration
}
11 changes: 10 additions & 1 deletion api/v1alpha1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 4 additions & 0 deletions config/crd/bases/agents.kruise.io_sandboxclaims.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,10 @@ spec:
be mounted into the sandbox
items:
properties:
attributes:
additionalProperties:
type: string
type: object
mountID:
type: string
mountPath:
Expand Down
86 changes: 63 additions & 23 deletions pkg/controller/sandboxclaim/core/common_control.go
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,10 @@ func (c *commonControl) buildClaimOptions(ctx context.Context, claim *agentsv1al
reserveFailedSandboxFor = ptr.To(consts.ReserveFailedSandboxForever)
}

// storageAuthAnnotation holds the annotation key-value pair built by the
// BuildStorageAuthAnnotation hook (populated later, captured by reference).
var storageAuthKey, storageAuthValue string

opts := infra.ClaimSandboxOptions{
User: string(claim.UID), // Use UID to ensure uniqueness across claim recreations
Template: sandboxSet.Name,
Expand All @@ -278,6 +282,16 @@ func (c *commonControl) buildClaimOptions(ctx context.Context, claim *agentsv1al
sbx.SetAnnotations(annotations)
}

// inject storage-auth annotation for RRSA-based storage mounts
if storageAuthKey != "" && storageAuthValue != "" {
annotations := sbx.GetAnnotations()
if annotations == nil {
annotations = make(map[string]string)
}
annotations[storageAuthKey] = storageAuthValue
sbx.SetAnnotations(annotations)
}

// propagate labels to sandbox
labels := sbx.GetLabels()
if labels == nil {
Expand Down Expand Up @@ -311,6 +325,7 @@ func (c *commonControl) buildClaimOptions(ctx context.Context, claim *agentsv1al
ReserveFailedSandboxFor: reserveFailedSandboxFor,
CreateOnNoStock: claim.Spec.CreateOnNoStock,
UserMetadataKeys: sandboxcr.BuildUserMetadataKeys(claim.Spec.Labels, claim.Spec.Annotations),
Claim: claim,
}

if claim.Spec.InplaceUpdate != nil {
Expand Down Expand Up @@ -371,31 +386,11 @@ func (c *commonControl) buildClaimOptions(ctx context.Context, claim *agentsv1al
}
}
if len(claim.Spec.DynamicVolumesMount) > 0 {
csiMountOptions := make([]config.MountConfig, 0, len(claim.Spec.DynamicVolumesMount))
csiClient := csiutils.NewCSIMountHandler(c.cache.GetClient(), c.cache.GetAPIReader(), c.storageRegistry, utils.DefaultSandboxDeployNamespace)
for _, mountConfig := range claim.Spec.DynamicVolumesMount {
driverName, csiReqConfigRaw, genErr := csiClient.CSIMountOptionsConfig(ctx, mountConfig)
if genErr != nil {
errMsg := "failed to generate csi mount options config for sandbox"
logger.Error(genErr, errMsg, "mountConfigRequest", mountConfig)
return opts, fmt.Errorf("%s, err: %v", errMsg, genErr)
}
csiMountOptions = append(csiMountOptions, config.MountConfig{
Driver: driverName,
RequestRaw: csiReqConfigRaw,
})
}
opts.CSIMount = &config.CSIMountOptions{
MountOptionList: csiMountOptions,
}

// json marshal csi mount config to raw string
csiMountOptionsRaw, err := json.Marshal(claim.Spec.DynamicVolumesMount)
var err error
opts.CSIMount, storageAuthKey, storageAuthValue, err = c.buildCSIMountOptions(ctx, claim.Spec.DynamicVolumesMount)
if err != nil {
logger.Error(err, "failed to marshal csi mount config")
return opts, fmt.Errorf("failed to marshal csi mount config, err: %v", err)
return opts, err
}
opts.CSIMount.MountOptionListRaw = string(csiMountOptionsRaw)
}

if len(claim.Spec.Runtimes) > 0 {
Expand All @@ -405,6 +400,51 @@ func (c *commonControl) buildClaimOptions(ctx context.Context, claim *agentsv1al
return sandboxcr.ValidateAndInitClaimOptions(opts)
}

// buildCSIMountOptions generates CSI mount options and storage-auth annotation
// metadata from the given mount configurations.
func (c *commonControl) buildCSIMountOptions(ctx context.Context, mounts []agentsv1alpha1.CSIMountConfig) (*config.CSIMountOptions, string, string, error) {
logger := logf.FromContext(ctx)
csiMountOptions := make([]config.MountConfig, 0, len(mounts))
csiClient := csiutils.NewCSIMountHandler(c.cache.GetClient(), c.cache.GetAPIReader(), c.storageRegistry, utils.DefaultSandboxDeployNamespace)
for _, mountConfig := range mounts {
driverName, csiReqConfigRaw, genErr := csiClient.CSIMountOptionsConfig(ctx, mountConfig)
if genErr != nil {
errMsg := "failed to generate csi mount options config for sandbox"
logger.Error(genErr, errMsg, "mountConfigRequest", mountConfig)
return nil, "", "", fmt.Errorf("%s, err: %v", errMsg, genErr)
}
csiMountOptions = append(csiMountOptions, config.MountConfig{
Driver: driverName,
RequestRaw: csiReqConfigRaw,
})
}

opts := &config.CSIMountOptions{
MountOptionList: csiMountOptions,
}

// json marshal csi mount config to raw string
csiMountOptionsRaw, err := json.Marshal(mounts)
if err != nil {
logger.Error(err, "failed to marshal csi mount config")
return nil, "", "", fmt.Errorf("failed to marshal csi mount config, err: %v", err)
}
opts.MountOptionListRaw = string(csiMountOptionsRaw)

// Build storage-auth annotation via hook when enterprise RRSA-based
// storage authentication is enabled.
var storageAuthKey, storageAuthValue string
if csiutils.BuildStorageAuthAnnotation != nil {
storageAuthKey, storageAuthValue, err = csiutils.BuildStorageAuthAnnotation(ctx, c.cache.GetClient(), mounts)
if err != nil {
logger.Error(err, "failed to build storage auth annotation")
return nil, "", "", fmt.Errorf("failed to build storage auth annotation: %v", err)
}
}

return opts, storageAuthKey, storageAuthValue, nil
}

// countClaimedSandboxes counts active sandboxes claimed by this claim.
func (c *commonControl) countClaimedSandboxes(ctx context.Context, claim *agentsv1alpha1.SandboxClaim) (int32, error) {
return c.cache.CountActiveSandboxes(ctx, cache.ListSandboxesOptions{
Expand Down
143 changes: 143 additions & 0 deletions pkg/controller/sandboxclaim/core/common_control_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ import (
"github.com/openkruise/agents/pkg/features"
"github.com/openkruise/agents/pkg/sandbox-manager/infra"
"github.com/openkruise/agents/pkg/sandbox-manager/infra/sandboxcr"
"github.com/openkruise/agents/pkg/utils/csiutils"
utilfeature "github.com/openkruise/agents/pkg/utils/feature"
)

Expand Down Expand Up @@ -2132,6 +2133,7 @@ func TestBuildClaimOptions_CSIMount_Test(t *testing.T) {
name string
claim *agentsv1alpha1.SandboxClaim
sandboxSet *agentsv1alpha1.SandboxSet
setup func(t *testing.T) // optional per-test setup; use t.Cleanup for teardown
expectError bool
errorContains string
expectedMountCount int
Expand Down Expand Up @@ -2611,6 +2613,143 @@ func TestBuildClaimOptions_CSIMount_Test(t *testing.T) {
expectError: true,
errorContains: "failed to generate csi mount options config",
},
{
name: "CSI mount with credentialProviderName attribute injects storage-auth annotation via Modifier",
claim: &agentsv1alpha1.SandboxClaim{
ObjectMeta: metav1.ObjectMeta{
Name: "test-claim-csi-cred-provider",
Namespace: "default",
UID: "test-uid-cred-provider",
},
Spec: agentsv1alpha1.SandboxClaimSpec{
TemplateName: "test-template",
DynamicVolumesMount: []agentsv1alpha1.CSIMountConfig{
{
PvName: "test-pv-nas",
MountPath: "/workspace",
Attributes: map[string]string{"credentialProviderName": "my-cred-provider"},
SubPath: "user-data",
},
},
},
},
setup: func(t *testing.T) {
// Register the BuildStorageAuthAnnotation hook to simulate enterprise deployment
csiutils.BuildStorageAuthAnnotation = func(_ context.Context, _ client.Client, mounts []agentsv1alpha1.CSIMountConfig) (string, string, error) {
type storageAuthItem struct {
CredentialProviderName string `json:"credentialProviderName"`
Attributes map[string]string `json:"attributes,omitempty"`
}
var items []storageAuthItem
for _, m := range mounts {
if cpName, ok := m.Attributes["credentialProviderName"]; ok {
items = append(items, storageAuthItem{
CredentialProviderName: cpName,
Attributes: map[string]string{"sub-path": m.SubPath},
})
}
}
if len(items) == 0 {
return "", "", nil
}
data, err := json.Marshal(items)
if err != nil {
return "", "", err
}
return "security.agents.kruise.io/storage-auth", string(data), nil
}
t.Cleanup(func() {
csiutils.BuildStorageAuthAnnotation = nil
})
},
sandboxSet: &agentsv1alpha1.SandboxSet{
ObjectMeta: metav1.ObjectMeta{
Name: "test-template",
Namespace: "default",
},
Spec: agentsv1alpha1.SandboxSetSpec{
Runtimes: []agentsv1alpha1.RuntimeConfig{
{Name: agentsv1alpha1.RuntimeConfigForInjectAgentRuntime},
},
},
},
expectError: false,
expectedMountCount: 1,
validate: func(t *testing.T, opts infra.ClaimSandboxOptions) {
require.NotNil(t, opts.CSIMount, "CSIMount should not be nil")
// Invoke the Modifier to verify storage-auth annotation injection
mockSandbox := &sandboxcr.Sandbox{
Sandbox: &agentsv1alpha1.Sandbox{
ObjectMeta: metav1.ObjectMeta{
Name: "test-sandbox",
Namespace: "default",
},
},
}
opts.Modifier(mockSandbox)
// Verify storage-auth annotation is set with correct JSON content
storageAuthVal := mockSandbox.GetAnnotations()["security.agents.kruise.io/storage-auth"]
assert.NotEmpty(t, storageAuthVal, "storage-auth annotation should be injected when credentialProviderName attribute is set")
// Parse and verify the JSON structure
var items []map[string]interface{}
err := json.Unmarshal([]byte(storageAuthVal), &items)
require.NoError(t, err, "storage-auth should be valid JSON")
assert.Len(t, items, 1, "Expected 1 storage auth item")
assert.Equal(t, "my-cred-provider", items[0]["credentialProviderName"], "credentialProviderName mismatch")
attrs, ok := items[0]["attributes"].(map[string]interface{})
require.True(t, ok, "attributes should be a map")
assert.Equal(t, "user-data", attrs["sub-path"], "sub-path attribute mismatch")
},
},
{
name: "CSI mount without credentialProviderName attribute does NOT inject storage-auth annotation",
claim: &agentsv1alpha1.SandboxClaim{
ObjectMeta: metav1.ObjectMeta{
Name: "test-claim-csi-no-cred",
Namespace: "default",
UID: "test-uid-no-cred",
},
Spec: agentsv1alpha1.SandboxClaimSpec{
TemplateName: "test-template",
DynamicVolumesMount: []agentsv1alpha1.CSIMountConfig{
{
PvName: "test-pv-nas",
MountPath: "/workspace",
},
},
},
},
sandboxSet: &agentsv1alpha1.SandboxSet{
ObjectMeta: metav1.ObjectMeta{
Name: "test-template",
Namespace: "default",
},
Spec: agentsv1alpha1.SandboxSetSpec{
Runtimes: []agentsv1alpha1.RuntimeConfig{
{Name: agentsv1alpha1.RuntimeConfigForInjectAgentRuntime},
},
},
},
expectError: false,
expectedMountCount: 1,
validate: func(t *testing.T, opts infra.ClaimSandboxOptions) {
require.NotNil(t, opts.CSIMount, "CSIMount should not be nil")
// Invoke the Modifier
mockSandbox := &sandboxcr.Sandbox{
Sandbox: &agentsv1alpha1.Sandbox{
ObjectMeta: metav1.ObjectMeta{
Name: "test-sandbox",
Namespace: "default",
},
},
}
opts.Modifier(mockSandbox)
// Verify storage-auth annotation is NOT set when credentialProviderName attribute is absent
annotations := mockSandbox.GetAnnotations()
_, exists := annotations["security.agents.kruise.io/storage-auth"]
assert.False(t, exists, "storage-auth annotation should NOT be injected when credentialProviderName attribute is absent")
},
},
{
name: "SkipInitRuntime=true with CSI mount should fail validation",
claim: &agentsv1alpha1.SandboxClaim{
Expand Down Expand Up @@ -2643,6 +2782,10 @@ func TestBuildClaimOptions_CSIMount_Test(t *testing.T) {

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if tt.setup != nil {
tt.setup(t)
}

opts, err := commonControl.buildClaimOptions(ctx, tt.claim, tt.sandboxSet)

// Check error expectations
Expand Down
2 changes: 1 addition & 1 deletion pkg/controller/securitytokenrefresh/core/refresher.go
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ func NewDefaultRefresher(c client.Client) Refresher {
func (r *defaultRefresher) Refresh(ctx context.Context, sbx *agentsv1alpha1.Sandbox) (*identity.TokenResponse, error) {
log := klog.FromContext(ctx).WithValues("sandbox", klog.KObj(sbx))

tokenResp, _, err := identity.IssueSandboxToken(ctx, sbx)
tokenResp, _, err := identity.IssueSandboxToken(ctx, sbx, nil)
if err != nil {
return nil, fmt.Errorf("issue token: %w", err)
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/controller/securitytokenrefresh/core/refresher_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ type stubIdentityProvider struct {
issueCalls int
}

func (s *stubIdentityProvider) IssueToken(_ context.Context, _ identity.TokenRequest) (*identity.TokenResponse, error) {
func (s *stubIdentityProvider) IssueToken(_ context.Context, _ *agentsv1alpha1.Sandbox, _ *agentsv1alpha1.SandboxClaim, _ identity.TokenRequest) (*identity.TokenResponse, error) {
s.issueCalls++
if s.issueErr != nil {
return nil, s.issueErr
Expand Down
8 changes: 1 addition & 7 deletions pkg/identity/common_provider.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,19 +30,13 @@ import (
// It implements IdentityProvider with no-op propagation.
type defaultTokenProvider struct{}

// NewDefaultTokenProvider creates a TokenProvider that generates random tokens.
// This is the default strategy used when no identity provider service is configured.
func NewDefaultTokenProvider() TokenProvider {
return &defaultTokenProvider{}
}

// NewDefaultIdentityProvider creates an IdentityProvider with default token issuance
// and no-op security token propagation. This is the community default.
func NewDefaultIdentityProvider() IdentityProvider {
return &defaultTokenProvider{}
}

func (u *defaultTokenProvider) IssueToken(_ context.Context, _ TokenRequest) (*TokenResponse, error) {
func (u *defaultTokenProvider) IssueToken(_ context.Context, _ *agentsv1alpha1.Sandbox, _ *agentsv1alpha1.SandboxClaim, _ TokenRequest) (*TokenResponse, error) {
return &TokenResponse{
RequestID: uuid.NewString(),
AccessToken: uuid.NewString(),
Expand Down
Loading
Loading