Skip to content

Using NFS as the cache resulted in an error: "failed to create diff tar stream: failed to get xattr" #6024

Closed
Bug
Closed
Using NFS as the cache resulted in an error: "failed to create diff tar stream: failed to get xattr"#6024
Bug
Labels
kind/enhancement

Description

@yanshicheng
yanshicheng
opened on Jun 11, 2025

Contributing guidelines and issue reporting guide

Well-formed report checklist

  • I have found a bug that the documentation does not mention anything about my problem
  • I have found a bug that there are no open or closed issues that are related to my problem
  • I have provided version/information about my environment and done my best to provide a reproducer

Description of bug

Bug description

NFS is used for cache storage.

Perhaps the current version does not support NFS as the storage method.

config

debug = true
trace = false
root = "/var/lib/buildkit"  # nfs 
insecure-entitlements = [ "network.host", "security.insecure" ]


[log]
    format = "text"

[grpc]
    address = [ "tcp://0.0.0.0:1234" ]
    uid = 1000
    gid = 1000
    [grpc.tls]
    cert = "/etc/buildkit/certs/buildkitd.crt"
    key = "/etc/buildkit/certs/buildkitd.key"
    ca = "/etc/buildkit/certs/ca.crt"

[history]
    maxAge = 172800
    maxEntries = 50

[worker.oci]
    enabled = true
    platforms = [ "linux/amd64", "linux/arm64" ]
    snapshotter = "overlayfs"
    rootless = true
    noProcessSandbox = true
    gc = true
    reservedSpace = "30%"
    maxUsedSpace = "60%"
    minFreeSpace = "20GB"
    binary = ""
    apparmor-profile = ""
    max-parallelism = 4
    cniPoolSize = 16
    [worker.oci.labels]
    "foo" = "bar"
    [[worker.oci.gcpolicy]]
    reservedSpace = "512MB"
    maxUsedSpace = "1GB"
    minFreeSpace = "10GB"
    keepDuration = "48h"
    filters = [ "type==source.local", "type==exec.cachemount", "type==source.git.checkout"]
    [[worker.oci.gcpolicy]]
    all = true
    reservedSpace = 1024000000

[worker.containerd]
    enabled = false
[registry."harbor.ikubeops.local"]
    mirrors = ["harbor.ikubeops.local"]
    http = false
    insecure = false
[frontend."dockerfile.v0"]
    enabled = true
[frontend."gateway.v0"]
    enabled = true
    allowedRepositories = []

[system]
    platformsCacheMaxAge = "1h"

build command

        buildctl --addr 'tcp://buildkitd.devops.svc.cluster.local:1234' \
        --tlscacert /etc/buildkit/certs/buildkitd.crt \
        --tlscert /etc/buildkit/certs/buildkitd.crt \
        --tlskey /etc/buildkit/certs/buildkitd.key \
        build \
        --frontend dockerfile.v0 \
        --opt filename=Dockerfile \
        --opt platform=linux/amd64,linux/arm64 \
        --local context=/tmp/workspace \
        --local dockerfile=/tmp/workspace \
        --import-cache type=registry,ref=harbor.ikubeops.local/public/nngins:buildcache \
        --export-cache type=registry,ref=harbor.ikubeops.local/public/nngins:buildcache \
        --output=type=image,name=harbor.ikubeops.local/public/nngins:v2,push=true

log

❯ kubectl logs -f -n devops buildkit-builder-test1
#1 [internal] load build definition from Dockerfile
#1 transferring dockerfile: 326B 0.0s done
#1 DONE 0.0s

#2 [auth] public/nginx:pull token for harbor.ikubeops.local
#2 DONE 0.0s

#3 [auth] public/nginx:pull token for harbor.ikubeops.local
#3 DONE 0.0s

#4 [linux/arm64 internal] load metadata for harbor.ikubeops.local/public/nginx:latest
#4 ERROR: failed to authorize: failed to fetch oauth token: Post "https://harbor.ikubeops.local/service/token": tls: failed to verify certificate: x509: certificate signed by unknown authority

#5 [linux/amd64 internal] load metadata for harbor.ikubeops.local/public/nginx:latest
#5 ERROR: failed to authorize: failed to fetch oauth token: Post "https://harbor.ikubeops.local/service/token": tls: failed to verify certificate: x509: certificate signed by unknown authority

#6 [auth] public/nginx:pull token for harbor.ikubeops.local
#6 DONE 0.0s

#7 [auth] public/nginx:pull token for harbor.ikubeops.local
#7 DONE 0.0s
------
 > [linux/amd64 internal] load metadata for harbor.ikubeops.local/public/nginx:latest:
------
------
 > [linux/arm64 internal] load metadata for harbor.ikubeops.local/public/nginx:latest:
------
Dockerfile:1
--------------------
   1 | >>> FROM harbor.ikubeops.local/public/nginx:latest
   2 |
   3 |     COPY ./index.html /usr/share/nginx/html/
--------------------
error: failed to solve: failed to fetch oauth token: Post "https://harbor.ikubeops.local/service/token": tls: failed to verify certificate: x509: certificate signed by unknown authority

~                                                                                                        base ⎈ context-cluster1
❯ kubectl apply -f client.yaml
error: the path "client.yaml" does not exist

~                                                                                                        base ⎈ context-cluster1
❯ kubectl logs -f -n devops buildkit-builder-test1
#1 [internal] load build definition from Dockerfile
#1 transferring dockerfile: 326B 0.0s done
#1 DONE 0.0s

#2 [auth] public/nginx:pull token for harbor.ikubeops.local
#2 DONE 0.0s

#3 [linux/arm64 internal] load metadata for harbor.ikubeops.local/public/nginx:latest
#3 DONE 0.2s

#4 [linux/amd64 internal] load metadata for harbor.ikubeops.local/public/nginx:latest
#4 DONE 0.2s

#5 [internal] load .dockerignore
#5 transferring context: 2B done
#5 DONE 0.0s

#6 [internal] load build context
#6 DONE 0.0s

#7 [linux/arm64 1/2] FROM harbor.ikubeops.local/public/nginx:latest@sha256:c9f91949187fa1c2b4615b88d3acf7902c7e2d4a2557f33ca0cf90164269a7ae
#7 resolve harbor.ikubeops.local/public/nginx:latest@sha256:c9f91949187fa1c2b4615b88d3acf7902c7e2d4a2557f33ca0cf90164269a7ae 0.0s done
#7 DONE 0.0s

#8 [linux/amd64 1/2] FROM harbor.ikubeops.local/public/nginx:latest@sha256:c9f91949187fa1c2b4615b88d3acf7902c7e2d4a2557f33ca0cf90164269a7ae
#8 resolve harbor.ikubeops.local/public/nginx:latest@sha256:c9f91949187fa1c2b4615b88d3acf7902c7e2d4a2557f33ca0cf90164269a7ae 0.0s done
#8 DONE 0.0s

#9 [auth] public/nngins:pull token for harbor.ikubeops.local
#9 DONE 0.0s

#10 importing cache manifest from harbor.ikubeops.local/public/nngins:buildcache
#10 ERROR: failed to configure registry cache importer: harbor.ikubeops.local/public/nngins:buildcache: not found

#8 [linux/amd64 1/2] FROM harbor.ikubeops.local/public/nginx:latest@sha256:c9f91949187fa1c2b4615b88d3acf7902c7e2d4a2557f33ca0cf90164269a7ae
#8 DONE 0.0s

#6 [internal] load build context
#6 transferring context: 359B 0.0s done
#6 DONE 0.1s

#7 [linux/arm64 1/2] FROM harbor.ikubeops.local/public/nginx:latest@sha256:c9f91949187fa1c2b4615b88d3acf7902c7e2d4a2557f33ca0cf90164269a7ae
#7 resolve harbor.ikubeops.local/public/nginx:latest@sha256:c9f91949187fa1c2b4615b88d3acf7902c7e2d4a2557f33ca0cf90164269a7ae 0.0s done
#7 DONE 0.3s

#11 [linux/amd64 2/2] COPY ./index.html /usr/share/nginx/html/
#11 CANCELED

#7 [linux/arm64 1/2] FROM harbor.ikubeops.local/public/nginx:latest@sha256:c9f91949187fa1c2b4615b88d3acf7902c7e2d4a2557f33ca0cf90164269a7ae
#7 sha256:97f5c0f51d43d499970597eef919f9170954289eff0c5d7b8f8afd73dbb57977 1.21kB / 1.21kB 0.1s done
#7 sha256:c22eb46e871ad1cda19691450312c6b5c25eb5e6836773821d8091cffb6327cc 1.40kB / 1.40kB 0.1s done
#7 sha256:373fe654e9845b69587105e1b82833209521db456bdc5bc26ac7260e3eb2dd52 405B / 405B 0.1s done
#7 sha256:e7e0ca015e553ccff5686ec2153c895313675686d3f6940144ce935c07554d85 955B / 955B 0.1s done
#7 sha256:417c4bccf5349be7cd4ba91b1a2077ecf0ab50b3831bb071ba31f2c8bac02ed1 627B / 627B 0.1s done
#7 sha256:5eaa34f5b9c2a13ef2217ceb966953dfd5c3a21a990767da307be1f57e5a1e4f 6.29MB / 43.95MB 0.2s
#7 sha256:99e83134a031fcb86256b4fbff2060f16589e2e43c92d9e22b1b1fda8cb7e0f0 1.05MB / 29.12MB 0.2s
#7 sha256:99e83134a031fcb86256b4fbff2060f16589e2e43c92d9e22b1b1fda8cb7e0f0 7.34MB / 29.12MB 0.3s
#7 sha256:5eaa34f5b9c2a13ef2217ceb966953dfd5c3a21a990767da307be1f57e5a1e4f 17.83MB / 43.95MB 0.5s
#7 sha256:99e83134a031fcb86256b4fbff2060f16589e2e43c92d9e22b1b1fda8cb7e0f0 17.83MB / 29.12MB 0.5s
#7 sha256:99e83134a031fcb86256b4fbff2060f16589e2e43c92d9e22b1b1fda8cb7e0f0 29.12MB / 29.12MB 0.6s
#7 sha256:5eaa34f5b9c2a13ef2217ceb966953dfd5c3a21a990767da307be1f57e5a1e4f 37.60MB / 43.95MB 0.8s
#7 sha256:99e83134a031fcb86256b4fbff2060f16589e2e43c92d9e22b1b1fda8cb7e0f0 29.12MB / 29.12MB 0.7s done
#7 extracting sha256:99e83134a031fcb86256b4fbff2060f16589e2e43c92d9e22b1b1fda8cb7e0f0
#7 sha256:5eaa34f5b9c2a13ef2217ceb966953dfd5c3a21a990767da307be1f57e5a1e4f 43.95MB / 43.95MB 0.8s done



#7 extracting sha256:99e83134a031fcb86256b4fbff2060f16589e2e43c92d9e22b1b1fda8cb7e0f0 12.9s done
#7 DONE 14.0s

#7 [linux/arm64 1/2] FROM harbor.ikubeops.local/public/nginx:latest@sha256:c9f91949187fa1c2b4615b88d3acf7902c7e2d4a2557f33ca0cf90164269a7ae
#7 extracting sha256:5eaa34f5b9c2a13ef2217ceb966953dfd5c3a21a990767da307be1f57e5a1e4f
#7 extracting sha256:5eaa34f5b9c2a13ef2217ceb966953dfd5c3a21a990767da307be1f57e5a1e4f 19.6s done
#7 DONE 33.6s

#7 [linux/arm64 1/2] FROM harbor.ikubeops.local/public/nginx:latest@sha256:c9f91949187fa1c2b4615b88d3acf7902c7e2d4a2557f33ca0cf90164269a7ae
#7 extracting sha256:417c4bccf5349be7cd4ba91b1a2077ecf0ab50b3831bb071ba31f2c8bac02ed1
#7 extracting sha256:417c4bccf5349be7cd4ba91b1a2077ecf0ab50b3831bb071ba31f2c8bac02ed1 20.1s done
#7 DONE 53.7s

#7 [linux/arm64 1/2] FROM harbor.ikubeops.local/public/nginx:latest@sha256:c9f91949187fa1c2b4615b88d3acf7902c7e2d4a2557f33ca0cf90164269a7ae
#7 extracting sha256:e7e0ca015e553ccff5686ec2153c895313675686d3f6940144ce935c07554d85
#7 extracting sha256:e7e0ca015e553ccff5686ec2153c895313675686d3f6940144ce935c07554d85 20.8s done
#7 DONE 74.4s

#7 [linux/arm64 1/2] FROM harbor.ikubeops.local/public/nginx:latest@sha256:c9f91949187fa1c2b4615b88d3acf7902c7e2d4a2557f33ca0cf90164269a7ae
#7 extracting sha256:373fe654e9845b69587105e1b82833209521db456bdc5bc26ac7260e3eb2dd52
#7 extracting sha256:373fe654e9845b69587105e1b82833209521db456bdc5bc26ac7260e3eb2dd52 20.5s done
#7 DONE 94.9s

#7 [linux/arm64 1/2] FROM harbor.ikubeops.local/public/nginx:latest@sha256:c9f91949187fa1c2b4615b88d3acf7902c7e2d4a2557f33ca0cf90164269a7ae
#7 extracting sha256:97f5c0f51d43d499970597eef919f9170954289eff0c5d7b8f8afd73dbb57977
#7 extracting sha256:97f5c0f51d43d499970597eef919f9170954289eff0c5d7b8f8afd73dbb57977 20.6s done
#7 DONE 115.5s

#7 [linux/arm64 1/2] FROM harbor.ikubeops.local/public/nginx:latest@sha256:c9f91949187fa1c2b4615b88d3acf7902c7e2d4a2557f33ca0cf90164269a7ae
#7 extracting sha256:c22eb46e871ad1cda19691450312c6b5c25eb5e6836773821d8091cffb6327cc
#7 extracting sha256:c22eb46e871ad1cda19691450312c6b5c25eb5e6836773821d8091cffb6327cc 19.2s done
#7 DONE 134.7s

#12 [linux/arm64 2/2] COPY ./index.html /usr/share/nginx/html/
#12 DONE 19.5s

#13 exporting to image
#13 exporting layers
#13 exporting layers 1.0s done
#13 ERROR: mount callback failed on /tmp/containerd-mount3898433494: mount callback failed on /tmp/containerd-mount3580067973: failed to write compressed diff: failed to create diff tar stream: failed to get xattr for /tmp/containerd-mount3898433494/bin: operation not supported
------
 > importing cache manifest from harbor.ikubeops.local/public/nngins:buildcache:
------
------
 > exporting to image:
------
error: failed to solve: mount callback failed on /tmp/containerd-mount3898433494: mount callback failed on /tmp/containerd-mount3580067973: failed to write compressed diff: failed to create diff tar stream: failed to get xattr for /tmp/containerd-mount3898433494/bin: operation not supported

Activity

  1. yanshicheng
    added
    status/triage
    on Jun 11, 2025
  2. yanshicheng
    added the
    Bug
    issue type on Jun 11, 2025
  3. github-project-automation
    added this to Issue Triageon Jun 11, 2025
  4. github-project-automation
    moved this to New in Issue Triageon Jun 11, 2025
  5. AkihiroSuda
    added
    kind/enhancement
    and removed
    status/triage
    on Jun 15, 2025

  6. crazy-max commented on Sep 14, 2026

    @crazy-max
    crazy-max
    on Sep 14, 2026
    Member

    The failed to get xattr for <path>: operation not supported error comes from continuity's compareCapabilities in the double-walking differ, which did not tolerate ENOTSUP from a filesystem without xattr support. Fixed upstream by containerd/continuity#274, vendored into BuildKit in 02c76d748 and shipped in v0.31.0; ENOTSUP is now ignored there. Separately, the kernel does not support an overlayfs upperdir on NFS, so a buildkitd root on NFS with the overlayfs snapshotter only works incidentally; keep root on a local disk or use the native snapshotter.

  7. crazy-max
    closed this as completedon Sep 14, 2026
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Metadata

Metadata

Assignees

No one assigned

    Labels

    kind/enhancement

    Type

    Bug

    Projects

      Milestone

      No milestone

      Relationships

      None yet

      Development

      No branches or pull requests

      Issue actions