local proxy can`t work

[nicefuture2016]

crictl pull  harbor-core.71djy.cn/devops/rockylinux:9.3.20231119-minimal

1.values file

global:
  imageRegistry: "harbor-core.71djy.cn"
  imagePullSecrets:
    - name: harbor-core
  storageClass: "ceph-filesystem"

manager:
  enable: true
  image:
    registry: harbor-core.71djy.cn
    repository: dragonflyoss/manager
    tag: v2.1.53
  initContainer:
    image:
      registry: harbor-core.71djy.cn
      repository: dragonflyoss/busybox
      tag: latest
      pullPolicy: IfNotPresent
  metrics:
    enable: true
  config:
    verbose: false
    console: false

scheduler:
  enable: true
  image:
    registry: harbor-core.71djy.cn
    repository: dragonflyoss/scheduler
    tag: v2.1.53
  metrics:
    enable: true
  initContainer:
    image:
      registry: harbor-core.71djy.cn
      repository: dragonflyoss/busybox
      tag: latest
  config:
    verbose: false
    console: false

seedClient:
  image:
    registry: harbor-core.71djy.cn
    repository: dragonflyoss/client
    tag: v0.1.94
  metrics:
    enable: true
  initContainer:
    image:
      registry: harbor-core.71djy.cn
      repository: dragonflyoss/busybox
      tag: latest
  config:
    verbose: false
    console: false

client:
  image:
    registry: harbor-core.71djy.cn
    repository: dragonflyoss/client
    tag: v0.1.94
  config:
    verbose: false
    console: false
  metrics:
    enable: true
  initContainer:
    image:
      registry: harbor-core.71djy.cn
      repository: dragonflyoss/busybox
      tag: latest
  dfinit:
    enable: true
    image:
      registry: harbor-core.71djy.cn
      repository: dragonflyoss/dfinit
      tag: v0.1.94
    config:
      containerRuntime:
        containerd:
          configPath: /etc/containerd/config.toml
          registries:
            - hostNamespace: harbor-core.71djy.cn
              serverAddr: https://harbor-core.71djy.cn
              capabilities: ['pull', 'resolve']

mysql:
  enable: true
  image:
    registry: harbor-core.71djy.cn
    repository: dragonflyoss/mysql
    tag: 8.0.36-debian-12-r10
    pullSecrets:
      - name: harbor-core
  auth:
    rootPassword: Size4@77157

redis:
  enable: true
  image:
    registry: harbor-core.71djy.cn
    repository: dragonflyoss/redis
    tag: 7.2.5-debian-12-r0
    pullSecrets:
      - name: harbor-core
  auth:
    enabled: true
    password: i9zA6zOf0nC3Uv0B9WnKF9IkEqcrQLNz

2.config.toml

disabled_plugins = []
imports = []
oom_score = 0
plugin_dir = ""
required_plugins = []
root = "/var/lib/containerd" 
state = "/run/containerd"
temp = ""
version = 2

[cgroup]
  path = ""

[debug]
  address = ""
  format = ""
  gid = 0
  level = ""
  uid = 0

[grpc]
  address = "/run/containerd/containerd.sock"
  gid = 0
  max_recv_message_size = 16777216
  max_send_message_size = 16777216
  tcp_address = ""
  tcp_tls_ca = ""
  tcp_tls_cert = ""
  tcp_tls_key = ""
  uid = 0

[metrics]
  address = ""
  grpc_histogram = false

[plugins]

  [plugins."io.containerd.gc.v1.scheduler"]
    deletion_threshold = 0
    mutation_threshold = 100
    pause_threshold = 0.02
    schedule_delay = "0s"
    startup_delay = "100ms"

  [plugins."io.containerd.grpc.v1.cri"]
    device_ownership_from_security_context = false
    disable_apparmor = false
    disable_cgroup = false
    disable_hugetlb_controller = true
    disable_proc_mount = false
    disable_tcp_service = true
    enable_selinux = false
    enable_tls_streaming = false
    enable_unprivileged_icmp = false
    enable_unprivileged_ports = false
    ignore_image_defined_volumes = false
    max_concurrent_downloads = 3
    max_container_log_line_size = 16384
    netns_mounts_under_state_dir = false
    restrict_oom_score_adj = false
    sandbox_image = "easzlab.io.local:5000/easzlab/pause:3.9"
    selinux_category_range = 1024
    stats_collect_period = 10
    stream_idle_timeout = "4h0m0s"
    stream_server_address = "127.0.0.1"
    stream_server_port = "0"
    systemd_cgroup = false
    tolerate_missing_hugetlb_controller = true
    unset_seccomp_profile = ""

    [plugins."io.containerd.grpc.v1.cri".cni]
      bin_dir = "/opt/cni/bin"
      conf_dir = "/etc/cni/net.d"
      conf_template = "/etc/cni/net.d/10-default.conf"
      max_conf_num = 1

    [plugins."io.containerd.grpc.v1.cri".containerd]
      default_runtime_name = "runc"
      disable_snapshot_annotations = true
      discard_unpacked_layers = false
      ignore_rdt_not_enabled_errors = false
      no_pivot = false
      snapshotter = "overlayfs"

      [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime]
        base_runtime_spec = ""
        container_annotations = []
        pod_annotations = []
        privileged_without_host_devices = false
        runtime_engine = ""
        runtime_root = ""
        runtime_type = ""

        [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime.options]

      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]

        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
          base_runtime_spec = ""
          container_annotations = []
          pod_annotations = []
          privileged_without_host_devices = false
          runtime_engine = ""
          runtime_root = ""
          runtime_type = "io.containerd.runc.v2"

          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
            BinaryName = ""
            CriuImagePath = ""
            CriuPath = ""
            CriuWorkPath = ""
            IoGid = 0
            IoUid = 0
            NoNewKeyring = false
            NoPivotRoot = false
            Root = ""
            ShimCgroup = ""
            SystemdCgroup = true

      [plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime]
        base_runtime_spec = ""
        container_annotations = []
        pod_annotations = []
        privileged_without_host_devices = false
        runtime_engine = ""
        runtime_root = ""
        runtime_type = ""

        [plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime.options]

    [plugins."io.containerd.grpc.v1.cri".image_decryption]
      key_model = "node"

    [plugins."io.containerd.grpc.v1.cri".registry]

      [plugins."io.containerd.grpc.v1.cri".registry.auths]

      [plugins."io.containerd.grpc.v1.cri".registry.configs]
        [plugins."io.containerd.grpc.v1.cri".registry.configs."easzlab.io.local:5000".tls]
          insecure_skip_verify = true

        [plugins."io.containerd.grpc.v1.cri".registry.configs."harbor.easzlab.io.local:8443".tls]
          insecure_skip_verify = true

      [plugins."io.containerd.grpc.v1.cri".registry.headers]
        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."easzlab.io.local:5000"]
          endpoint = ["http://easzlab.io.local:5000"]
        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."harbor.easzlab.io.local:8443"]
          endpoint = ["https://harbor.easzlab.io.local:8443"]
        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
          endpoint = ["https://docker.nju.edu.cn/", "https://kuamavit.mirror.aliyuncs.com"]
        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."gcr.io"]
          endpoint = ["https://gcr.nju.edu.cn"]
        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."k8s.gcr.io"]
          endpoint = ["https://gcr.nju.edu.cn/google-containers/"]
        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."quay.io"]
          endpoint = ["https://quay.nju.edu.cn"]
        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."ghcr.io"]
          endpoint = ["https://ghcr.nju.edu.cn"]
        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."nvcr.io"]
          endpoint = ["https://ngc.nju.edu.cn"]

[plugins."io.containerd.grpc.v1.cri".registry.mirrors."harbor-core.71djy.cn"]
endpoint = ["http://127.0.0.1:4001", "https://harbor-core.71djy.cn"]

[plugins."io.containerd.grpc.v1.cri".registry.mirrors."harbor-core.71djy.cn".header]
X-Dragonfly-Registry = "https://harbor-core.71djy.cn"

[plugins."io.containerd.grpc.v1.cri".registry.configs."harbor-core.71djy.cn".auth]
  username = "xxxx"
  password = "xxxxx"
[plugins."io.containerd.grpc.v1.cri".registry.configs."127.0.0.1:4001".auth]
  username = "xxxx"
  password = "xxxxx"

    [plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming]
      tls_cert_file = ""
      tls_key_file = ""

  [plugins."io.containerd.internal.v1.opt"]
    path = "/opt/containerd"

  [plugins."io.containerd.internal.v1.restart"]
    interval = "10s"

  [plugins."io.containerd.metadata.v1.bolt"]
    content_sharing_policy = "shared"

  [plugins."io.containerd.monitor.v1.cgroups"]
    no_prometheus = false

  [plugins."io.containerd.runtime.v1.linux"]
    no_shim = false
    runtime = "runc"
    runtime_root = ""
    shim = "containerd-shim"
    shim_debug = false

  [plugins."io.containerd.runtime.v2.task"]
    platforms = ["linux/amd64"]

  [plugins."io.containerd.service.v1.diff-service"]
    default = ["walking"]

  [plugins."io.containerd.snapshotter.v1.aufs"]
    root_path = ""

  [plugins."io.containerd.snapshotter.v1.btrfs"]
    root_path = ""

  [plugins."io.containerd.snapshotter.v1.devmapper"]
    async_remove = false
    base_image_size = ""
    pool_name = ""
    root_path = ""

  [plugins."io.containerd.snapshotter.v1.native"]
    root_path = ""

  [plugins."io.containerd.snapshotter.v1.overlayfs"]
    root_path = ""

  [plugins."io.containerd.snapshotter.v1.zfs"]
    root_path = ""

[proxy_plugins]

[stream_processors]

  [stream_processors."io.containerd.ocicrypt.decoder.v1.tar"]
    accepts = ["application/vnd.oci.image.layer.v1.tar+encrypted"]
    args = ["--decryption-keys-path", "/etc/containerd/ocicrypt/keys"]
    env = ["OCICRYPT_KEYPROVIDER_CONFIG=/etc/containerd/ocicrypt/ocicrypt_keyprovider.conf"]
    path = "ctd-decoder"
    returns = "application/vnd.oci.image.layer.v1.tar"

  [stream_processors."io.containerd.ocicrypt.decoder.v1.tar.gzip"]
    accepts = ["application/vnd.oci.image.layer.v1.tar+gzip+encrypted"]
    args = ["--decryption-keys-path", "/etc/containerd/ocicrypt/keys"]
    env = ["OCICRYPT_KEYPROVIDER_CONFIG=/etc/containerd/ocicrypt/ocicrypt_keyprovider.conf"]
    path = "ctd-decoder"
    returns = "application/vnd.oci.image.layer.v1.tar+gzip"

[timeouts]
  "io.containerd.timeout.shim.cleanup" = "5s"
  "io.containerd.timeout.shim.load" = "5s"
  "io.containerd.timeout.shim.shutdown" = "3s"
  "io.containerd.timeout.task.state" = "2s"

[ttrpc]
  address = ""
  gid = 0
  uid = 0

3.containerd logs

127.0.0.1:4001/v2/devops/rockylinux/manifests/9.3.20231119-minimal?ns=harbor-core.71djy.cn\": EOF" host="127.0.0.1:4001"
ps/rockylinux:9.3.20231119-minimal,Labels:map[string]string{io.cri-containerd.image: managed,},XXX_unrecognized:[],}"
17a49763232bfd37f88bc0ee14cc273c5c1a089dda20fa,Labels:map[string]string{io.cri-containerd.image: managed,},XXX_unrecognized:[],}"
ps/rockylinux:9.3.20231119-minimal,Labels:map[string]string{io.cri-containerd.image: managed,},XXX_unrecognized:[],}"
ps/rockylinux@sha256:d1f81ec149c64c10e0626b8f4ae561b50b3567901a65cd52406dbff065196f9d,Labels:map[string]string{io.cri-containerd.image: managed,},XXX_unrecognized:[],>
19-minimal\" returns image reference \"sha256:dfaa211c6b309e0e1017a49763232bfd37f88bc0ee14cc273c5c1a089dda20fa\""
[root@worker09 containerd]# journalctl -u containerd.service  -f
Aug 06 19:49:57 worker09 containerd[1670613]: time="2024-08-06T19:49:57.788868392+08:00" level=info msg="ImageDelete event &ImageDelete{Name:harbor-core.71djy.cn/devops/rockylinux@sha256:d1f81ec149c64c10e0626b8f4ae561b50b3567901a65cd52406dbff065196f9d,XXX_unrecognized:[],}"
Aug 06 19:49:57 worker09 containerd[1670613]: time="2024-08-06T19:49:57.792843039+08:00" level=info msg="ImageDelete event &ImageDelete{Name:sha256:dfaa211c6b309e0e1017a49763232bfd37f88bc0ee14cc273c5c1a089dda20fa,XXX_unrecognized:[],}"
Aug 06 19:49:57 worker09 containerd[1670613]: time="2024-08-06T19:49:57.995535887+08:00" level=info msg="RemoveImage \"harbor-core.71djy.cn/devops/rockylinux:9.3.20231119-minimal\" returns successfully"
Aug 06 19:50:05 worker09 containerd[1670613]: time="2024-08-06T19:50:05.317440742+08:00" level=info msg="PullImage \"harbor-core.71djy.cn/devops/rockylinux:9.3.20231119-minimal\""
Aug 06 19:50:05 worker09 containerd[1670613]: time="2024-08-06T19:50:05.616912906+08:00" level=info msg="trying next host" error="failed to do request: Head \"http://127.0.0.1:4001/v2/devops/rockylinux/manifests/9.3.20231119-minimal?ns=harbor-core.71djy.cn\": EOF" host="127.0.0.1:4001"
Aug 06 19:50:13 worker09 containerd[1670613]: time="2024-08-06T19:50:13.641624263+08:00" level=info msg="ImageCreate event &ImageCreate{Name:harbor-core.71djy.cn/devops/rockylinux:9.3.20231119-minimal,Labels:map[string]string{io.cri-containerd.image: managed,},XXX_unrecognized:[],}"
Aug 06 19:50:13 worker09 containerd[1670613]: time="2024-08-06T19:50:13.646305607+08:00" level=info msg="ImageCreate event &ImageCreate{Name:sha256:dfaa211c6b309e0e1017a49763232bfd37f88bc0ee14cc273c5c1a089dda20fa,Labels:map[string]string{io.cri-containerd.image: managed,},XXX_unrecognized:[],}"
Aug 06 19:50:13 worker09 containerd[1670613]: time="2024-08-06T19:50:13.650046427+08:00" level=info msg="ImageUpdate event &ImageUpdate{Name:harbor-core.71djy.cn/devops/rockylinux:9.3.20231119-minimal,Labels:map[string]string{io.cri-containerd.image: managed,},XXX_unrecognized:[],}"
Aug 06 19:50:13 worker09 containerd[1670613]: time="2024-08-06T19:50:13.653405950+08:00" level=info msg="ImageCreate event &ImageCreate{Name:harbor-core.71djy.cn/devops/rockylinux@sha256:d1f81ec149c64c10e0626b8f4ae561b50b3567901a65cd52406dbff065196f9d,Labels:map[string]string{io.cri-containerd.image: managed,},XXX_unrecognized:[],}"
Aug 06 19:50:13 worker09 containerd[1670613]: time="2024-08-06T19:50:13.653808929+08:00" level=info msg="PullImage \"harbor-core.71djy.cn/devops/rockylinux:9.3.20231119-minimal\" returns image reference \"sha256:dfaa211c6b309e0e1017a49763232bfd37f88bc0ee14cc273c5c1a089dda20fa\""	

4.dfdamon log

2024-08-06T11:50:05.321420726+00:00  INFO run: dragonfly-client/src/proxy/mod.rs:172: accepted connection from 127.0.0.1:35240

2024-08-06T11:50:05.321672659+00:00  INFO handler:registry_mirror_http_handler:http_handler: dragonfly-client/src/proxy/mod.rs:280: handle HTTP request: Request { method: HEAD, uri: https://index.docker.io/v2/devops/rockylinux/manifests/9.3.20231119-minimal, version: HTTP/1.1, headers: {"host": "index.docker.io", "user-agent": "containerd/v1.6.20", "accept": "application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*"}, body: Body(Empty) }

2024-08-06T11:50:05.321797199+00:00  INFO handler:registry_mirror_http_handler:http_handler: dragonfly-client/src/proxy/mod.rs:296: proxy HTTPS request directly to remote server for method: HEAD, uri: https://index.docker.io/v2/devops/rockylinux/manifests/9.3.20231119-minimal

2024-08-06T11:50:05.616592727+00:00 ERROR dragonfly-client/src/proxy/mod.rs:191: failed to serve connection from 127.0.0.1:35240: error from user's Service

2024-08-06T11:50:05.794428758+00:00  INFO run: dragonfly-client/src/proxy/mod.rs:172: accepted connection from 127.0.0.1:35256

2024-08-06T11:50:05.794648889+00:00  INFO handler:registry_mirror_http_handler:http_handler: dragonfly-client/src/proxy/mod.rs:280: handle HTTP request: Request { method: GET, uri: https://index.docker.io/v2/devops/rockylinux/manifests/sha256:d1f81ec149c64c10e0626b8f4ae561b50b3567901a65cd52406dbff065196f9d, version: HTTP/1.1, headers: {"host": "index.docker.io", "user-agent": "containerd/v1.6.20", "accept": "application/vnd.docker.distribution.manifest.v2+json, */*", "accept-encoding": "gzip"}, body: Body(Empty) }

2024-08-06T11:50:05.794745759+00:00  INFO handler:registry_mirror_http_handler:http_handler: dragonfly-client/src/proxy/mod.rs:296: proxy HTTPS request directly to remote server for method: GET, uri: https://index.docker.io/v2/devops/rockylinux/manifests/sha256:d1f81ec149c64c10e0626b8f4ae561b50b3567901a65cd52406dbff065196f9d

2024-08-06T11:50:06.060647540+00:00 ERROR dragonfly-client/src/proxy/mod.rs:191: failed to serve connection from 127.0.0.1:35256: error from user's Service

2024-08-06T11:50:06.589554724+00:00  INFO run: dragonfly-client/src/proxy/mod.rs:172: accepted connection from 127.0.0.1:35264

2024-08-06T11:50:06.589801294+00:00  INFO handler:registry_mirror_http_handler:http_handler: dragonfly-client/src/proxy/mod.rs:280: handle HTTP request: Request { method: GET, uri: https://index.docker.io/v2/devops/rockylinux/blobs/sha256:dfaa211c6b309e0e1017a49763232bfd37f88bc0ee14cc273c5c1a089dda20fa, version: HTTP/1.1, headers: {"host": "index.docker.io", "user-agent": "containerd/v1.6.20", "accept": "application/vnd.docker.container.image.v1+json, */*", "accept-encoding": "gzip"}, body: Body(Empty) }

2024-08-06T11:50:06.589973355+00:00  INFO handler:registry_mirror _http_handler:http_handler: dragonfly-client/src/proxy/mod.rs:287: proxy HTTP request via dfdaemon for method: GET, uri: https://index.docker.io/v2/devops/rockylinux/blobs/sha256:dfaa211c6b309e0e1017a49763232bfd37f88bc0ee14cc273c5c1a089dda20fa

2024-08-06T11:50:06.590650057+00:00  INFO download_task: dragonfly-client/src/grpc/dfdaemon_download.rs:178: download task in download server

2024-08-06T11:50:06.590777349+00:00  INFO download_task: dragonfly-client/src/grpc/dfdaemon_download.rs:221: download task started: Download { url: "https://index.docker.io/v2/devops/rockylinux/blobs/sha256:dfaa211c6b309e0e1017a49763232bfd37f88bc0ee14cc273c5c1a089dda20fa", digest: None, range: None, r#type: Dfdaemon, tag: None, application: None, priority: Level6, filtered_query_params: ["X-Goog-Expires", "Expires", "X-Goog-Credential", "X-Amz-Date", "OSSAccessKeyId", "SecurityToken", "q-sign-time", "X-Amz-Expires", "X-Goog-SignedHeaders", "X-Goog-Signature", "q-sign-algorithm", "X-Amz-Signature", "X-Amz-User-Agent", "AccessKeyId", "q-header-list", "q-ak", "X-Goog-Date", "X-Amz-Credential", "X-Amz-Algorithm", "Signature", "q-url-param-list", "x-cos-security-token", "X-Obs-Security-Token", "q-signature", "q-key-time", "X-Goog-Algorithm", "X-Amz-Security-Token", "X-Amz-SignedHeaders", "X-Obs-Date"], request_header: {"accept-encoding": "gzip", "user-agent": "containerd/v1.6.20", "accept": "application/vnd.docker.container.image.v1+json, */*"}, piece_length: 4194304, output_path: None, timeout: None, disable_back_to_source: false, need_back_to_source: false, certificate_chain: [], prefetch: false, object_storage: None } host_id="172.25.2.120-worker09" task_id="231278a3022d7ac929797d8b969556cb97b2333e14e7c366316c289393e28fd9" peer_id="172.25.2.120-worker09-efaa1ea4-acd4-44d1-b5e9-79e906b4d420"

2024-08-06T11:50:06.591844834+00:00  INFO download_task: dragonfly-client-backend/src/http.rs:69: head request 231278a3022d7ac929797d8b969556cb97b2333e14e7c366316c289393e28fd9 https://index.docker.io/v2/devops/rockylinux/blobs/sha256:dfaa211c6b309e0e1017a49763232bfd37f88bc0ee14cc273c5c1a089dda20fa: Some({"accept-encoding": "gzip", "user-agent": "containerd/v1.6.20", "accept": "application/vnd.docker.container.image.v1+json, */*"}) host_id="172.25.2.120-worker09" task_id="231278a3022d7ac929797d8b969556cb97b2333e14e7c366316c289393e28fd9" peer_id="172.25.2.120-worker09-efaa1ea4-acd4-44d1-b5e9-79e906b4d420"

2024-08-06T11:50:06.866646826+00:00 ERROR download_task: dragonfly-client-backend/src/http.rs:89: head request failed 231278a3022d7ac929797d8b969556cb97b2333e14e7c366316c289393e28fd9 https://index.docker.io/v2/devops/rockylinux/blobs/sha256:dfaa211c6b309e0e1017a49763232bfd37f88bc0ee14cc273c5c1a089dda20fa: error sending request for url (https://index.docker.io/v2/devops/rockylinux/blobs/sha256:dfaa211c6b309e0e1017a49763232bfd37f88bc0ee14cc273c5c1a089dda20fa) host_id="172.25.2.120-worker09" task_id="231278a3022d7ac929797d8b969556cb97b2333e14e7c366316c289393e28fd9" peer_id="172.25.2.120-worker09-efaa1ea4-acd4-44d1-b5e9-79e906b4d420"

2024-08-06T11:50:06.866697986+00:00 ERROR download_task: dragonfly-client/src/grpc/dfdaemon_download.rs:253: download started failed: error sending request for url (https://index.docker.io/v2/devops/rockylinux/blobs/sha256:dfaa211c6b309e0e1017a49763232bfd37f88bc0ee14cc273c5c1a089dda20fa) host_id="172.25.2.120-worker09" task_id="231278a3022d7ac929797d8b969556cb97b2333e14e7c366316c289393e28fd9" peer_id="172.25.2.120-worker09-efaa1ea4-acd4-44d1-b5e9-79e906b4d420"

2024-08-06T11:50:06.866979825+00:00 ERROR handler:registry_mirror_http_handler:http_handler:proxy_by_dfdaemon: dragonfly-client/src/proxy/mod.rs:517: download task failed: status: Internal, message: "error sending request for url (https://index.docker.io/v2/devops/rockylinux/blobs/sha256:dfaa211c6b309e0e1017a49763232bfd37f88bc0ee14cc273c5c1a089dda20fa)", details: [], metadata: MetadataMap { headers: {"content-type": "application/grpc", "date": "Tue, 06 Aug 2024 11:50:06 GMT", "content-length": "0"} }

2024-08-06T11:50:07.011440553+00:00  INFO handler:registry_mirror_http_handler:http_handler: dragonfly-client/src/proxy/mod.rs:280: handle HTTP request: Request { method: GET, uri: https://index.docker.io/v2/devops/rockylinux/blobs/sha256:8ec988941d6694de13ed8cb1505c0eb38bf3777bab0acc157ff18974d7350470, version: HTTP/1.1, headers: {"host": "index.docker.io", "user-agent": "containerd/v1.6.20", "accept": "application/vnd.docker.image.rootfs.diff.tar.gzip, */*", "accept-encoding": "gzip"}, body: Body(Empty) }

2024-08-06T11:50:07.011593943+00:00  INFO handler:registry_mirror_http_handler:http_handler: dragonfly-client/src/proxy/mod.rs:287: proxy HTTP request via dfdaemon for method: GET, uri: https://index.docker.io/v2/devops/rockylinux/blobs/sha256:8ec988941d6694de13ed8cb1505c0eb38bf3777bab0acc157ff18974d7350470

2024-08-06T11:50:07.012274115+00:00  INFO download_task: dragonfly-client/src/grpc/dfdaemon_download.rs:178: download task in download server

2024-08-06T11:50:07.012344827+00:00  INFO download_task: dragonfly-client/src/grpc/dfdaemon_download.rs:221: download task started: Download { url: "https://index.docker.io/v2/devops/rockylinux/blobs/sha256:8ec988941d6694de13ed8cb1505c0eb38bf3777bab0acc157ff18974d7350470", digest: None, range: None, r#type: Dfdaemon, tag: None, application: None, priority: Level6, filtered_query_params: ["X-Goog-Expires", "Expires", "X-Goog-Credential", "X-Amz-Date", "OSSAccessKeyId", "SecurityToken", "q-sign-time", "X-Amz-Expires", "X-Goog-SignedHeaders", "X-Goog-Signature", "q-sign-algorithm", "X-Amz-Signature", "X-Amz-User-Agent", "AccessKeyId", "q-header-list", "q-ak", "X-Goog-Date", "X-Amz-Credential", "X-Amz-Algorithm", "Signature", "q-url-param-list", "x-cos-security-token", "X-Obs-Security-Token", "q-signature", "q-key-time", "X-Goog-Algorithm", "X-Amz-Security-Token", "X-Amz-SignedHeaders", "X-Obs-Date"], request_header: {"accept-encoding": "gzip", "accept": "application/vnd.docker.image.rootfs.diff.tar.gzip, */*", "user-agent": "containerd/v1.6.20"}, piece_length: 4194304, output_path: None, timeout: None, disable_back_to_source: false, need_back_to_source: false, certificate_chain: [], prefetch: false, object_storage: None } host_id="172.25.2.120-worker09" task_id="219f552db52f9d634151150d40c26f8d862d0d07c41713d2a33a8018c32cc295" peer_id="172.25.2.120-worker09-016f3122-67f2-4649-a3d0-db244931c211"

2024-08-06T11:50:07.013624303+00:00  INFO download_task: dragonfly-client-backend/src/http.rs:69: head request 219f552db52f9d634151150d40c26f8d862d0d07c41713d2a33a8018c32cc295 https://index.docker.io/v2/devops/rockylinux/blobs/sha256:8ec988941d6694de13ed8cb1505c0eb38bf3777bab0acc157ff18974d7350470: Some({"accept-encoding": "gzip", "accept": "application/vnd.docker.image.rootfs.diff.tar.gzip, */*", "user-agent": "containerd/v1.6.20"}) host_id="172.25.2.120-worker09" task_id="219f552db52f9d634151150d40c26f8d862d0d07c41713d2a33a8018c32cc295" peer_id="172.25.2.120-worker09-016f3122-67f2-4649-a3d0-db244931c211"

2024-08-06T11:50:07.284236594+00:00 ERROR download_task: dragonfly-client-backend/src/http.rs:89: head request failed 219f552db52f9d634151150d40c26f8d862d0d07c41713d2a33a8018c32cc295 https://index.docker.io/v2/devops/rockylinux/blobs/sha256:8ec988941d6694de13ed8cb1505c0eb38bf3777bab0acc157ff18974d7350470: error sending request for url (https://index.docker.io/v2/devops/rockylinux/blobs/sha256:8ec988941d6694de13ed8cb1505c0eb38bf3777bab0acc157ff18974d7350470) host_id="172.25.2.120-worker09" task_id="219f552db52f9d634151150d40c26f8d862d0d07c41713d2a33a8018c32cc295" peer_id="172.25.2.120-worker09-016f3122-67f2-4649-a3d0-db244931c211"

2024-08-06T11:50:07.284359218+00:00 ERROR download_task: dragonfly-client/src/grpc/dfdaemon_download.rs:253: download started failed: error sending request for url (https://index.docker.io/v2/devops/rockylinux/blobs/sha256:8ec988941d6694de13ed8cb1505c0eb38bf3777bab0acc157ff18974d7350470) host_id="172.25.2.120-worker09" task_id="219f552db52f9d634151150d40c26f8d862d0d07c41713d2a33a8018c32cc295" peer_id="172.25.2.120-worker09-016f3122-67f2-4649-a3d0-db244931c211"

2024-08-06T11:50:07.284785475+00:00 ERROR handler:registry_mirror_http_handler:http_handler:proxy_by_dfdaemon: dragonfly-client/src/proxy/mod.rs:517: download task failed: status: Internal, message: "error sending request for url (https://index.docker.io/v2/devops/rockylinux/blobs/sha256:8ec988941d6694de13ed8cb1505c0eb38bf3777bab0acc157ff18974d7350470)", details: [], metadata: MetadataMap { headers: {"content-type": "application/grpc", "date": "Tue, 06 Aug 2024 11:50:07 GMT", "content-length": "0"} }

2024-08-06T11:51:49.303532977+00:00  INFO announce_host:update_available_scheduler_addrs: dragonfly-client/src/grpc/scheduler.rs:498: available schedulers is not changed: ["172.20.5.8", "172.20.210.249", "172.20.100.235"] request=AnnounceHostRequest { host: Some(Host { id: "172.25.2.120-worker09", r#type: 0, hostname: "worker09", ip: "172.25.2.120", port: 4000, download_port: 4000, os: "linux", platform: "linux", platform_family: "unix", platform_version: "12", kernel_version: "5.14.0-362.8.1.el9_3.x86_64", cpu: Some(Cpu { logical_count: 8, physical_count: 8, percent: 32.56880569458008, process_percent: 77.06422424316406, times: None }), memory: Some(Memory { total: 33374547968, available: 16649515008, used: 16725032960, used_percent: 0.0, process_used_percent: 0.0, free: 3201753088 }), network: Some(Network { tcp_connection_count: 0, upload_tcp_connection_count: 0, location: Some(""), idc: Some("") }), disk: Some(Disk { total: 37509660672, free: 13153431552, used: 24356229120, used_percent: 64.93321635986247, inodes_total: 0, inodes_used: 0, inodes_free: 0, inodes_used_percent: 0.0 }), build: Some(Build { git_version: "0.1.94", git_commit: Some(""), go_version: None, rust_version: Some(""), platform: None }), scheduler_cluster_id: 0 }), interval: Some(Duration { seconds: 300, nanos: 0 }) }

2024-08-06T11:51:49.303794086+00:00  INFO announce_host: dragonfly-client/src/grpc/scheduler.rs:170: announce host to 172.20.100.235:8002 request=AnnounceHostRequest { host: Some(Host { id: "172.25.2.120-worker09", r#type: 0, hostname: "worker09", ip: "172.25.2.120", port: 4000, download_port: 4000, os: "linux", platform: "linux", platform_family: "unix", platform_version: "12", kernel_version: "5.14.0-362.8.1.el9_3.x86_64", cpu: Some(Cpu { logical_count: 8, physical_count: 8, percent: 32.56880569458008, process_percent: 77.06422424316406, times: None }), memory: Some(Memory { total: 33374547968, available: 16649515008, used: 16725032960, used_percent: 0.0, process_used_percent: 0.0, free: 3201753088 }), network: Some(Network { tcp_connection_count: 0, upload_tcp_connection_count: 0, location: Some(""), idc: Some("") }), disk: Some(Disk { total: 37509660672, free: 13153431552, used: 24356229120, used_percent: 64.93321635986247, inodes_total: 0, inodes_used: 0, inodes_free: 0, inodes_used_percent: 0.0 }), build: Some(Build { git_version: "0.1.94", git_commit: Some(""), go_version: None, rust_version: Some(""), platform: None }), scheduler_cluster_id: 0 }), interval: Some(Duration { seconds: 300, nanos: 0 }) }

2024-08-06T11:51:49.303822900+00:00  INFO announce_host: dragonfly-client/src/grpc/scheduler.rs:170: announce host to 172.20.5.8:8002 request=AnnounceHostRequest { host: Some(Host { id: "172.25.2.120-worker09", r#type: 0, hostname: "worker09", ip: "172.25.2.120", port: 4000, download_port: 4000, os: "linux", platform: "linux", platform_family: "unix", platform_version: "12", kernel_version: "5.14.0-362.8.1.el9_3.x86_64", cpu: Some(Cpu { logical_count: 8, physical_count: 8, percent: 32.56880569458008, process_percent: 77.06422424316406, times: None }), memory: Some(Memory { total: 33374547968, available: 16649515008, used: 16725032960, used_percent: 0.0, process_used_percent: 0.0, free: 3201753088 }), network: Some(Network { tcp_connection_count: 0, upload_tcp_connection_count: 0, location: Some(""), idc: Some("") }), disk: Some(Disk { total: 37509660672, free: 13153431552, used: 24356229120, used_percent: 64.93321635986247, inodes_total: 0, inodes_used: 0, inodes_free: 0, inodes_used_percent: 0.0 }), build: Some(Build { git_version: "0.1.94", git_commit: Some(""), go_version: None, rust_version: Some(""), platform: None }), scheduler_cluster_id: 0 }), interval: Some(Duration { seconds: 300, nanos: 0 }) }

2024-08-06T11:51:49.304106393+00:00  INFO announce_host: dragonfly-client/src/grpc/scheduler.rs:170: announce host to 172.20.210.249:8002 request=AnnounceHostRequest { host: Some(Host { id: "172.25.2.120-worker09", r#type: 0, hostname: "worker09", ip: "172.25.2.120", port: 4000, download_port: 4000, os: "linux", platform: "linux", platform_family: "unix", platform_version: "12", kernel_version: "5.14.0-362.8.1.el9_3.x86_64", cpu: Some(Cpu { logical_count: 8, physical_count: 8, percent: 32.56880569458008, process_percent: 77.06422424316406, times: None }), memory: Some(Memory { total: 33374547968, available: 16649515008, used: 16725032960, used_percent: 0.0, process_used_percent: 0.0, free: 3201753088 }), network: Some(Network { tcp_connection_count: 0, upload_tcp_connection_count: 0, location: Some(""), idc: Some("") }), disk: Some(Disk { total: 37509660672, free: 13153431552, used: 24356229120, used_percent: 64.93321635986247, inodes_total: 0, inodes_used: 0, inodes_free: 0, inodes_used_percent: 0.0 }), build: Some(Build { git_version: "0.1.94", git_commit: Some(""), go_version: None, rust_version: Some(""), platform: None }), scheduler_cluster_id: 0 }), interval: Some(Duration { seconds: 300, nanos: 0 }) }

[gaius-qi]

Can the access index.docker.io in the instance?

[nicefuture2016]

Can the access index.docker.io in the instance?

I do not understand , why is https://index.docker.io/v2/devops/rockylinux/manifests/9.3.20231119-minimal,

my pull is private image with username and password, and config them in config.toml

crictl pull  harbor-core.71djy.cn/devops/rockylinux:9.3.20231119-minimal

[nicefuture2016]

seems you fixed it with :dragonflyoss/helm-charts#308

but i don`t work for me

[gaius-qi]

@nicefuture2016 I haven't found a good way to support setting custom headers in mirror mode. However, setting custom headers is supported in multi registries mode.

Issues: #3372, dragonflyoss/helm-charts#308.

Plan 1:
Use registry config:
https://d7y.io/docs/next/operations/integrations/container-runtime/containerd/#more-configurations
https://github.com/containerd/containerd/blob/main/docs/hosts.md#registry-configuration---examples

Plan 2:
Change the default mirror address configuration of dfdaemon so that all image pulling will use this default mirror, refer to https://d7y.io/docs/next/reference/configuration/client/dfdaemon/.

[nicefuture2016]

@nicefuture2016 I haven't found a good way to support setting custom headers in mirror mode. However, setting custom headers is supported in multi registries mode.

Issues: #3372, dragonflyoss/helm-charts#308.

Plan 1: Use registry config: https://d7y.io/docs/next/operations/integrations/container-runtime/containerd/#more-configurations https://github.com/containerd/containerd/blob/main/docs/hosts.md#registry-configuration---examples

Plan 2: Change the default mirror address configuration of dfdaemon so that all image pulling will use this default mirror, refer to https://d7y.io/docs/next/reference/configuration/client/dfdaemon/.

I noticed that both seedClient and client have the registryMirror field. Do I need to configure both of them?

seedClient:
  config:
    verbose: false
    console: false
    proxy:
      server:
        registryMirror:
          addr: https://harbor-core.71djy.cn
client:
  image:
    registry: harbor-core.71djy.cn
    repository: dragonflyoss/client
    tag: v0.1.94
  config:
    verbose: false
    console: false
  metrics:
    enable: true
  initContainer:
    image:
      registry: harbor-core.71djy.cn
      repository: dragonflyoss/busybox
      tag: latest
  dfinit:
    enable: true
    image:
      registry: harbor-core.71djy.cn
      repository: dragonflyoss/dfinit
      tag: v0.1.94
    config:
      containerRuntime:
        containerd:
          configPath: /etc/containerd/config.toml
          registries:
            - hostNamespace: harbor-core.71djy.cn
              serverAddr: https://harbor-core.71djy.cn
              capabilities: ['pull', 'resolve']
      proxy:
        registryMirror:
          addr: https://harbor-core.71djy.cn

[gaius-qi]

Seed Client do not need to change.