2023-07-25 06:57:31.135319 [PID=24] [PROCESS=taskmgr] [MODULE=DB] [LEVEL=DEBUG] make_payload: 'query 89 #26 INSERT INTO taskmgr_task ( %11 escape_name 1 1 #2 id  #1 , %11 escape_name 1 1 #4 name  #1 , %11 escape_name 1 1 #8 bin_path  #1 , %11 escape_name 1 1 #8 work_dir  #1 , %11 escape_name 1 1 #7 service  #1 , %11 escape_name 1 1 #11 instance_id  #1 , %11 escape_name 1 1 #3 ttl  #1 , %11 escape_name 1 1 #5 queue  #1 , %11 escape_name 1 1 #8 lock_tag  #1 , %11 escape_name 1 1 #10 depends_on  #1 , %11 escape_name 1 1 #10 defer_wait  #1 , %11 escape_name 1 1 #13 defer_resolve  #1 , %11 escape_name 1 1 #3 env  #1 , %11 escape_name 1 1 #8 bin_args  #1 , %11 escape_name 1 1 #5 stdin  #1 , %11 escape_name 1 1 #8 on_start  #1 , %11 escape_name 1 1 #15 callback_params  #1 , %11 escape_name 1 1 #12 request_info  #1 , %11 escape_name 1 1 #6 notify  #1 , %11 escape_name 1 1 #14 before_execute  #1 , %11 escape_name 1 1 #11 return_code  #1 , %11 escape_name 1 1 #6 output  #9 ) VALUES( $3 247 #1 , $12 host_restart #1 , $27 /opt/ispsystem/vm/bin/vmctl #1 , $17 /opt/ispsystem/vm #1 , $5 vm/v3 #1 , $1 1 #1 , $4 3600 #1 , $21 ["host.3.","host.3."] #1 , $18 ["node.5.host.3."] #1 , $2 [] #1 , $2 [] #1 , $2 [] #1 , $2 {} #1 , $227 ["--action","host_restart","--virt-type","kvm","--params","{\"action\":\"restart\",\"ha_enabled\":false,\"id\":\"3\",\"internal_name\":\"3_shadow_chiastolite\",\"node\":{\"id\":5,\"ip_addr\":\"172.31.36.36\",\"ssh_port\":22}}"] #1 , $1828 {"metadata":{"account":{"email":"admin@example.com","id":3},"base_resources":{"cpu_number":1,"ram_mib":768},"cluster":{"datacenter_type":"common","id":1,"interface_count":1,"virtualization_type":"kvm"},"disks":[{"boot_order":1,"expand_part":"sda2","id":3,"name":"shadow_chiastolite","pool":{"id":5,"name":"Network_lvm_dev_sda"},"size_mib":10240,"storage":{"id":2}}],"domain":"shadow-chiastolite.example.com","id":3,"interfaces":[{"cluster_interface":0,"id":3,"is_vxlan":false,"mac_address":"52:54:00:55:8E:36","model":"virtio","name":"eth0","node_interface":5,"node_interface_name":"vmbr0","vxlan_id":0}],"internal_name":"3_shadow_chiastolite","ipv4":[{"domain":"shadow-chiastolite.example.com","gateway":"10.40.80.1","interface_id":3,"ip_addr":"10.40.80.4","ippool_id":1,"local_id":3,"net_prefix":24,"network_id":1,"remote_id":6,"state":"active"},{"domain":"ggg.example.com","gateway":"10.40.80.1","interface_id":3,"ip_addr":"10.40.80.3","ippool_id":1,"local_id":7,"net_prefix":24,"network_id":1,"remote_id":10,"state":"active"}],"ipv6":[],"metadata_version":"1.0.1","name":"shadow_chiastolite","node":{"connection":"eyJzc2hfYWRkcmVzcyI6IjE3Mi4zMS4zNi4zNiIsInNzaF9wb3J0IjoyMiwic3NoX3VzZXIiOiJyb290In0=","id":5,"ip_addr":"172.31.36.36","os_version":"AlmaLinux 8.8","port":22,"virt_type":"kvm"},"os":{"group":"alma","id":22,"name":"Alma Linux 8","tags":["linux","alma","alma8"]},"preset":7,"state":{"disabled":false,"rescue_mode":false},"tuned_resources":{"anti_spoofing":true,"anti_spoofing_settings":{"vlan":false},"cpu_custom_model":null,"cpu_mode":"default","cpu_weight":1024,"firewall_rules":null,"io_read_iops":0,"io_read_mbitps":0,"io_weight":500,"io_write_iops":0,"io_write_mbitps":0,"nesting":false,"net_in_mbitps":0,"net_out_mbitps":0,"net_weight":5,"process_number":-1,"tcp_connections_in":0,"tcp_connections_out":0}}} #1 , $2 {} #1 , $37 {"host_id":3,"stop_maintenance":true} #1 , $221 {"callback_counter":7,"first_start":1219037584656360,"need_log":true,"request":{"extra_tags":[],"id":"81b6737c83820112c69fa7506bb6a725","ip":"172.18.0.27","user":{"id":5,"name":"n.mamaev@ispsystem.com"}},"runtime":816984} #1 , $37 ["node/5/task/247","host/3/task/247"] #1 , $2 {} #1 , $1 0 #1 , $0  #1 ) '
2023-07-25 06:58:20.732428 [PID=24] [PROCESS=taskmgr] [LEVEL=WARNING] [TaskManager] Task progress watchdog got warning: 'wait timeout reached [coroutine:1]'
2023-07-25 06:58:20.732550 [PID=24] [PROCESS=taskmgr] [MODULE=HTTP] [LEVEL=INFO] request GET http://consul:8500/v1/kv/task_manager/progress/?recurse&index=209439
2023-07-25 06:58:20.732859 [PID=24] [PROCESS=taskmgr] [LEVEL=WARNING] [TaskManager] Task result watchdog got warning: 'wait timeout reached [coroutine:1]'
2023-07-25 06:58:20.732907 [PID=24] [PROCESS=taskmgr] [MODULE=HTTP] [LEVEL=INFO] request GET http://consul:8500/v1/kv/task_manager/result/?recurse&index=209439
2023-07-25 06:58:20.733444 [PID=24] [PROCESS=taskmgr] [MODULE=HTTP] [LEVEL=INFO] request GET http://consul:8500/v1/kv/task_manager/progress/?recurse&index=209649
2023-07-25 06:58:20.733596 [PID=24] [PROCESS=taskmgr] [MODULE=HTTP] [LEVEL=INFO] request GET http://consul:8500/v1/kv/task_manager/result/?recurse&index=209649
2023-07-25 06:58:26.235359 [PID=13] [PROCESS=taskmgr_v4] [MODE=writer] [PROXY=yes] [REQUEST_ID=9e16ae280f618c00f44cf8f8017279b6] [LEVEL=INFO] Begin processing request target: '/task', content length:  2701
2023-07-25 06:58:26.235385 [PID=13] [PROCESS=taskmgr_v4] [MODE=writer] [PROXY=yes] [REQUEST_ID=9e16ae280f618c00f44cf8f8017279b6] [LEVEL=DEBUG] Request has Internal-Auth: on
2023-07-25 06:58:26.235393 [PID=13] [PROCESS=taskmgr_v4] [MODE=writer] [PROXY=yes] [REQUEST_ID=9e16ae280f618c00f44cf8f8017279b6] [LEVEL=DEBUG] Auth roles: {"roles":["@internal"]}
2023-07-25 06:58:26.235413 [PID=13] [PROCESS=taskmgr_v4] [MODE=writer] [PROXY=yes] [REQUEST_ID=9e16ae280f618c00f44cf8f8017279b6] [LEVEL=DEBUG] Find handler in schema for target: /task
2023-07-25 06:58:26.235527 [PID=13] [PROCESS=taskmgr_v4] [MODE=writer] [PROXY=yes] [REQUEST_ID=9e16ae280f618c00f44cf8f8017279b6] [LEVEL=INFO] Connect to 127.0.0.1:320
2023-07-25 06:58:26.235602 [PID=24] [PROCESS=taskmgr] [MODULE=MICRO] [LEVEL=DEBUG] Got connection
2023-07-25 06:58:26.235739 [PID=24] [PROCESS=taskmgr] [REQUEST_ID=9e16ae280f618c00f44cf8f8017279b6] [MODULE=MICRO] [LEVEL=DEBUG] processing handler: POST /task HTTP/1.0
2023-07-25 06:58:26.235739 [PID=24] [PROCESS=taskmgr] [REQUEST_ID=9e16ae280f618c00f44cf8f8017279b6] [MODULE=MICRO] [LEVEL=DEBUG]   X-Request-ID: 9e16ae280f618c00f44cf8f8017279b6
2023-07-25 06:58:26.235739 [PID=24] [PROCESS=taskmgr] [REQUEST_ID=9e16ae280f618c00f44cf8f8017279b6] [MODULE=MICRO] [LEVEL=DEBUG]   X-Forwarded-For: 172.18.0.27
2023-07-25 06:58:26.235739 [PID=24] [PROCESS=taskmgr] [REQUEST_ID=9e16ae280f618c00f44cf8f8017279b6] [MODULE=MICRO] [LEVEL=DEBUG]   Global-Attributes: {"callback_counter":7,"first_start":1219092685667953,"need_log":true,"request":{"extra_tags":[],"id":"e415aa771e7af6591c729e9992062de0","ip":"172.18.0.27","user":{"id":5,"name":"n.mamaev@ispsystem.com"}},"runtime":1034626}
2023-07-25 06:58:26.235739 [PID=24] [PROCESS=taskmgr] [REQUEST_ID=9e16ae280f618c00f44cf8f8017279b6] [MODULE=MICRO] [LEVEL=DEBUG]   Connection: close
2023-07-25 06:58:26.235739 [PID=24] [PROCESS=taskmgr] [REQUEST_ID=9e16ae280f618c00f44cf8f8017279b6] [MODULE=MICRO] [LEVEL=DEBUG]   x-remote-ip: 172.18.0.27
2023-07-25 06:58:26.235739 [PID=24] [PROCESS=taskmgr] [REQUEST_ID=9e16ae280f618c00f44cf8f8017279b6] [MODULE=MICRO] [LEVEL=DEBUG]   internal-auth: eyJyb2xlcyI6WyJAaW50ZXJuYWwiXX0=
2023-07-25 06:58:26.235739 [PID=24] [PROCESS=taskmgr] [REQUEST_ID=9e16ae280f618c00f44cf8f8017279b6] [MODULE=MICRO] [LEVEL=DEBUG]   Content-Length: 2701
2023-07-25 06:58:26.235739 [PID=24] [PROCESS=taskmgr] [REQUEST_ID=9e16ae280f618c00f44cf8f8017279b6] [MODULE=MICRO] [LEVEL=DEBUG]   Host: instance-auth
2023-07-25 06:58:26.235739 [PID=24] [PROCESS=taskmgr] [REQUEST_ID=9e16ae280f618c00f44cf8f8017279b6] [MODULE=MICRO] [LEVEL=DEBUG]
2023-07-25 06:58:26.235739 [PID=24] [PROCESS=taskmgr] [REQUEST_ID=9e16ae280f618c00f44cf8f8017279b6] [MODULE=MICRO] [LEVEL=DEBUG]
2023-07-25 06:58:26.235884 [PID=24] [PROCESS=taskmgr] [LEVEL=DEBUG] New cursor. Row size = 25