Usage¶
Every long running service process should have a call to install a signal handler which will trigger the guru meditation framework upon receipt of SIGUSR1/SIGUSR2. This will result in the process dumping a complete report of its current state to stderr.
For RPC listeners, it may also be desirable to install some kind of hook in the RPC request dispatcher that will save a guru meditation report whenever the processing of a request results in an uncaught exception. It could save these reports to a well known directory (/var/log/openstack/<project>/<service>/) for later analysis by the sysadmin or automated bug analysis tools.
To use oslo.reports in a project, you need to add the following call to
setup_autorun() somewhere really
early in the startup sequence of the process:
from oslo_reports import guru_meditation_report as gmr
gmr.TextGuruMeditation.setup_autorun(version='13.0.0')
Note that the version parameter is the version of the component itself.
To trigger the report to be generated:
kill -SIGUSR2 <process_id>
Note
On SELinux platforms the report process may fail with an AccessDenied
exception. If this happens, temporarily disable SELinux enforcement
by running sudo setenforce 0, trigger the report, then turn SELinux
back on by running sudo setenforce 1.
Here is a sample report:
========================================================================
==== Guru Meditation ====
========================================================================
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
========================================================================
==== Package ====
========================================================================
product = OpenStack Nova
vendor = OpenStack Foundation
version = 13.0.0
========================================================================
==== Threads ====
========================================================================
------ Thread #140417215547200 ------
/usr/local/lib/python2.7/dist-packages/eventlet/hubs/hub.py:346 in run
`self.wait(sleep_time)`
/usr/local/lib/python2.7/dist-packages/eventlet/hubs/poll.py:82 in wait
`sleep(seconds)`
========================================================================
==== Green Threads ====
========================================================================
------ Green Thread ------
/usr/local/bin/nova-api:10 in <module>
`sys.exit(main())`
/opt/stack/nova/nova/cmd/api.py:57 in main
`launcher.wait()`
/usr/local/lib/python2.7/dist-packages/oslo_service/service.py:511 in wait
`self._respawn_children()`
/usr/local/lib/python2.7/dist-packages/oslo_service/service.py:495 in _respawn_children
`eventlet.greenthread.sleep(self.wait_interval)`
/usr/local/lib/python2.7/dist-packages/eventlet/greenthread.py:34 in sleep
`hub.switch()`
/usr/local/lib/python2.7/dist-packages/eventlet/hubs/hub.py:294 in switch
`return self.greenlet.switch()`
------ Green Thread ------
No Traceback!
========================================================================
==== Processes ====
========================================================================
Process 14756 (under 1) [ run by: stack (1001), state: running ]
Process 14770 (under 14756) [ run by: stack (1001), state: sleeping ]
Process 14771 (under 14756) [ run by: stack (1001), state: sleeping ]
Process 14776 (under 14756) [ run by: stack (1001), state: sleeping ]
Process 14777 (under 14756) [ run by: stack (1001), state: sleeping ]
Process 14784 (under 14756) [ run by: stack (1001), state: sleeping ]
Process 14785 (under 14756) [ run by: stack (1001), state: sleeping ]
========================================================================
==== Configuration ====
========================================================================
api_database:
connection = ***
connection_debug = 0
connection_trace = False
idle_timeout = 3600
max_overflow = None
max_pool_size = None
max_retries = 10
mysql_sql_mode = TRADITIONAL
pool_timeout = None
retry_interval = 10
slave_connection = ***
sqlite_synchronous = True
cells:
bandwidth_update_interval = 600
call_timeout = 60
capabilities =
hypervisor=xenserver;kvm
os=linux;windows
cell_type = compute
enable = False
instance_update_sync_database_limit = 100
manager = nova.cells.manager.CellsManager
mute_child_interval = 300
name = nova
reserve_percent = 10.0
topic = cells
cinder:
cafile = None
catalog_info = volumev2:cinderv2:publicURL
certfile = None
cross_az_attach = True
endpoint_template = None
http_retries = 3
insecure = False
keyfile = None
os_region_name = RegionOne
timeout = None
conductor:
manager = nova.conductor.manager.ConductorManager
topic = conductor
use_local = False
workers = 2
database:
backend = sqlalchemy
connection = ***
connection_debug = 0
connection_trace = False
db_inc_retry_interval = True
db_max_retries = 20
db_max_retry_interval = 10
db_retry_interval = 1
idle_timeout = 3600
max_overflow = None
max_pool_size = None
max_retries = 10
min_pool_size = 1
mysql_sql_mode = TRADITIONAL
pool_timeout = None
retry_interval = 10
slave_connection = ***
sqlite_db = nova.sqlite
sqlite_synchronous = True
use_db_reconnect = False
use_tpool = False
default:
allow_instance_snapshots = True
allow_resize_to_same_host = True
allow_same_net_traffic = True
api_paste_config = /etc/nova/api-paste.ini
api_rate_limit = False
auth_strategy = keystone
auto_assign_floating_ip = False
bandwidth_poll_interval = 600
bindir = /usr/local/bin
block_device_allocate_retries = 60
block_device_allocate_retries_interval = 3
boot_script_template = /opt/stack/nova/nova/cloudpipe/bootscript.template
ca_file = cacert.pem
ca_path = /opt/stack/data/nova/CA
cert_manager = nova.cert.manager.CertManager
cert_topic = cert
client_socket_timeout = 900
cnt_vpn_clients = 0
compute_available_monitors = None
compute_driver = libvirt.LibvirtDriver
compute_manager = nova.compute.manager.ComputeManager
compute_monitors =
compute_resources =
vcpu
compute_stats_class = nova.compute.stats.Stats
compute_topic = compute
config-dir = None
config-file =
/etc/nova/nova.conf
config_drive_format = iso9660
config_drive_skip_versions = 1.0 2007-01-19 2007-03-01 2007-08-29 2007-10-10 2007-12-15 2008-02-01 2008-09-01
console_host = dims-ubuntu
console_manager = nova.console.manager.ConsoleProxyManager
console_topic = console
consoleauth_manager = nova.consoleauth.manager.ConsoleAuthManager
consoleauth_topic = consoleauth
control_exchange = nova
cpu_allocation_ratio = 0.0
create_unique_mac_address_attempts = 5
crl_file = crl.pem
db_driver = nova.db
debug = True
default_access_ip_network_name = None
default_availability_zone = nova
default_ephemeral_format = ext4
default_flavor = m1.small
default_floating_pool = public
default_log_levels =
amqp=WARN
amqplib=WARN
boto=WARN
glanceclient=WARN
iso8601=WARN
keystonemiddleware=WARN
oslo_messaging=INFO
qpid=WARN
requests.packages.urllib3.connectionpool=WARN
routes.middleware=WARN
sqlalchemy=WARN
stevedore=WARN
suds=INFO
urllib3.connectionpool=WARN
websocket=WARN
default_notification_level = INFO
default_publisher_id = None
default_schedule_zone = None
defer_iptables_apply = False
dhcp_domain = novalocal
dhcp_lease_time = 86400
dhcpbridge = /usr/local/bin/nova-dhcpbridge
dhcpbridge_flagfile =
/etc/nova/nova.conf
dmz_cidr =
dmz_mask = 255.255.255.0
dmz_net = 10.0.0.0
dns_server =
dns_update_periodic_interval = -1
dnsmasq_config_file =
ebtables_exec_attempts = 3
ebtables_retry_interval = 1.0
ec2_dmz_host = 10.0.0.9
ec2_host = 10.0.0.9
ec2_listen = 0.0.0.0
ec2_listen_port = 8773
ec2_path = /
ec2_port = 8773
ec2_private_dns_show_ip = False
ec2_scheme = http
ec2_strict_validation = True
ec2_timestamp_expiry = 300
ec2_workers = 2
enable_instance_password = True
enable_network_quota = False
enable_new_services = True
enabled_apis =
ec2
metadata
osapi_compute
enabled_ssl_apis =
fake_call = False
fake_network = False
fatal_deprecations = False
fatal_exception_format_errors = False
firewall_driver = nova.virt.firewall.NoopFirewallDriver
fixed_ip_disassociate_timeout = 600
fixed_range_v6 = fd00::/48
flat_injected = False
flat_interface = None
flat_network_bridge = None
flat_network_dns = 8.8.4.4
floating_ip_dns_manager = nova.network.noop_dns_driver.NoopDNSDriver
force_config_drive = True
force_dhcp_release = True
force_raw_images = True
force_snat_range =
forward_bridge_interface =
all
fping_path = /usr/sbin/fping
gateway = None
gateway_v6 = None
heal_instance_info_cache_interval = 60
host = dims-ubuntu
image_cache_manager_interval = 2400
image_cache_subdirectory_name = _base
image_decryption_dir = /tmp
injected_network_template = /opt/stack/nova/nova/virt/interfaces.template
instance_build_timeout = 0
instance_delete_interval = 300
instance_dns_domain =
instance_dns_manager = nova.network.noop_dns_driver.NoopDNSDriver
instance_format = [instance: %(uuid)s]
instance_name_template = instance-%08x
instance_usage_audit = False
instance_usage_audit_period = month
instance_uuid_format = [instance: %(uuid)s]
instances_path = /opt/stack/data/nova/instances
internal_service_availability_zone = internal
iptables_bottom_regex =
iptables_drop_action = DROP
iptables_top_regex =
ipv6_backend = rfc2462
key_file = private/cakey.pem
keys_path = /opt/stack/data/nova/keys
keystone_ec2_insecure = False
keystone_ec2_url = http://10.0.0.9:5000/v2.0/ec2tokens
l3_lib = nova.network.l3.LinuxNetL3
linuxnet_interface_driver =
linuxnet_ovs_integration_bridge = br-int
live_migration_retry_count = 30
lockout_attempts = 5
lockout_minutes = 15
lockout_window = 15
log-config-append = None
log-date-format = %Y-%m-%d %H:%M:%S
log-dir = None
log-file = None
log-format = None
log_options = True
logging_context_format_string = %(asctime)s.%(msecs)03d %(color)s%(levelname)s %(name)s [%(request_id)s %(user_name)s %(project_name)s%(color)s] %(instance)s%(color)s%(message)s
logging_debug_format_suffix = from (pid=%(process)d) %(funcName)s %(pathname)s:%(lineno)d
logging_default_format_string = %(asctime)s.%(msecs)03d %(color)s%(levelname)s %(name)s [-%(color)s] %(instance)s%(color)s%(message)s
logging_exception_prefix = %(color)s%(asctime)s.%(msecs)03d TRACE %(name)s %(instance)s
max_age = 0
max_concurrent_builds = 10
max_concurrent_live_migrations = 1
max_header_line = 16384
max_local_block_devices = 3
maximum_instance_delete_attempts = 5
memcached_servers = None
metadata_cache_expiration = 15
metadata_host = 10.0.0.9
metadata_listen = 0.0.0.0
metadata_listen_port = 8775
metadata_manager = nova.api.manager.MetadataManager
metadata_port = 8775
metadata_workers = 2
migrate_max_retries = -1
mkisofs_cmd = genisoimage
monkey_patch = False
monkey_patch_modules =
nova.api.ec2.cloud:nova.notifications.notify_decorator
nova.compute.api:nova.notifications.notify_decorator
multi_host = False
my_block_storage_ip = 10.0.0.9
my_ip = 10.0.0.9
network_allocate_retries = 0
network_api_class = nova.network.neutronv2.api.API
network_device_mtu = None
network_driver = nova.network.linux_net
network_manager = nova.network.manager.VlanManager
network_size = 256
network_topic = network
networks_path = /opt/stack/data/nova/networks
neutron_default_tenant_id = default
non_inheritable_image_properties =
bittorrent
cache_in_nova
notification_driver =
notification_topics =
notifications
notify_api_faults = False
notify_on_state_change = None
null_kernel = nokernel
num_networks = 1
osapi_compute_ext_list =
osapi_compute_extension =
nova.api.openstack.compute.legacy_v2.contrib.standard_extensions
osapi_compute_link_prefix = None
osapi_compute_listen = 0.0.0.0
osapi_compute_listen_port = 8774
osapi_compute_unique_server_name_scope =
osapi_compute_workers = 2
osapi_glance_link_prefix = None
osapi_hide_server_address_states =
building
osapi_max_limit = 1000
ovs_vsctl_timeout = 120
password_length = 12
pci_alias =
pci_passthrough_whitelist =
periodic_enable = True
periodic_fuzzy_delay = 60
policy_default_rule = default
policy_dirs =
policy.d
policy_file = policy.json
preallocate_images = none
project_cert_subject = /C=US/ST=California/O=OpenStack/OU=NovaDev/CN=project-ca-%.16s-%s
public_interface = eth0
publish_errors = False
pybasedir = /opt/stack/nova
quota_cores = 20
quota_driver = nova.quota.DbQuotaDriver
quota_fixed_ips = -1
quota_floating_ips = 10
quota_injected_file_content_bytes = 10240
quota_injected_file_path_length = 255
quota_injected_files = 5
quota_instances = 10
quota_key_pairs = 100
quota_metadata_items = 128
quota_networks = 3
quota_ram = 51200
quota_security_group_rules = 20
quota_security_groups = 10
quota_server_group_members = 10
quota_server_groups = 10
ram_allocation_ratio = 0.0
reboot_timeout = 0
reclaim_instance_interval = 0
region_list =
remove_unused_base_images = True
remove_unused_original_minimum_age_seconds = 86400
report_interval = 10
rescue_timeout = 0
reservation_expire = 86400
reserved_host_disk_mb = 0
reserved_host_memory_mb = 512
resize_confirm_window = 0
resize_fs_using_block_device = False
resume_guests_state_on_host_boot = False
rootwrap_config = /etc/nova/rootwrap.conf
routing_source_ip = 10.0.0.9
rpc_backend = rabbit
rpc_response_timeout = 60
run_external_periodic_tasks = True
running_deleted_instance_action = reap
running_deleted_instance_poll_interval = 1800
running_deleted_instance_timeout = 0
s3_access_key = notchecked
s3_affix_tenant = False
s3_host = 10.0.0.9
s3_port = 3333
s3_secret_key = notchecked
s3_use_ssl = False
scheduler_available_filters =
nova.scheduler.filters.all_filters
scheduler_default_filters =
AvailabilityZoneFilter
ComputeCapabilitiesFilter
ComputeFilter
DiskFilter
ImagePropertiesFilter
RamFilter
RetryFilter
ServerGroupAffinityFilter
ServerGroupAntiAffinityFilter
scheduler_instance_sync_interval = 120
scheduler_manager = nova.scheduler.manager.SchedulerManager
scheduler_max_attempts = 3
scheduler_topic = scheduler
scheduler_tracks_instance_changes = True
scheduler_weight_classes =
nova.scheduler.weights.all_weighers
secure_proxy_ssl_header = None
security_group_api = neutron
send_arp_for_ha = False
send_arp_for_ha_count = 3
service_down_time = 60
servicegroup_driver = db
share_dhcp_address = False
shelved_offload_time = 0
shelved_poll_interval = 3600
shutdown_timeout = 60
snapshot_name_template = snapshot-%s
ssl_ca_file = None
ssl_cert_file = None
ssl_key_file = None
state_path = /opt/stack/data/nova
sync_power_state_interval = 600
syslog-log-facility = LOG_USER
tcp_keepidle = 600
teardown_unused_network_gateway = False
tempdir = None
transport_url = None
until_refresh = 0
update_dns_entries = False
update_resources_interval = 0
use-syslog = False
use-syslog-rfc-format = True
use_cow_images = True
use_forwarded_for = False
use_ipv6 = False
use_network_dns_servers = False
use_neutron_default_nets = False
use_project_ca = False
use_rootwrap_daemon = False
use_single_default_gateway = False
use_stderr = True
user_cert_subject = /C=US/ST=California/O=OpenStack/OU=NovaDev/CN=%.16s-%.16s-%s
vcpu_pin_set = None
vendordata_driver = nova.api.metadata.vendordata_json.JsonFileVendorData
verbose = True
vif_plugging_is_fatal = True
vif_plugging_timeout = 300
virt_mkfs =
vlan_interface = None
vlan_start = 100
volume_api_class = nova.volume.cinder.API
volume_usage_poll_interval = 0
vpn_flavor = m1.tiny
vpn_image_id = 0
vpn_ip = 10.0.0.9
vpn_key_suffix = -vpn
vpn_start = 1000
wsgi_default_pool_size = 1000
wsgi_keep_alive = True
wsgi_log_format = %(client_ip)s "%(request_line)s" status: %(status_code)s len: %(body_length)s time: %(wall_seconds).7f
ephemeral_storage_encryption:
cipher = aes-xts-plain64
enabled = False
key_size = 512
glance:
allowed_direct_url_schemes =
api_insecure = False
api_servers =
http://10.0.0.9:9292
host = 10.0.0.9
num_retries = 0
port = 9292
protocol = http
guestfs:
debug = False
image_file_url:
filesystems =
ironic:
admin_auth_token = ***
admin_password = ***
admin_tenant_name = None
admin_url = None
admin_username = None
api_endpoint = None
api_max_retries = 60
api_retry_interval = 2
api_version = 1
client_log_level = None
keymgr:
api_class = nova.keymgr.conf_key_mgr.ConfKeyManager
keystone_authtoken:
admin_password = ***
admin_tenant_name = admin
admin_token = ***
admin_user = None
auth-url = http://10.0.0.9:35357
auth_admin_prefix =
auth_host = 127.0.0.1
auth_plugin = password
auth_port = 35357
auth_protocol = https
auth_section = None
www_authenticate_uri = http://10.0.0.9:5000
auth_version = None
cache = None
cafile = /opt/stack/data/ca-bundle.pem
certfile = None
check_revocations_for_cached = False
delay_auth_decision = False
domain-id = None
domain-name = None
enforce_token_bind = permissive
hash_algorithms =
md5
http_connect_timeout = None
http_request_max_retries = 3
identity_uri = None
include_service_catalog = True
insecure = False
keyfile = None
memcache_pool_conn_get_timeout = 10
memcache_pool_dead_retry = 300
memcache_pool_maxsize = 10
memcache_pool_socket_timeout = 3
memcache_pool_unused_timeout = 60
memcache_secret_key = ***
memcache_security_strategy = None
memcache_use_advanced_pool = False
memcached_servers = None
password = password
project-domain-id = default
project-domain-name = None
project-id = None
project-name = service
region_name = None
revocation_cache_time = 10
signing_dir = /var/cache/nova
tenant-id = None
tenant-name = None
token_cache_time = 300
trust-id = None
user-domain-id = default
user-domain-name = None
user-id = None
user-name = nova
libvirt:
block_migration_flag = VIR_MIGRATE_UNDEFINE_SOURCE, VIR_MIGRATE_PEER2PEER, VIR_MIGRATE_LIVE, VIR_MIGRATE_TUNNELLED, VIR_MIGRATE_NON_SHARED_INC
checksum_base_images = False
checksum_interval_seconds = 3600
connection_uri =
cpu_mode = none
cpu_model = None
disk_cachemodes =
disk_prefix = None
gid_maps =
hw_disk_discard = None
hw_machine_type = None
image_info_filename_pattern = /opt/stack/data/nova/instances/_base/%(image)s.info
images_rbd_ceph_conf =
images_rbd_pool = rbd
images_type = default
images_volume_group = None
inject_key = False
inject_partition = -2
inject_password = False
iscsi_iface = None
iscsi_use_multipath = False
live_migration_bandwidth = 0
live_migration_completion_timeout = 800
live_migration_downtime = 500
live_migration_downtime_delay = 75
live_migration_downtime_steps = 10
live_migration_flag = VIR_MIGRATE_UNDEFINE_SOURCE, VIR_MIGRATE_PEER2PEER, VIR_MIGRATE_LIVE, VIR_MIGRATE_TUNNELLED
live_migration_progress_timeout = 150
live_migration_uri = qemu+ssh://stack@%s/system
mem_stats_period_seconds = 10
num_iscsi_scan_tries = 5
qemu_allowed_storage_drivers =
rbd_secret_uuid = None
rbd_user = None
remote_filesystem_transport = ssh
remove_unused_kernels = True
remove_unused_resized_minimum_age_seconds = 3600
rescue_image_id = None
rescue_kernel_id = None
rescue_ramdisk_id = None
rng_dev_path = None
snapshot_compression = False
snapshot_image_format = None
snapshots_directory = /opt/stack/data/nova/instances/snapshots
sparse_logical_volumes = False
sysinfo_serial = auto
uid_maps =
use_usb_tablet = False
use_virtio_for_bridges = True
virt_type = kvm
volume_clear = zero
volume_clear_size = 0
wait_soft_reboot_seconds = 120
xen_hvmloader_path = /usr/lib/xen/boot/hvmloader
mks:
enabled = False
mksproxy_base_url = http://127.0.0.1:6090/
neutron:
admin_auth_url = http://10.0.0.9:35357/v2.0
admin_password = ***
admin_tenant_id = None
admin_tenant_name = service
admin_user_id = None
admin_username = neutron
auth_plugin = None
auth_section = None
auth_strategy = keystone
cafile = None
certfile = None
extension_sync_interval = 600
insecure = False
keyfile = None
metadata_proxy_shared_secret = ***
ovs_bridge = br-int
region_name = RegionOne
service_metadata_proxy = True
timeout = None
url = http://10.0.0.9:9696
osapi_v21:
enabled = True
extensions_blacklist =
extensions_whitelist =
oslo_concurrency:
disable_process_locking = False
lock_path = /opt/stack/data/nova
oslo_messaging_rabbit:
amqp_auto_delete = False
amqp_durable_queues = False
fake_rabbit = False
heartbeat_rate = 2
heartbeat_timeout_threshold = 60
kombu_reconnect_delay = 1.0
kombu_reconnect_timeout = 60
kombu_ssl_ca_certs =
kombu_ssl_certfile =
kombu_ssl_keyfile =
kombu_ssl_version =
rabbit_ha_queues = False
rabbit_host = localhost
rabbit_hosts =
10.0.0.9
rabbit_login_method = AMQPLAIN
rabbit_max_retries = 0
rabbit_password = ***
rabbit_port = 5672
rabbit_retry_backoff = 2
rabbit_retry_interval = 1
rabbit_use_ssl = False
rabbit_userid = stackrabbit
rabbit_virtual_host = /
rpc_conn_pool_size = 30
send_single_reply = False
oslo_middleware:
max_request_body_size = 114688
oslo_versionedobjects:
fatal_exception_format_errors = False
rdp:
enabled = False
html5_proxy_base_url = http://127.0.0.1:6083/
remote_debug:
host = None
port = None
serial_console:
base_url = ws://127.0.0.1:6083/
enabled = False
listen = 127.0.0.1
port_range = 10000:20000
proxyclient_address = 127.0.0.1
spice:
agent_enabled = True
enabled = False
html5proxy_base_url = http://10.0.0.9:6082/spice_auto.html
keymap = en-us
server_listen = 127.0.0.1
server_proxyclient_address = 127.0.0.1
ssl:
ca_file = None
cert_file = None
key_file = None
upgrade_levels:
baseapi = None
cells = None
cert = None
compute = None
conductor = None
console = None
consoleauth = None
network = None
scheduler = None
vnc:
enabled = False
keymap = en-us
novncproxy_base_url = http://10.0.0.9:6080/vnc_auto.html
vncserver_listen = 127.0.0.1
vncserver_proxyclient_address = 127.0.0.1
xvpvncproxy_base_url = http://10.0.0.9:6081/console
workarounds:
destroy_after_evacuate = True
disable_libvirt_livesnapshot = True
disable_rootwrap = False
handle_virt_lifecycle_events = True
