UsageΒΆ
Every long running service process should have a call to install a signal handler which will trigger the guru meditation framework upon receipt of SIGUSR1/SIGUSR2. This will result in the process dumping a complete report of its current state to stderr.
For RPC listeners, it may also be desirable to install some kind of hook in the RPC request dispatcher that will save a guru meditation report whenever the processing of a request results in an uncaught exception. It could save these reports to a well known directory (/var/log/openstack/<project>/<service>/) for later analysis by the sysadmin or automated bug analysis tools.
To use oslo.reports in a project, you need to add the following call to
setup_autorun() somewhere really
early in the startup sequence of the process:
from oslo_reports import guru_meditation_report as gmr
gmr.TextGuruMeditation.setup_autorun(version='13.0.0')
Note that the version parameter is the version of the component itself.
To trigger the report to be generated:
kill -SIGUSR2 <process_id>
Here is a sample report:
========================================================================
==== Guru Meditation ====
========================================================================
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
========================================================================
==== Package ====
========================================================================
product = OpenStack Nova
vendor = OpenStack Foundation
version = 13.0.0
========================================================================
==== Threads ====
========================================================================
------ Thread #140417215547200 ------
/usr/local/lib/python2.7/dist-packages/eventlet/hubs/hub.py:346 in run
`self.wait(sleep_time)`
/usr/local/lib/python2.7/dist-packages/eventlet/hubs/poll.py:82 in wait
`sleep(seconds)`
========================================================================
==== Green Threads ====
========================================================================
------ Green Thread ------
/usr/local/bin/nova-api:10 in <module>
`sys.exit(main())`
/opt/stack/nova/nova/cmd/api.py:57 in main
`launcher.wait()`
/usr/local/lib/python2.7/dist-packages/oslo_service/service.py:511 in wait
`self._respawn_children()`
/usr/local/lib/python2.7/dist-packages/oslo_service/service.py:495 in _respawn_children
`eventlet.greenthread.sleep(self.wait_interval)`
/usr/local/lib/python2.7/dist-packages/eventlet/greenthread.py:34 in sleep
`hub.switch()`
/usr/local/lib/python2.7/dist-packages/eventlet/hubs/hub.py:294 in switch
`return self.greenlet.switch()`
------ Green Thread ------
No Traceback!
========================================================================
==== Processes ====
========================================================================
Process 14756 (under 1) [ run by: stack (1001), state: running ]
Process 14770 (under 14756) [ run by: stack (1001), state: sleeping ]
Process 14771 (under 14756) [ run by: stack (1001), state: sleeping ]
Process 14776 (under 14756) [ run by: stack (1001), state: sleeping ]
Process 14777 (under 14756) [ run by: stack (1001), state: sleeping ]
Process 14784 (under 14756) [ run by: stack (1001), state: sleeping ]
Process 14785 (under 14756) [ run by: stack (1001), state: sleeping ]
========================================================================
==== Configuration ====
========================================================================
api_database:
connection = ***
connection_debug = 0
connection_trace = False
idle_timeout = 3600
max_overflow = None
max_pool_size = None
max_retries = 10
mysql_sql_mode = TRADITIONAL
pool_timeout = None
retry_interval = 10
slave_connection = ***
sqlite_synchronous = True
cells:
bandwidth_update_interval = 600
call_timeout = 60
capabilities =
hypervisor=xenserver;kvm
os=linux;windows
cell_type = compute
enable = False
instance_update_sync_database_limit = 100
manager = nova.cells.manager.CellsManager
mute_child_interval = 300
name = nova
reserve_percent = 10.0
topic = cells
cinder:
cafile = None
catalog_info = volumev2:cinderv2:publicURL
certfile = None
cross_az_attach = True
endpoint_template = None
http_retries = 3
insecure = False
keyfile = None
os_region_name = RegionOne
timeout = None
conductor:
manager = nova.conductor.manager.ConductorManager
topic = conductor
use_local = False
workers = 2
database:
backend = sqlalchemy
connection = ***
connection_debug = 0
connection_trace = False
db_inc_retry_interval = True
db_max_retries = 20
db_max_retry_interval = 10
db_retry_interval = 1
idle_timeout = 3600
max_overflow = None
max_pool_size = None
max_retries = 10
min_pool_size = 1
mysql_sql_mode = TRADITIONAL
pool_timeout = None
retry_interval = 10
slave_connection = ***
sqlite_db = nova.sqlite
sqlite_synchronous = True
use_db_reconnect = False
use_tpool = False
default:
allow_instance_snapshots = True
allow_resize_to_same_host = True
allow_same_net_traffic = True
api_paste_config = /etc/nova/api-paste.ini
api_rate_limit = False
auth_strategy = keystone
auto_assign_floating_ip = False
bandwidth_poll_interval = 600
bindir = /usr/local/bin
block_device_allocate_retries = 60
block_device_allocate_retries_interval = 3
boot_script_template = /opt/stack/nova/nova/cloudpipe/bootscript.template
ca_file = cacert.pem
ca_path = /opt/stack/data/nova/CA
cert_manager = nova.cert.manager.CertManager
cert_topic = cert
client_socket_timeout = 900
cnt_vpn_clients = 0
compute_available_monitors = None
compute_driver = libvirt.LibvirtDriver
compute_manager = nova.compute.manager.ComputeManager
compute_monitors =
compute_resources =
vcpu
compute_stats_class = nova.compute.stats.Stats
compute_topic = compute
config-dir = None
config-file =
/etc/nova/nova.conf
config_drive_format = iso9660
config_drive_skip_versions = 1.0 2007-01-19 2007-03-01 2007-08-29 2007-10-10 2007-12-15 2008-02-01 2008-09-01
console_host = dims-ubuntu
console_manager = nova.console.manager.ConsoleProxyManager
console_topic = console
consoleauth_manager = nova.consoleauth.manager.ConsoleAuthManager
consoleauth_topic = consoleauth
control_exchange = nova
cpu_allocation_ratio = 0.0
create_unique_mac_address_attempts = 5
crl_file = crl.pem
db_driver = nova.db
debug = True
default_access_ip_network_name = None
default_availability_zone = nova
default_ephemeral_format = ext4
default_flavor = m1.small
default_floating_pool = public
default_log_levels =
amqp=WARN
amqplib=WARN
boto=WARN
glanceclient=WARN
iso8601=WARN
keystonemiddleware=WARN
oslo_messaging=INFO
qpid=WARN
requests.packages.urllib3.connectionpool=WARN
routes.middleware=WARN
sqlalchemy=WARN
stevedore=WARN
suds=INFO
urllib3.connectionpool=WARN
websocket=WARN
default_notification_level = INFO
default_publisher_id = None
default_schedule_zone = None
defer_iptables_apply = False
dhcp_domain = novalocal
dhcp_lease_time = 86400
dhcpbridge = /usr/local/bin/nova-dhcpbridge
dhcpbridge_flagfile =
/etc/nova/nova.conf
dmz_cidr =
dmz_mask = 255.255.255.0
dmz_net = 10.0.0.0
dns_server =
dns_update_periodic_interval = -1
dnsmasq_config_file =
ebtables_exec_attempts = 3
ebtables_retry_interval = 1.0
ec2_dmz_host = 10.0.0.9
ec2_host = 10.0.0.9
ec2_listen = 0.0.0.0
ec2_listen_port = 8773
ec2_path = /
ec2_port = 8773
ec2_private_dns_show_ip = False
ec2_scheme = http
ec2_strict_validation = True
ec2_timestamp_expiry = 300
ec2_workers = 2
enable_instance_password = True
enable_network_quota = False
enable_new_services = True
enabled_apis =
ec2
metadata
osapi_compute
enabled_ssl_apis =
fake_call = False
fake_network = False
fatal_deprecations = False
fatal_exception_format_errors = False
firewall_driver = nova.virt.firewall.NoopFirewallDriver
fixed_ip_disassociate_timeout = 600
fixed_range_v6 = fd00::/48
flat_injected = False
flat_interface = None
flat_network_bridge = None
flat_network_dns = 8.8.4.4
floating_ip_dns_manager = nova.network.noop_dns_driver.NoopDNSDriver
force_config_drive = True
force_dhcp_release = True
force_raw_images = True
force_snat_range =
forward_bridge_interface =
all
fping_path = /usr/sbin/fping
gateway = None
gateway_v6 = None
heal_instance_info_cache_interval = 60
host = dims-ubuntu
image_cache_manager_interval = 2400
image_cache_subdirectory_name = _base
image_decryption_dir = /tmp
injected_network_template = /opt/stack/nova/nova/virt/interfaces.template
instance_build_timeout = 0
instance_delete_interval = 300
instance_dns_domain =
instance_dns_manager = nova.network.noop_dns_driver.NoopDNSDriver
instance_format = [instance: %(uuid)s]
instance_name_template = instance-%08x
instance_usage_audit = False
instance_usage_audit_period = month
instance_uuid_format = [instance: %(uuid)s]
instances_path = /opt/stack/data/nova/instances
internal_service_availability_zone = internal
iptables_bottom_regex =
iptables_drop_action = DROP
iptables_top_regex =
ipv6_backend = rfc2462
key_file = private/cakey.pem
keys_path = /opt/stack/data/nova/keys
keystone_ec2_insecure = False
keystone_ec2_url = http://10.0.0.9:5000/v2.0/ec2tokens
l3_lib = nova.network.l3.LinuxNetL3
linuxnet_interface_driver =
linuxnet_ovs_integration_bridge = br-int
live_migration_retry_count = 30
lockout_attempts = 5
lockout_minutes = 15
lockout_window = 15
log-config-append = None
log-date-format = %Y-%m-%d %H:%M:%S
log-dir = None
log-file = None
log-format = None
log_options = True
logging_context_format_string = %(asctime)s.%(msecs)03d %(color)s%(levelname)s %(name)s [%(request_id)s %(user_name)s %(project_name)s%(color)s] %(instance)s%(color)s%(message)s
logging_debug_format_suffix = from (pid=%(process)d) %(funcName)s %(pathname)s:%(lineno)d
logging_default_format_string = %(asctime)s.%(msecs)03d %(color)s%(levelname)s %(name)s [-%(color)s] %(instance)s%(color)s%(message)s
logging_exception_prefix = %(color)s%(asctime)s.%(msecs)03d TRACE %(name)s %(instance)s
max_age = 0
max_concurrent_builds = 10
max_concurrent_live_migrations = 1
max_header_line = 16384
max_local_block_devices = 3
maximum_instance_delete_attempts = 5
memcached_servers = None
metadata_cache_expiration = 15
metadata_host = 10.0.0.9
metadata_listen = 0.0.0.0
metadata_listen_port = 8775
metadata_manager = nova.api.manager.MetadataManager
metadata_port = 8775
metadata_workers = 2
migrate_max_retries = -1
mkisofs_cmd = genisoimage
monkey_patch = False
monkey_patch_modules =
nova.api.ec2.cloud:nova.notifications.notify_decorator
nova.compute.api:nova.notifications.notify_decorator
multi_host = False
multi_instance_display_name_template = %(name)s-%(count)d
my_block_storage_ip = 10.0.0.9
my_ip = 10.0.0.9
network_allocate_retries = 0
network_api_class = nova.network.neutronv2.api.API
network_device_mtu = None
network_driver = nova.network.linux_net
network_manager = nova.network.manager.VlanManager
network_size = 256
network_topic = network
networks_path = /opt/stack/data/nova/networks
neutron_default_tenant_id = default
non_inheritable_image_properties =
bittorrent
cache_in_nova
notification_driver =
notification_topics =
notifications
notify_api_faults = False
notify_on_state_change = None
null_kernel = nokernel
num_networks = 1
osapi_compute_ext_list =
osapi_compute_extension =
nova.api.openstack.compute.legacy_v2.contrib.standard_extensions
osapi_compute_link_prefix = None
osapi_compute_listen = 0.0.0.0
osapi_compute_listen_port = 8774
osapi_compute_unique_server_name_scope =
osapi_compute_workers = 2
osapi_glance_link_prefix = None
osapi_hide_server_address_states =
building
osapi_max_limit = 1000
ovs_vsctl_timeout = 120
password_length = 12
pci_alias =
pci_passthrough_whitelist =
periodic_enable = True
periodic_fuzzy_delay = 60
policy_default_rule = default
policy_dirs =
policy.d
policy_file = policy.json
preallocate_images = none
project_cert_subject = /C=US/ST=California/O=OpenStack/OU=NovaDev/CN=project-ca-%.16s-%s
public_interface = eth0
publish_errors = False
pybasedir = /opt/stack/nova
quota_cores = 20
quota_driver = nova.quota.DbQuotaDriver
quota_fixed_ips = -1
quota_floating_ips = 10
quota_injected_file_content_bytes = 10240
quota_injected_file_path_length = 255
quota_injected_files = 5
quota_instances = 10
quota_key_pairs = 100
quota_metadata_items = 128
quota_networks = 3
quota_ram = 51200
quota_security_group_rules = 20
quota_security_groups = 10
quota_server_group_members = 10
quota_server_groups = 10
ram_allocation_ratio = 0.0
reboot_timeout = 0
reclaim_instance_interval = 0
region_list =
remove_unused_base_images = True
remove_unused_original_minimum_age_seconds = 86400
report_interval = 10
rescue_timeout = 0
reservation_expire = 86400
reserved_host_disk_mb = 0
reserved_host_memory_mb = 512
resize_confirm_window = 0
resize_fs_using_block_device = False
resume_guests_state_on_host_boot = False
rootwrap_config = /etc/nova/rootwrap.conf
routing_source_ip = 10.0.0.9
rpc_backend = rabbit
rpc_response_timeout = 60
run_external_periodic_tasks = True
running_deleted_instance_action = reap
running_deleted_instance_poll_interval = 1800
running_deleted_instance_timeout = 0
s3_access_key = notchecked
s3_affix_tenant = False
s3_host = 10.0.0.9
s3_port = 3333
s3_secret_key = notchecked
s3_use_ssl = False
scheduler_available_filters =
nova.scheduler.filters.all_filters
scheduler_default_filters =
AvailabilityZoneFilter
ComputeCapabilitiesFilter
ComputeFilter
DiskFilter
ImagePropertiesFilter
RamFilter
RetryFilter
ServerGroupAffinityFilter
ServerGroupAntiAffinityFilter
scheduler_instance_sync_interval = 120
scheduler_manager = nova.scheduler.manager.SchedulerManager
scheduler_max_attempts = 3
scheduler_topic = scheduler
scheduler_tracks_instance_changes = True
scheduler_weight_classes =
nova.scheduler.weights.all_weighers
secure_proxy_ssl_header = None
security_group_api = neutron
send_arp_for_ha = False
send_arp_for_ha_count = 3
service_down_time = 60
servicegroup_driver = db
share_dhcp_address = False
shelved_offload_time = 0
shelved_poll_interval = 3600
shutdown_timeout = 60
snapshot_name_template = snapshot-%s
ssl_ca_file = None
ssl_cert_file = None
ssl_key_file = None
state_path = /opt/stack/data/nova
sync_power_state_interval = 600
syslog-log-facility = LOG_USER
tcp_keepidle = 600
teardown_unused_network_gateway = False
tempdir = None
transport_url = None
until_refresh = 0
update_dns_entries = False
update_resources_interval = 0
use-syslog = False
use-syslog-rfc-format = True
use_cow_images = True
use_forwarded_for = False
use_ipv6 = False
use_network_dns_servers = False
use_neutron_default_nets = False
use_project_ca = False
use_rootwrap_daemon = False
use_single_default_gateway = False
use_stderr = True
user_cert_subject = /C=US/ST=California/O=OpenStack/OU=NovaDev/CN=%.16s-%.16s-%s
vcpu_pin_set = None
vendordata_driver = nova.api.metadata.vendordata_json.JsonFileVendorData
verbose = True
vif_plugging_is_fatal = True
vif_plugging_timeout = 300
virt_mkfs =
vlan_interface = None
vlan_start = 100
volume_api_class = nova.volume.cinder.API
volume_usage_poll_interval = 0
vpn_flavor = m1.tiny
vpn_image_id = 0
vpn_ip = 10.0.0.9
vpn_key_suffix = -vpn
vpn_start = 1000
wsgi_default_pool_size = 1000
wsgi_keep_alive = True
wsgi_log_format = %(client_ip)s "%(request_line)s" status: %(status_code)s len: %(body_length)s time: %(wall_seconds).7f
ephemeral_storage_encryption:
cipher = aes-xts-plain64
enabled = False
key_size = 512
glance:
allowed_direct_url_schemes =
api_insecure = False
api_servers =
http://10.0.0.9:9292
host = 10.0.0.9
num_retries = 0
port = 9292
protocol = http
guestfs:
debug = False
image_file_url:
filesystems =
ironic:
admin_auth_token = ***
admin_password = ***
admin_tenant_name = None
admin_url = None
admin_username = None
api_endpoint = None
api_max_retries = 60
api_retry_interval = 2
api_version = 1
client_log_level = None
keymgr:
api_class = nova.keymgr.conf_key_mgr.ConfKeyManager
keystone_authtoken:
admin_password = ***
admin_tenant_name = admin
admin_token = ***
admin_user = None
auth-url = http://10.0.0.9:35357
auth_admin_prefix =
auth_host = 127.0.0.1
auth_plugin = password
auth_port = 35357
auth_protocol = https
auth_section = None
auth_uri = http://10.0.0.9:5000
auth_version = None
cache = None
cafile = /opt/stack/data/ca-bundle.pem
certfile = None
check_revocations_for_cached = False
delay_auth_decision = False
domain-id = None
domain-name = None
enforce_token_bind = permissive
hash_algorithms =
md5
http_connect_timeout = None
http_request_max_retries = 3
identity_uri = None
include_service_catalog = True
insecure = False
keyfile = None
memcache_pool_conn_get_timeout = 10
memcache_pool_dead_retry = 300
memcache_pool_maxsize = 10
memcache_pool_socket_timeout = 3
memcache_pool_unused_timeout = 60
memcache_secret_key = ***
memcache_security_strategy = None
memcache_use_advanced_pool = False
memcached_servers = None
password = password
project-domain-id = default
project-domain-name = None
project-id = None
project-name = service
region_name = None
revocation_cache_time = 10
signing_dir = /var/cache/nova
tenant-id = None
tenant-name = None
token_cache_time = 300
trust-id = None
user-domain-id = default
user-domain-name = None
user-id = None
user-name = nova
libvirt:
block_migration_flag = VIR_MIGRATE_UNDEFINE_SOURCE, VIR_MIGRATE_PEER2PEER, VIR_MIGRATE_LIVE, VIR_MIGRATE_TUNNELLED, VIR_MIGRATE_NON_SHARED_INC
checksum_base_images = False
checksum_interval_seconds = 3600
connection_uri =
cpu_mode = none
cpu_model = None
disk_cachemodes =
disk_prefix = None
gid_maps =
hw_disk_discard = None
hw_machine_type = None
image_info_filename_pattern = /opt/stack/data/nova/instances/_base/%(image)s.info
images_rbd_ceph_conf =
images_rbd_pool = rbd
images_type = default
images_volume_group = None
inject_key = False
inject_partition = -2
inject_password = False
iscsi_iface = None
iscsi_use_multipath = False
live_migration_bandwidth = 0
live_migration_completion_timeout = 800
live_migration_downtime = 500
live_migration_downtime_delay = 75
live_migration_downtime_steps = 10
live_migration_flag = VIR_MIGRATE_UNDEFINE_SOURCE, VIR_MIGRATE_PEER2PEER, VIR_MIGRATE_LIVE, VIR_MIGRATE_TUNNELLED
live_migration_progress_timeout = 150
live_migration_uri = qemu+ssh://stack@%s/system
mem_stats_period_seconds = 10
num_iscsi_scan_tries = 5
qemu_allowed_storage_drivers =
rbd_secret_uuid = None
rbd_user = None
remote_filesystem_transport = ssh
remove_unused_kernels = True
remove_unused_resized_minimum_age_seconds = 3600
rescue_image_id = None
rescue_kernel_id = None
rescue_ramdisk_id = None
rng_dev_path = None
snapshot_compression = False
snapshot_image_format = None
snapshots_directory = /opt/stack/data/nova/instances/snapshots
sparse_logical_volumes = False
sysinfo_serial = auto
uid_maps =
use_usb_tablet = False
use_virtio_for_bridges = True
virt_type = kvm
volume_clear = zero
volume_clear_size = 0
wait_soft_reboot_seconds = 120
xen_hvmloader_path = /usr/lib/xen/boot/hvmloader
mks:
enabled = False
mksproxy_base_url = http://127.0.0.1:6090/
neutron:
admin_auth_url = http://10.0.0.9:35357/v2.0
admin_password = ***
admin_tenant_id = None
admin_tenant_name = service
admin_user_id = None
admin_username = neutron
auth_plugin = None
auth_section = None
auth_strategy = keystone
cafile = None
certfile = None
extension_sync_interval = 600
insecure = False
keyfile = None
metadata_proxy_shared_secret = ***
ovs_bridge = br-int
region_name = RegionOne
service_metadata_proxy = True
timeout = None
url = http://10.0.0.9:9696
osapi_v21:
enabled = True
extensions_blacklist =
extensions_whitelist =
oslo_concurrency:
disable_process_locking = False
lock_path = /opt/stack/data/nova
oslo_messaging_rabbit:
amqp_auto_delete = False
amqp_durable_queues = False
fake_rabbit = False
heartbeat_rate = 2
heartbeat_timeout_threshold = 60
kombu_reconnect_delay = 1.0
kombu_reconnect_timeout = 60
kombu_ssl_ca_certs =
kombu_ssl_certfile =
kombu_ssl_keyfile =
kombu_ssl_version =
rabbit_ha_queues = False
rabbit_host = localhost
rabbit_hosts =
10.0.0.9
rabbit_login_method = AMQPLAIN
rabbit_max_retries = 0
rabbit_password = ***
rabbit_port = 5672
rabbit_retry_backoff = 2
rabbit_retry_interval = 1
rabbit_use_ssl = False
rabbit_userid = stackrabbit
rabbit_virtual_host = /
rpc_conn_pool_size = 30
send_single_reply = False
oslo_middleware:
max_request_body_size = 114688
oslo_versionedobjects:
fatal_exception_format_errors = False
rdp:
enabled = False
html5_proxy_base_url = http://127.0.0.1:6083/
remote_debug:
host = None
port = None
serial_console:
base_url = ws://127.0.0.1:6083/
enabled = False
listen = 127.0.0.1
port_range = 10000:20000
proxyclient_address = 127.0.0.1
spice:
agent_enabled = True
enabled = False
html5proxy_base_url = http://10.0.0.9:6082/spice_auto.html
keymap = en-us
server_listen = 127.0.0.1
server_proxyclient_address = 127.0.0.1
ssl:
ca_file = None
cert_file = None
key_file = None
upgrade_levels:
baseapi = None
cells = None
cert = None
compute = None
conductor = None
console = None
consoleauth = None
network = None
scheduler = None
vnc:
enabled = False
keymap = en-us
novncproxy_base_url = http://10.0.0.9:6080/vnc_auto.html
vncserver_listen = 127.0.0.1
vncserver_proxyclient_address = 127.0.0.1
xvpvncproxy_base_url = http://10.0.0.9:6081/console
workarounds:
destroy_after_evacuate = True
disable_libvirt_livesnapshot = True
disable_rootwrap = False
handle_virt_lifecycle_events = True