# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import socket
import time
from ironic_lib import utils
from oslo_concurrency import processutils
from oslo_log import log
from tooz import coordination
from ironic_python_agent import errors
from ironic_python_agent import hardware
LOG = log.getLogger(__name__)
NETWORK_BURNIN_ROLES = frozenset(['writer', 'reader'])
NETWORK_READER_CYCLE = 30
[docs]
def stress_ng_cpu(node):
"""Burn-in the CPU with stress-ng
Run stress-ng on a configurable number of CPUs for
a configurable amount of time. Without config use
all CPUs and stress them for 24 hours.
:param node: Ironic node object
:raises: CommandExecutionError if the execution of stress-ng fails.
"""
info = node.get('driver_info', {})
cpu = info.get('agent_burnin_cpu_cpu', 0)
timeout = info.get('agent_burnin_cpu_timeout', 86400)
outputfile = info.get('agent_burnin_cpu_outputfile', None)
args = ('stress-ng', '--cpu', cpu, '--timeout', timeout,
'--metrics-brief')
if outputfile:
args += ('--log-file', outputfile,)
LOG.debug('Burn-in stress_ng_cpu command: %s', args)
try:
_, err = utils.execute(*args)
# stress-ng reports on stderr only
LOG.info(err)
except (processutils.ProcessExecutionError, OSError) as e:
error_msg = "stress-ng (cpu) failed with error %s" % e
LOG.error(error_msg)
raise errors.CommandExecutionError(error_msg)
[docs]
def stress_ng_vm(node):
"""Burn-in the memory with the vm stressor in stress-ng
Run stress-ng with a configurable number of workers on
a configurable amount of the available memory for
a configurable amount of time. Without config use
as many workers as CPUs, 98% of the memory and stress
it for 24 hours.
:param node: Ironic node object
:raises: CommandExecutionError if the execution of stress-ng fails.
"""
info = node.get('driver_info', {})
vm = info.get('agent_burnin_vm_vm', 0)
vm_bytes = info.get('agent_burnin_vm_vm-bytes', '98%')
timeout = info.get('agent_burnin_vm_timeout', 86400)
outputfile = info.get('agent_burnin_vm_outputfile', None)
args = ('stress-ng', '--vm', vm, '--vm-bytes', vm_bytes,
'--timeout', timeout, '--metrics-brief')
if outputfile:
args += ('--log-file', outputfile,)
LOG.debug('Burn-in stress_ng_vm command: %s', args)
try:
_, err = utils.execute(*args)
# stress-ng reports on stderr only
LOG.info(err)
except (processutils.ProcessExecutionError, OSError) as e:
error_msg = "stress-ng (vm) failed with error %s" % e
LOG.error(error_msg)
raise errors.CommandExecutionError(error_msg)
def _smart_test_status(device):
"""Get the SMART test status of a device
:param device: The device to check.
:raises: CommandExecutionError if the execution of smartctl fails.
:returns: A string with the SMART test status of the device and
None if the status is not available.
"""
args = ['smartctl', '-ja', device.name]
try:
out, _ = utils.execute(*args)
smart_info = json.loads(out)
if smart_info:
return smart_info['ata_smart_data'][
'self_test']['status']['string']
except (processutils.ProcessExecutionError, OSError, KeyError) as e:
LOG.error('SMART test on %(device)s failed with '
'%(err)s', {'device': device.name, 'err': e})
return None
def _run_smart_test(devices):
"""Launch a SMART test on the passed devices
:param devices: A list of device objects to check.
:raises: CommandExecutionError if the execution of smartctl fails.
:raises: CleaningError if the SMART test on any of the devices fails.
"""
failed_devices = []
for device in devices:
args = ['smartctl', '-t', 'long', device.name]
LOG.info('SMART self test command: %s',
' '.join(map(str, args)))
try:
utils.execute(*args)
except (processutils.ProcessExecutionError, OSError) as e:
LOG.error("Starting SMART test on %(device)s failed with: "
"%(err)s", {'device': device.name, 'err': e})
failed_devices.append(device.name)
if failed_devices:
error_msg = ("fio (disk) failed to start SMART self test on %s",
', '.join(failed_devices))
raise errors.CleaningError(error_msg)
# wait for the test to finish and report the test results
failed_devices = []
while True:
for device in list(devices):
status = _smart_test_status(device)
if status is None:
devices.remove(device)
continue
if "in progress" in status:
msg = "SMART test still running on %s ..." % device.name
LOG.debug(msg)
continue
if "completed without error" in status:
msg = "%s passed SMART test" % device.name
LOG.info(msg)
devices.remove(device)
continue
failed_devices.append(device.name)
LOG.warning("%(device)s failed SMART test with: %(err)s",
{'device': device.name, 'err': status})
devices.remove(device)
if not devices:
break
LOG.info("SMART tests still running ...")
time.sleep(30)
# fail the clean step if the SMART test has failed
if failed_devices:
msg = ('fio (disk) SMART test failed for %s' % ' '.join(
map(str, failed_devices)))
raise errors.CleaningError(msg)
[docs]
def fio_disk(node):
"""Burn-in the disks with fio
Run an fio randrw job for a configurable number of iterations
or a given amount of time.
:param node: Ironic node object
:raises: CommandExecutionError if the execution of fio fails.
"""
info = node.get('driver_info', {})
# 4 iterations, same as badblock's default
loops = info.get('agent_burnin_fio_disk_loops', 4)
runtime = info.get('agent_burnin_fio_disk_runtime', 0)
outputfile = info.get('agent_burnin_fio_disk_outputfile', None)
args = ['fio', '--rw', 'readwrite', '--bs', '4k', '--direct', 1,
'--ioengine', 'libaio', '--iodepth', '32', '--verify',
'crc32c', '--verify_dump', 1, '--continue_on_error', 'verify',
'--loops', loops, '--runtime', runtime, '--time_based']
if outputfile:
args.extend(['--output-format', 'json', '--output', outputfile])
devices = hardware.list_all_block_devices()
for device in devices:
args.extend(['--name', device.name])
LOG.debug('Burn-in fio disk command: %s', ' '.join(map(str, args)))
try:
out, _ = utils.execute(*args)
# fio reports on stdout
LOG.info(out)
except (processutils.ProcessExecutionError, OSError) as e:
error_msg = "fio (disk) failed with error %s" % e
LOG.error(error_msg)
raise errors.CommandExecutionError(error_msg)
# if configured, run a smart self test on all devices and fail the
# step if any of the devices reports an error
smart_test = info.get('agent_burnin_fio_disk_smart_test', False)
if smart_test:
_run_smart_test(devices)
def _do_fio_network(writer, runtime, partner, outputfile):
args = ['fio', '--ioengine', 'net', '--port', '9000', '--fill_device', 1,
'--group_reporting', '--gtod_reduce', 1, '--numjobs', 16]
if writer:
xargs = ['--name', 'writer', '--rw', 'write', '--runtime', runtime,
'--time_based', '--listen']
else:
xargs = ['--name', 'reader', '--rw', 'read', '--hostname', partner]
args.extend(xargs)
if outputfile:
args.extend(['--output-format', 'json', '--output', outputfile])
while True:
LOG.info('Burn-in fio network command: %s', ' '.join(map(str, args)))
try:
out, err = utils.execute(*args)
# fio reports on stdout
LOG.info(out)
break
except processutils.ProcessExecutionError as e:
error_msg = "fio (network) failed with error %s" % e
LOG.error(error_msg)
if writer:
raise errors.CommandExecutionError(error_msg)
# While the writer blocks in fio, the reader fails with
# 'Connection {refused, timeout}' errors if the partner
# is not ready, so we need to wait explicitly. Using the
# exit code accounts for both, logging to stderr as well
# as to a file.
if e.exit_code == 16:
LOG.info("fio (network): reader retrying in %s seconds ...",
NETWORK_READER_CYCLE)
time.sleep(NETWORK_READER_CYCLE)
else:
raise errors.CommandExecutionError(error_msg)
def _find_network_burnin_partner_and_role(backend_url, group_name, timeout):
"""Find a partner node for network burn-in and get our role.
:param backend_url: The tooz backend url.
:param group_name: The tooz group name for pairing.
:param timeout:Timeout in seconds for a node to wait for a partner.
:returns: A set with the partner node and the role of the local node.
"""
member_id = socket.gethostname()
coordinator = coordination.get_coordinator(backend_url, member_id)
coordinator.start(start_heart=True)
groups = coordinator.get_groups()
for group in groups.get():
if group_name == group.decode('utf-8'):
LOG.debug("Found group %s", group_name)
break
else:
LOG.info("Creating group %s", group_name)
coordinator.create_group(group_name)
def join_group(group_name):
request = coordinator.join_group(group_name)
request.get()
def leave_group(group_name):
request = coordinator.leave_group(group_name)
request.get()
# Attempt to get the pairing lock. The lock is released when:
# a) a node enters the group and is the first to join, or
# b) a node enters second, finished pairing, sees
# the pairing node exiting, and left itself.
# The lock 'walls' all nodes willing to pair.
group_lock = coordinator.get_lock("group_lock")
with group_lock:
# we need the initial members in order to know the first
# node (which may leave quickly when we join)
init_members = coordinator.get_members(group_name)
LOG.info("Original group members are %s", init_members.get())
members_cnt = len(init_members.get())
join_group(group_name)
# we assign the first node the writer role since it will
# leave the group first, it may be ready once the second
# node leaves the group, and we save one wait cycle
if not members_cnt:
first = True
role = "writer"
group_lock.release() # allow second node to enter
else:
first = False
role = "reader"
partner = None
start_pairing = time.time()
while time.time() - start_pairing < timeout:
if first:
# we are the first and therefore need to wait
# for another node to join
members = coordinator.get_members(group_name)
members_cnt = len(members.get())
else:
# use the initial members in case the other
# node leaves before we get an updated list
members = init_members
assert members_cnt < 3
if members_cnt == 2 or not first:
LOG.info("Two members, start pairing...")
for member in members.get():
node = member.decode('utf-8')
if node != member_id:
partner = node
if not partner:
error_msg = ("fio (network) no partner to pair found")
raise errors.CleaningError(error_msg)
# if you are the second to enter, wait for the first to exit
if not first:
members = coordinator.get_members(group_name)
while (len(members.get()) == 2):
time.sleep(0.2)
members = coordinator.get_members(group_name)
leave_group(group_name)
group_lock.release()
else:
leave_group(group_name)
break
else:
LOG.info("One member, waiting for second node to join ...")
time.sleep(1)
else:
leave_group(group_name)
error_msg = ("fio (network) timed out to find partner")
raise errors.CleaningError(error_msg)
return (partner, role)
[docs]
def fio_network(node):
"""Burn-in the network with fio
Run an fio network job for a pair of nodes for a configurable
amount of time. The pair is either statically defined in
driver_info via 'agent_burnin_fio_network_config' or the role
and partner is found dynamically via a tooz backend.
The writer will wait for the reader to connect, then write to the
network. Upon completion, the roles are swapped.
:param node: Ironic node object
:raises: CommandExecutionError if the execution of fio fails.
:raises: CleaningError if the configuration is incomplete.
"""
info = node.get('driver_info', {})
runtime = info.get('agent_burnin_fio_network_runtime', 21600)
outputfile = info.get('agent_burnin_fio_network_outputfile', None)
# get our role and identify our partner
config = info.get('agent_burnin_fio_network_config')
if config:
LOG.debug("static agent_burnin_fio_network_config is %s",
config)
role = config.get('role')
partner = config.get('partner')
else:
timeout = info.get(
'agent_burnin_fio_network_pairing_timeout', 900)
group_name = info.get(
'agent_burnin_fio_network_pairing_group_name',
'ironic.network-burnin')
backend_url = info.get(
'agent_burnin_fio_network_pairing_backend_url', None)
if not backend_url:
msg = ('fio (network): dynamic pairing config is missing '
'agent_burnin_fio_network_pairing_backend_url')
raise errors.CleaningError(msg)
LOG.info("dynamic pairing for network burn-in ...")
(partner, role) = _find_network_burnin_partner_and_role(
backend_url=backend_url,
group_name=group_name,
timeout=timeout)
if role not in NETWORK_BURNIN_ROLES:
error_msg = "fio (network) found an unknown role: %s" % role
raise errors.CleaningError(error_msg)
if not partner:
error_msg = "fio (network) failed to find partner"
raise errors.CleaningError(error_msg)
LOG.info("fio (network): partner %s, role is %s", partner, role)
logfilename = None
if outputfile:
logfilename = outputfile + '.' + role
_do_fio_network(role == 'writer', runtime, partner, logfilename)
LOG.debug("fio (network): first direction done, swapping roles ...")
if outputfile:
irole = "reader" if (role == "writer") else "writer"
logfilename = outputfile + '.' + irole
_do_fio_network(not role == 'writer', runtime, partner, logfilename)
