#!/usr/bin/env python3
"""GPUmodules - Classes to represent GPUs and sets of GPUs used in rickslab-gpu-utils.
Copyright (C) 2019 RicksLab
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
__author__ = 'RueiKe'
__copyright__ = 'Copyright (C) 2019 RicksLab'
__credits__ = ['Craig Echt - Testing, Debug, Verification, and Documentation',
'Keith Myers - Testing, Debug, Verification of NV Capability']
__license__ = 'GNU General Public License'
__program_name__ = 'gpu-utils'
__maintainer__ = 'RueiKe'
__docformat__ = 'reStructuredText'
# pylint: disable=multiple-statements
# pylint: disable=line-too-long
# pylint: disable=bad-continuation
import re
import subprocess
import shlex
import os
import sys
import logging
from typing import Union, List, Dict, TextIO, IO, Generator
from pathlib import Path
from uuid import uuid4
from enum import Enum
import glob
from numpy import nan as np_nan
from GPUmodules import __version__, __status__
try:
from GPUmodules import env
except ImportError:
import env
LOGGER = logging.getLogger('gpu-utils')
PATTERNS = env.GutConst.PATTERNS
class GpuEnum(Enum):
"""
Replace __str__ method of Enum so that name excludes type and can be used as key in other dicts.
"""
def __str__(self):
return self.name
class ObjDict(dict):
"""
Allow access of dictionary keys by key name.
"""
# pylint: disable=attribute-defined-outside-init
# pylint: disable=too-many-instance-attributes
def __getattr__(self, name):
if name in self:
return self[name]
raise AttributeError('No such attribute: {}'.format(name))
def __setattr__(self, name, value):
self[name] = value
def __delattr__(self, name):
if name in self:
del self[name]
else:
raise AttributeError('No such attribute: {}'.format(name))
class GpuItem:
"""An object to store GPU details.
"""
# pylint: disable=attribute-defined-outside-init
# pylint: disable=too-many-instance-attributes
_finalized = False
_button_labels = {'loading': 'Load%',
'power': 'Power',
'power_cap': 'PowerCap',
'temp_val': 'Temp',
'vddgfx_val': 'VddGfx',
'sclk_ps_val': 'SCLK Pstate',
'sclk_f_val': 'SCLK',
'mclk_ps_val': 'MCLK Pstate',
'mclk_f_val': 'MCLK'}
_fan_item_list = ['fan_enable', 'pwm_mode', 'fan_target',
'fan_speed', 'fan_pwm', 'fan_speed_range', 'fan_pwm_range']
short_list = ['vendor', 'readable', 'writable', 'compute', 'card_num', 'id', 'model_device_decode',
'gpu_type', 'card_path', 'sys_card_path', 'hwmon_path', 'pcie_id']
_GPU_NC_Param_List = ['compute', 'readable', 'writable', 'vendor', 'model', 'card_num', 'sys_card_path',
'gpu_type', 'card_path', 'hwmon_path', 'pcie_id', 'driver', 'id', 'model_device_decode']
# Vendor and Type skip lists for reporting
AMD_Skip_List = ['frequencies_max', 'compute_mode', 'serial_number', 'card_index']
NV_Skip_List = ['fan_enable', 'fan_speed', 'fan_pwm_range', 'fan_speed_range', 'pwm_mode',
'mem_gtt_total', 'mem_gtt_used', 'mem_gtt_usage',
'mclk_ps', 'mclk_f_range', 'sclk_f_range', 'vddc_range', 'power_dpm_force',
'temp_crits', 'voltages']
LEGACY_Skip_List = ['vbios', 'loading', 'mem_loading', 'sclk_ps', 'mclk_ps', 'ppm', 'power', 'power_cap',
'power_cap_range', 'mem_vram_total', 'mem_vram_used', 'mem_gtt_total', 'mem_gtt_used',
'mem_vram_usage', 'mem_gtt_usage', 'fan_speed_range', 'fan_enable', 'fan_target',
'fan_speed', 'voltages', 'vddc_range', 'frequencies', 'sclk_f_range', 'mclk_f_range']
# Define Class Labels
GPU_Type = GpuEnum('type', 'Undefined Unsupported Supported Legacy APU PStatesNE PStates CurvePts')
GPU_Comp = GpuEnum('Compatibility', 'None ALL ReadWrite ReadOnly WriteOnly Readable Writable')
GPU_Vendor = GpuEnum('vendor', 'Undefined ALL AMD NVIDIA INTEL ASPEED MATROX PCIE')
_apu_gpus = ['Carrizo', 'Picasso', 'Renoir']
# Table parameters labels.
table_parameters = ['model_display', 'loading', 'mem_loading', 'mem_vram_usage', 'mem_gtt_usage',
'power', 'power_cap', 'energy', 'temp_val', 'vddgfx_val',
'fan_pwm', 'sclk_f_val', 'sclk_ps_val', 'mclk_f_val', 'mclk_ps_val', 'ppm']
_table_param_labels = {'model_display': 'Model',
'loading': 'GPU Load %',
'mem_loading': 'Mem Load %',
'mem_vram_usage': 'VRAM Usage %',
'mem_gtt_usage': 'GTT Usage %',
'power': 'Power (W)',
'power_cap': 'Power Cap (W)',
'energy': 'Energy (kWh)',
'temp_val': 'T (C)',
'vddgfx_val': 'VddGFX (mV)',
'fan_pwm': 'Fan Spd (%)',
'sclk_f_val': 'Sclk (MHz)',
'sclk_ps_val': 'Sclk Pstate',
'mclk_f_val': 'Mclk (MHz)',
'mclk_ps_val': 'Mclk Pstate',
'ppm': 'Perf Mode'}
# Complete GPU print items, use skip lists where appropriate
_GPU_CLINFO_Labels = {'sep4': '#',
'opencl_version': ' Device OpenCL C Version',
'device_name': ' Device Name',
'device_version': ' Device Version',
'driver_version': ' Driver Version',
'max_cu': ' Max Compute Units',
'simd_per_cu': ' SIMD per CU',
'simd_width': ' SIMD Width',
'simd_ins_width': ' SIMD Instruction Width',
'max_mem_allocation': ' CL Max Memory Allocation',
'max_wi_dim': ' Max Work Item Dimensions',
'max_wi_sizes': ' Max Work Item Sizes',
'max_wg_size': ' Max Work Group Size',
'prf_wg_size': ' Preferred Work Group Size',
'prf_wg_multiple': ' Preferred Work Group Multiple'}
_GPU_Param_Labels = {'card_num': 'Card Number',
'vendor': 'Vendor',
'readable': 'Readable',
'writable': 'Writable',
'compute': 'Compute',
'unique_id': 'GPU UID',
'serial_number': 'GPU S/N',
'id': 'Device ID',
'model_device_decode': 'Decoded Device ID',
'model': 'Card Model',
'model_display': 'Display Card Model',
'card_index': 'Card Index',
'pcie_id': 'PCIe ID',
'link_spd': ' Link Speed',
'link_wth': ' Link Width',
'sep1': '#',
'driver': 'Driver',
'vbios': 'vBIOS Version',
'compute_platform': 'Compute Platform',
'compute_mode': 'Compute Mode',
'gpu_type': 'GPU Type',
'hwmon_path': 'HWmon',
'card_path': 'Card Path',
'sys_card_path': 'System Card Path',
'sep2': '#',
'power': 'Current Power (W)',
'power_cap': 'Power Cap (W)',
'power_cap_range': ' Power Cap Range (W)',
'fan_enable': 'Fan Enable',
'pwm_mode': 'Fan PWM Mode',
'fan_target': 'Fan Target Speed (rpm)',
'fan_speed': 'Current Fan Speed (rpm)',
'fan_pwm': 'Current Fan PWM (%)',
'fan_speed_range': ' Fan Speed Range (rpm)',
'fan_pwm_range': ' Fan PWM Range (%)',
'sep3': '#',
'loading': 'Current GPU Loading (%)',
'mem_loading': 'Current Memory Loading (%)',
'mem_gtt_usage': 'Current GTT Memory Usage (%)',
'mem_gtt_used': ' Current GTT Memory Used (GB)',
'mem_gtt_total': ' Total GTT Memory (GB)',
'mem_vram_usage': 'Current VRAM Usage (%)',
'mem_vram_used': ' Current VRAM Used (GB)',
'mem_vram_total': ' Total VRAM (GB)',
'temperatures': 'Current Temps (C)',
'temp_crits': 'Critical Temps (C)',
'voltages': 'Current Voltages (V)',
'vddc_range': ' Vddc Range',
'frequencies': 'Current Clk Frequencies (MHz)',
'frequencies_max': 'Maximum Clk Frequencies (MHz)',
'sclk_ps': 'Current SCLK P-State',
'sclk_f_range': ' SCLK Range',
'mclk_ps': 'Current MCLK P-State',
'mclk_f_range': ' MCLK Range',
'ppm': 'Power Profile Mode',
'power_dpm_force': 'Power DPM Force Performance Level'}
# GPU sensor reading details
SensorSet = Enum('set', 'None Test Static Dynamic Info State Monitor All')
sensor_sets = {SensorSet.Static: {'HWMON': ['power_cap_range', 'temp_crits',
'fan_speed_range', 'fan_pwm_range']},
SensorSet.Dynamic: {'HWMON': ['power', 'power_cap', 'temperatures', 'voltages',
'frequencies', 'fan_enable', 'fan_target',
'fan_speed', 'pwm_mode', 'fan_pwm']},
SensorSet.Info: {'DEVICE': ['unique_id', 'vbios', 'mem_vram_total', 'mem_gtt_total']},
SensorSet.State: {'DEVICE': ['loading', 'mem_loading', 'mem_gtt_used', 'mem_vram_used',
'link_spd', 'link_wth', 'sclk_ps', 'mclk_ps', 'ppm',
'power_dpm_force']},
SensorSet.Monitor: {'HWMON': ['power', 'power_cap', 'temperatures', 'voltages',
'frequencies', 'fan_pwm'],
'DEVICE': ['loading', 'mem_loading', 'mem_gtt_used', 'mem_vram_used',
'sclk_ps', 'mclk_ps', 'ppm']},
SensorSet.All: {'DEVICE': ['unique_id', 'vbios', 'loading', 'mem_loading',
'link_spd', 'link_wth', 'sclk_ps', 'mclk_ps', 'ppm',
'power_dpm_force', 'mem_vram_total', 'mem_gtt_total',
'mem_vram_used', 'mem_gtt_used'],
'HWMON': ['power_cap_range', 'temp_crits', 'power', 'power_cap',
'temperatures', 'voltages', 'frequencies',
'fan_speed_range', 'fan_pwm_range', 'fan_enable', 'fan_target',
'fan_speed', 'pwm_mode', 'fan_pwm']}}
SensorType = Enum('type', 'SingleParam SingleString SingleStringSelect MinMax MLSS InputLabel InputLabelX MLMS')
_gbcf = 1.0/(1024*1024*1024)
_sensor_details = {GPU_Vendor.AMD: {
'HWMON': {
'power': {'type': SensorType.SingleParam,
'cf': 0.000001, 'sensor': ['power1_average']},
'power_cap': {'type': SensorType.SingleParam,
'cf': 0.000001, 'sensor': ['power1_cap']},
'power_cap_range': {'type': SensorType.MinMax,
'cf': 0.000001, 'sensor': ['power1_cap_min', 'power1_cap_max']},
'fan_enable': {'type': SensorType.SingleParam,
'cf': 1, 'sensor': ['fan1_enable']},
'fan_target': {'type': SensorType.SingleParam,
'cf': 1, 'sensor': ['fan1_target']},
'fan_speed': {'type': SensorType.SingleParam,
'cf': 1, 'sensor': ['fan1_input']},
'fan_speed_range': {'type': SensorType.MinMax,
'cf': 1, 'sensor': ['fan1_min', 'fan1_max']},
'pwm_mode': {'type': SensorType.SingleParam,
'cf': 1, 'sensor': ['pwm1_enable']},
'fan_pwm': {'type': SensorType.SingleParam,
'cf': 0.39216, 'sensor': ['pwm1']},
'fan_pwm_range': {'type': SensorType.MinMax,
'cf': 0.39216, 'sensor': ['pwm1_min', 'pwm1_max']},
'temp_crits': {'type': SensorType.InputLabelX,
'cf': 0.001, 'sensor': ['temp*_crit']},
'frequencies': {'type': SensorType.InputLabelX,
'cf': 0.000001, 'sensor': ['freq*_input']},
'voltages': {'type': SensorType.InputLabelX,
'cf': 1, 'sensor': ['in*_input']},
'temperatures': {'type': SensorType.InputLabelX,
'cf': 0.001, 'sensor': ['temp*_input']},
'vddgfx': {'type': SensorType.InputLabelX,
'cf': 0.001, 'sensor': ['in*_input']}},
'DEVICE': {
'id': {'type': SensorType.MLMS,
'cf': None, 'sensor': ['vendor', 'device',
'subsystem_vendor', 'subsystem_device']},
'unique_id': {'type': SensorType.SingleString,
'cf': None, 'sensor': ['unique_id']},
'loading': {'type': SensorType.SingleParam,
'cf': 1, 'sensor': ['gpu_busy_percent']},
'mem_loading': {'type': SensorType.SingleParam,
'cf': 1, 'sensor': ['mem_busy_percent']},
'mem_vram_total': {'type': SensorType.SingleParam,
'cf': _gbcf, 'sensor': ['mem_info_vram_total']},
'mem_vram_used': {'type': SensorType.SingleParam,
'cf': _gbcf, 'sensor': ['mem_info_vram_used']},
'mem_gtt_total': {'type': SensorType.SingleParam,
'cf': _gbcf, 'sensor': ['mem_info_gtt_total']},
'mem_gtt_used': {'type': SensorType.SingleParam,
'cf': _gbcf, 'sensor': ['mem_info_gtt_used']},
'link_spd': {'type': SensorType.SingleString,
'cf': None, 'sensor': ['current_link_speed']},
'link_wth': {'type': SensorType.SingleString,
'cf': None, 'sensor': ['current_link_width']},
'sclk_ps': {'type': SensorType.MLSS,
'cf': None, 'sensor': ['pp_dpm_sclk']},
'mclk_ps': {'type': SensorType.MLSS,
'cf': None, 'sensor': ['pp_dpm_mclk']},
'power_dpm_force': {'type': SensorType.SingleString,
'cf': None,
'sensor': ['power_dpm_force_performance_level']},
'ppm': {'type': SensorType.SingleStringSelect,
'cf': None, 'sensor': ['pp_power_profile_mode']},
'vbios': {'type': SensorType.SingleString,
'cf': None, 'sensor': ['vbios_version']}}},
GPU_Vendor.PCIE: {
'DEVICE': {
'id': {'type': SensorType.MLMS,
'cf': None, 'sensor': ['vendor', 'device',
'subsystem_vendor',
'subsystem_device']}}}}
nv_query_items = {SensorSet.Static: {
'power_cap': ['power.limit'],
'power_cap_range': ['power.min_limit', 'power.max_limit'],
'mem_vram_total': ['memory.total'],
'frequencies_max': ['clocks.max.gr', 'clocks.max.sm', 'clocks.max.mem'],
'vbios': ['vbios_version'],
'compute_mode': ['compute_mode'],
'driver': ['driver_version'],
'model': ['name'],
'serial_number': ['serial'],
'card_index': ['index'],
'unique_id': ['gpu_uuid']},
SensorSet.Dynamic: {
'power': ['power.draw'],
'temperatures': ['temperature.gpu', 'temperature.memory'],
'frequencies': ['clocks.gr', 'clocks.sm', 'clocks.mem', 'clocks.video'],
'loading': ['utilization.gpu'],
'mem_loading': ['utilization.memory'],
'mem_vram_used': ['memory.used'],
'fan_speed': ['fan.speed'],
'ppm': ['gom.current'],
'link_wth': ['pcie.link.width.current'],
'link_spd': ['pcie.link.gen.current'],
'pstates': ['pstate']},
SensorSet.Monitor: {
'power': ['power.draw'],
'power_cap': ['power.limit'],
'temperatures': ['temperature.gpu'],
'frequencies': ['clocks.gr', 'clocks.mem'],
'loading': ['utilization.gpu'],
'mem_loading': ['utilization.memory'],
'mem_vram_used': ['memory.used'],
'fan_speed': ['fan.speed'],
'ppm': ['gom.current'],
'pstates': ['pstate']},
SensorSet.All: {
'power_cap': ['power.limit'],
'power_cap_range': ['power.min_limit', 'power.max_limit'],
'mem_vram_total': ['memory.total'],
'vbios': ['vbios_version'],
'driver': ['driver_version'],
'compute_mode': ['compute_mode'],
'model': ['name'],
'serial_number': ['serial'],
'card_index': ['index'],
'unique_id': ['gpu_uuid'],
'power': ['power.draw'],
'temperatures': ['temperature.gpu', 'temperature.memory'],
'frequencies': ['clocks.gr', 'clocks.sm', 'clocks.mem', 'clocks.video'],
'frequencies_max': ['clocks.max.gr', 'clocks.max.sm', 'clocks.max.mem'],
'loading': ['utilization.gpu'],
'mem_loading': ['utilization.memory'],
'mem_vram_used': ['memory.used'],
'fan_speed': ['fan.speed'],
'ppm': ['gom.current'],
'link_wth': ['pcie.link.width.current'],
'link_spd': ['pcie.link.gen.current'],
'pstates': ['pstate']}}
def __repr__(self) -> Dict[str, any]:
"""
Return dictionary representing all parts of the GpuItem object.
:return: Dictionary of core GPU parameters.
"""
return {'params': self.prm, 'clinfo': self.clinfo,
'sclk_state': self.sclk_state, 'mclk_state': self.mclk_state,
'vddc_curve': self.vddc_curve, 'vddc_curve_range': self.vddc_curve_range,
'ppm_modes': self.ppm_modes}
def __str__(self) -> str:
"""
Return simple string representing the GpuItem object.
:return: GPU_item informational string
"""
return 'GPU_Item: uuid={}'.format(self.prm.uuid)
def __init__(self, item_id: str):
"""
Initialize GpuItem object.
:param item_id: UUID of the new item.
"""
time_0 = env.GUT_CONST.now(env.GUT_CONST.USELTZ)
self.validated_sensors = False
self.energy = {'t0': time_0, 'tn': time_0, 'cumulative': 0.0}
self.read_disabled = [] # List of parameters that failed during read.
self.write_disabled = [] # List of parameters that failed during write.
self.prm = ObjDict({'uuid': item_id,
'unique_id': '',
'card_num': None,
'pcie_id': '',
'driver': '',
'vendor': self.GPU_Vendor.Undefined,
'readable': False,
'writable': False,
'compute': False,
'compute_platform': None,
'compute_mode': None,
'gpu_type': self.GPU_Type.Undefined,
'id': {'vendor': '', 'device': '', 'subsystem_vendor': '', 'subsystem_device': ''},
'model_device_decode': 'UNDETERMINED',
'model': '',
'model_display': '',
'serial_number': '',
'card_index': '',
'card_path': '',
'sys_card_path': '',
'hwmon_path': '',
'energy': 0.0,
'power': None,
'power_cap': None,
'power_cap_range': [None, None],
'fan_enable': None,
'pwm_mode': [None, 'UNK'],
'fan_pwm': None,
'fan_speed': None,
'fan_speed_range': [None, None],
'fan_pwm_range': [None, None],
'fan_target': None,
'temp_crits': None,
'vddgfx': None,
'vddc_range': ['', ''],
'temperatures': None,
'voltages': None,
'frequencies': None,
'frequencies_max': None,
'loading': None,
'mem_loading': None,
'mem_vram_total': None,
'mem_vram_used': None,
'mem_vram_usage': None,
'mem_gtt_total': None,
'mem_gtt_used': None,
'mem_gtt_usage': None,
'pstate': None,
'mclk_ps': ['', ''],
'mclk_f_range': ['', ''],
'mclk_mask': '',
'sclk_ps': ['', ''],
'sclk_f_range': ['', ''],
'sclk_mask': '',
'link_spd': '',
'link_wth': '',
'ppm': '',
'power_dpm_force': '',
'vbios': ''})
self.clinfo = ObjDict({'device_name': '',
'device_version': '',
'driver_version': '',
'opencl_version': '',
'pcie_id': '',
'max_cu': '',
'simd_per_cu': '',
'simd_width': '',
'simd_ins_width': '',
'max_mem_allocation': '',
'max_wi_dim': '',
'max_wi_sizes': '',
'max_wg_size': '',
'prf_wg_size': '',
'prf_wg_multiple': ''})
self.sclk_dpm_state = {} # {'1': 'Mhz'}
self.mclk_dpm_state = {} # {'1': 'Mhz'}
self.sclk_state = {} # {'1': ['Mhz', 'mV']}
self.mclk_state = {} # {'1': ['Mhz', 'mV']}
self.vddc_curve = {} # {'1': ['Mhz', 'mV']}
self.vddc_curve_range = {} # {'1': {SCLK: ['val1', 'val2'], VOLT: ['val1', 'val2']}
self.ppm_modes = {} # {'1': ['Name', 'Description']}
self.finalize_fan_option()
@classmethod
def finalize_fan_option(cls) -> None:
"""
Finalize class variables of gpu parameters based on command line options. This must be
done after setting of env. Doing at at the instantiation of a GpuItem assures that.
"""
if cls._finalized:
return
cls.finalized = True
if not env.GUT_CONST.show_fans:
for fan_item in cls._fan_item_list:
# Remove fan params from GPU_Param_Labels
if fan_item in cls._GPU_Param_Labels.keys():
del cls._GPU_Param_Labels[fan_item]
# Remove fan params from Table_Param_Labels
if fan_item in cls._table_param_labels.keys():
del cls._table_param_labels[fan_item]
# Remove fan params from SensorSets
if fan_item in cls.sensor_sets[cls.SensorSet.Static]['HWMON']:
cls.sensor_sets[cls.SensorSet.Static]['HWMON'].remove(fan_item)
if fan_item in cls.sensor_sets[cls.SensorSet.Dynamic]['HWMON']:
cls.sensor_sets[cls.SensorSet.Dynamic]['HWMON'].remove(fan_item)
if fan_item in cls.sensor_sets[cls.SensorSet.Monitor]['HWMON']:
cls.sensor_sets[cls.SensorSet.Monitor]['HWMON'].remove(fan_item)
if fan_item in cls.sensor_sets[cls.SensorSet.All]['HWMON']:
cls.sensor_sets[cls.SensorSet.All]['HWMON'].remove(fan_item)
# Remove fan params from table param list
if fan_item in cls.table_parameters:
cls.table_parameters.remove(fan_item)
@classmethod
def is_apu(cls, name: str) -> bool:
"""
Check if given GPU name is an APU.
:param name: Target GPU name
:return: True if name matches APU name
"""
for apu_name in cls._apu_gpus:
if name in apu_name:
return True
return False
@classmethod
def get_button_label(cls, name: str) -> str:
"""
Return button label for given parameter name.
:param name: Parameter name
:return: Button label
"""
if name not in cls._button_labels.keys():
raise KeyError('{} not in button_label dict'.format(name))
return cls._button_labels[name]
def set_params_value(self, name: str, value: Union[int, float, str, list, None]) -> None:
"""
Set parameter value for give name.
:param name: Parameter name
:param value: parameter value
"""
if isinstance(value, tuple):
self.prm[name] = list(value)
elif name == 'pwm_mode':
self.prm[name][0] = value
if value == 0: self.prm[name][1] = 'None'
elif value == 1: self.prm[name][1] = 'Manual'
else: self.prm[name][1] = 'Dynamic'
elif name == 'ppm':
self.prm[name] = re.sub(PATTERNS['PPM_CHK'], '', value).strip()
self.prm[name] = re.sub(PATTERNS['PPM_NOTCHK'], '-', self.prm[name])
elif name == 'power':
time_n = env.GUT_CONST.now(env.GUT_CONST.USELTZ)
self.prm[name] = value
delta_hrs = ((time_n - self.energy['tn']).total_seconds()) / 3600
self.energy['tn'] = time_n
self.energy['cumulative'] += delta_hrs * value / 1000
self.prm['energy'] = round(self.energy['cumulative'], 6)
elif name == 'sclk_ps':
mask = ''
for ps_val in value:
if not mask:
mask = ps_val.split(':')[0].strip()
else:
mask += ',' + ps_val.split(':')[0].strip()
sclk_ps = ps_val.strip('*').strip().split(': ')
self.sclk_dpm_state.update({int(sclk_ps[0]): sclk_ps[1]})
if '*' in ps_val:
self.prm.sclk_ps[0] = int(sclk_ps[0])
self.prm.sclk_ps[1] = sclk_ps[1]
self.prm.sclk_mask = mask
LOGGER.debug('Mask: [%s], ps: [%s, %s]', mask, self.prm.sclk_ps[0], self.prm.sclk_ps[1])
elif name == 'mclk_ps':
mask = ''
for ps_val in value:
if not mask:
mask = ps_val.split(':')[0].strip()
else:
mask += ',' + ps_val.split(':')[0].strip()
mclk_ps = ps_val.strip('*').strip().split(': ')
self.mclk_dpm_state.update({int(mclk_ps[0]): mclk_ps[1]})
if '*' in ps_val:
self.prm.mclk_ps[0] = int(mclk_ps[0])
self.prm.mclk_ps[1] = mclk_ps[1]
self.prm.mclk_mask = mask
LOGGER.debug('Mask: [%s], ps: [%s, %s]', mask, self.prm.mclk_ps[0], self.prm.mclk_ps[1])
elif name == 'fan_pwm':
if isinstance(value, int):
self.prm.fan_pwm = value
elif isinstance(value, float):
self.prm.fan_pwm = int(value)
elif isinstance(value, str):
self.prm.fan_pwm = int(value) if value.isnumeric() else None
else:
self.prm.fan_pwm = None
elif re.fullmatch(PATTERNS['GPUMEMTYPE'], name):
self.prm[name] = value
self.set_memory_usage()
elif name == 'id':
self.prm.id = dict(zip(['vendor', 'device', 'subsystem_vendor', 'subsystem_device'], list(value)))
self.prm.model_display = self.prm.model_device_decode = self.read_pciid_model()
else:
self.prm[name] = value
def get_params_value(self, name: str, num_as_int: bool = False) -> Union[int, float, str, list, None]:
"""
Get parameter value for give name.
:param name: Parameter name
:param num_as_int: Convert float to in if True
:return: Parameter value
"""
# Parameters with '_val' as a suffix are derived from a direct source.
if re.fullmatch(PATTERNS['VAL_ITEM'], name):
if self.prm.gpu_type == self.GPU_Type.Legacy:
return None
if name == 'temp_val':
if not self.prm['temperatures']:
return None
if 'edge' in self.prm['temperatures'].keys():
if num_as_int:
return int(self.prm['temperatures']['edge'])
return round(self.prm['temperatures']['edge'], 1)
if 'temperature.gpu' in self.prm['temperatures'].keys():
if num_as_int:
return int(self.prm['temperatures']['temperature.gpu'])
return round(self.prm['temperatures']['temperature.gpu'], 1)
if self.prm['temperatures'].keys():
return list(self.prm['temperatures'].keys())[0]
return None
if name == 'vddgfx_val':
if not self.prm['voltages']:
return np_nan
if 'vddgfx' not in self.prm['voltages']:
return np_nan
return int(self.prm['voltages']['vddgfx'])
if name == 'sclk_ps_val':
return self.prm['sclk_ps'][0]
if name == 'sclk_f_val':
if not self.prm['frequencies']:
return None
if 'sclk' in self.prm['frequencies'].keys():
return int(self.prm['frequencies']['sclk'])
if 'clocks.gr' in self.prm['frequencies'].keys():
return int(self.prm['frequencies']['clocks.gr'])
return self.prm['sclk_ps'][1]
if name == 'mclk_ps_val':
return self.prm['mclk_ps'][0]
if name == 'mclk_f_val':
if not self.prm['frequencies']:
return None
if 'mclk' in self.prm['frequencies'].keys():
return int(self.prm['frequencies']['mclk'])
if 'clocks.mem' in self.prm['frequencies'].keys():
return int(self.prm['frequencies']['clocks.mem'])
return self.prm['mclk_ps'][1]
# Set type for params that could be float or int
if name in ['fan_pwm', 'fan_speed', 'power_cap', 'power']:
if num_as_int:
if isinstance(self.prm[name], int):
return self.prm[name]
if isinstance(self.prm[name], float):
return int(self.prm[name])
if isinstance(self.prm[name], str):
return int(self.prm[name]) if self.prm[name].isnumeric() else None
return self.prm[name]
return self.prm[name]
def set_memory_usage(self) -> None:
"""
Set system and vram memory usage percentage.
:return: A tuple of the system and vram memory usage percentage.
"""
if self.prm.mem_gtt_used is None or self.prm.mem_gtt_total is None:
self.prm.mem_gtt_usage = None
else:
self.prm.mem_gtt_usage = 100.0 * self.prm.mem_gtt_used / self.prm.mem_gtt_total
if self.prm.mem_vram_used is None or self.prm.mem_vram_total is None:
self.prm.mem_vram_usage = None
else:
self.prm.mem_vram_usage = 100.0 * self.prm.mem_vram_used / self.prm.mem_vram_total
def read_pciid_model(self) -> str:
"""
Read the model name from the system pcid.ids file
:return: GPU model name
"""
LOGGER.debug('Logger active in module')
if not env.GUT_CONST.sys_pciid:
print('Error: Can not access system pci.ids file [{}]'.format(env.GUT_CONST.sys_pciid))
return ''
if not os.path.isfile(env.GUT_CONST.sys_pciid):
print('Error: Can not access system pci.ids file [{}]'.format(env.GUT_CONST.sys_pciid))
return ''
with open(env.GUT_CONST.sys_pciid, 'r', encoding='utf8') as pci_id_file_ptr:
model_str = ''
level = 0
for line_item in pci_id_file_ptr:
line = line_item.rstrip()
if len(line) < 4:
continue
if line[0] == '#':
continue
if level == 0:
if re.fullmatch(PATTERNS['PCIIID_L0'], line):
if line[:4] == self.prm.id['vendor'].replace('0x', ''):
level += 1
continue
elif level == 1:
if re.fullmatch(PATTERNS['PCIIID_L0'], line):
break
if re.fullmatch(PATTERNS['PCIIID_L1'], line):
if line[1:5] == self.prm.id['device'].replace('0x', ''):
model_str = line[5:]
level += 1
continue
elif level == 2:
if re.fullmatch(PATTERNS['PCIIID_L0'], line):
break
if re.fullmatch(PATTERNS['PCIIID_L1'], line):
break
if re.fullmatch(PATTERNS['PCIIID_L2'], line):
if line[2:6] == self.prm.id['subsystem_vendor'].replace('0x', ''):
if line[7:11] == self.prm.id['subsystem_device'].replace('0x', ''):
model_str = line[11:]
break
return model_str.strip()
def populate_prm_from_dict(self, params: Dict[str, any]) -> None:
"""
Populate elements of a GpuItem with items from a dict with keys that align to elements of GpuItem.
:param params: A dictionary of parameters with keys that align to GpuItem elements.
"""
LOGGER.debug('prm dict:\n%s', params)
set_ocl_ver = None
for source_name, source_value in params.items():
# Set primary parameter
if source_name not in self.prm.keys():
raise KeyError('Populate dict contains unmatched key: {}'.format(source_name))
self.prm[source_name] = source_value
# Set secondary parameters
if source_name == 'card_path' and source_value:
card_num_str = source_value.replace('{}card'.format(env.GUT_CONST.card_root), '').replace('/device', '')
self.prm.card_num = int(card_num_str) if card_num_str.isnumeric() else None
elif source_name == 'compute_platform':
set_ocl_ver = source_value
elif source_name == 'gpu_type' and source_value:
self.prm.gpu_type = source_value
if source_value == GpuItem.GPU_Type.Legacy:
self.read_disabled = GpuItem.LEGACY_Skip_List[:]
elif source_value == GpuItem.GPU_Type.APU:
self.read_disabled = GpuItem._fan_item_list[:]
# compute platform requires that the compute bool be set first
if set_ocl_ver:
self.prm.compute_platform = set_ocl_ver if self.prm.compute else 'None'
def populate_ocl(self, ocl_dict: dict) -> None:
"""
Populate ocl parameters in GpuItem.
:param ocl_dict: Dictionary of parameters for specific pcie_id
"""
for ocl_name, ocl_val in ocl_dict.items():
if ocl_name in self.clinfo.keys():
self.set_clinfo_value(ocl_name, ocl_val)
def set_clinfo_value(self, name: str, value: Union[int, str, list]) -> None:
"""
Set clinfo values in GPU item dictionary.
:param name: clinfo parameter name
:param value: parameter value
"""
self.clinfo[name] = value
def get_clinfo_value(self, name: str) -> Union[int, str, list]:
"""
Get clinfo parameter value for give name.
:param name: clinfo Parameter name
:return: clinfo Parameter value
.. note: Maybe not needed
"""
return self.clinfo[name]
def get_nc_params_list(self) -> List[str]:
"""
Get list of parameter names for use with non-compatible cards.
:return: List of parameter names
"""
return self._GPU_NC_Param_List
def is_valid_power_cap(self, power_cap: int) -> bool:
"""
Check if a given power_cap value is valid.
:param power_cap: Target power cap value to be tested.
:return: True if valid
"""
power_cap_range = self.prm.power_cap_range
if power_cap_range[0] <= power_cap <= power_cap_range[1]:
return True
if power_cap < 0:
# negative values will be interpreted as reset request
return True
return False
def is_valid_fan_pwm(self, pwm_value: int) -> bool:
"""
Check if a given fan_pwm value is valid.
:param pwm_value: Target fan_pwm value to be tested.
:return: True if valid
"""
pwm_range = self.prm.fan_pwm_range
if pwm_range[0] <= pwm_value <= pwm_range[1]:
return True
if pwm_value < 0:
# negative values will be interpreted as reset request
return True
return False
def is_valid_mclk_pstate(self, pstate: List[int]) -> bool:
"""
Check if given mclk pstate value is valid.
:param pstate: pstate = [pstate_number, clk_value, vddc_value]
:return: Return True if valid
"""
mclk_range = self.prm.mclk_f_range
mclk_min = int(re.sub(PATTERNS['END_IN_ALPHA'], '', str(mclk_range[0])))
mclk_max = int(re.sub(PATTERNS['END_IN_ALPHA'], '', str(mclk_range[1])))
if pstate[1] < mclk_min or pstate[1] > mclk_max:
return False
if self.prm.gpu_type in [self.GPU_Type.PStatesNE, self.GPU_Type.PStates]:
vddc_range = self.prm.vddc_range
vddc_min = int(re.sub(PATTERNS['END_IN_ALPHA'], '', str(vddc_range[0])))
vddc_max = int(re.sub(PATTERNS['END_IN_ALPHA'], '', str(vddc_range[1])))
if pstate[2] < vddc_min or pstate[2] > vddc_max:
return False
return True
def is_valid_sclk_pstate(self, pstate: List[int]) -> bool:
"""
Check if given sclk pstate value is valid.
:param pstate: pstate = [pstate_number, clk_value, vddc_value]
:return: Return True if valid
"""
sclk_range = self.prm.sclk_f_range
sclk_min = int(re.sub(PATTERNS['END_IN_ALPHA'], '', str(sclk_range[0])))
sclk_max = int(re.sub(PATTERNS['END_IN_ALPHA'], '', str(sclk_range[1])))
if pstate[1] < sclk_min or pstate[1] > sclk_max:
return False
if self.prm.gpu_type in [self.GPU_Type.PStatesNE, self.GPU_Type.PStates]:
vddc_range = self.prm.vddc_range
vddc_min = int(re.sub(PATTERNS['END_IN_ALPHA'], '', str(vddc_range[0])))
vddc_max = int(re.sub(PATTERNS['END_IN_ALPHA'], '', str(vddc_range[1])))
if pstate[2] < vddc_min or pstate[2] > vddc_max:
return False
return True
def is_changed_sclk_pstate(self, pstate: List[int]) -> bool:
"""
Check if given sclk pstate value different from current.
:param pstate: pstate = [pstate_number, clk_value, vddc_value]
:return: Return True if changed
"""
if int(re.sub(PATTERNS['END_IN_ALPHA'], '', self.sclk_state[pstate[0]][0])) != pstate[1]:
return True
if self.prm.gpu_type in [self.GPU_Type.PStatesNE, self.GPU_Type.PStates]:
if int(re.sub(PATTERNS['END_IN_ALPHA'], '', self.sclk_state[pstate[0]][1])) != pstate[2]:
return True
return False
def is_changed_mclk_pstate(self, pstate: List[int]) -> bool:
"""
Check if given mclk pstate value different from current.
:param pstate: pstate = [pstate_number, clk_value, vddc_value]
:return: Return True if changed
"""
if int(re.sub(PATTERNS['END_IN_ALPHA'], '', self.mclk_state[pstate[0]][0])) != pstate[1]:
return True
if self.prm.gpu_type in [self.GPU_Type.PStatesNE, self.GPU_Type.PStates]:
if int(re.sub(PATTERNS['END_IN_ALPHA'], '', self.mclk_state[pstate[0]][1])) != pstate[2]:
return True
return False
def is_changed_vddc_curve_pt(self, pstate: List[int]) -> bool:
"""
Check if given vddc curve point value different from current.
:param pstate: curve_point = [point_number, clk_value, vddc_value]
:return: Return True if changed
"""
if int(re.sub(PATTERNS['END_IN_ALPHA'], '', self.vddc_curve[pstate[0]][0])) != pstate[1]:
return True
if int(re.sub(PATTERNS['END_IN_ALPHA'], '', self.vddc_curve[pstate[0]][1])) != pstate[2]:
return True
return False
def is_valid_vddc_curve_pts(self, curve_pts: List[int]) -> bool:
"""
Check if given sclk pstate value is valid.
:param curve_pts: curve_point = [point_number, clk_value, vddc_value]
:return: Return True if valid
"""
sclk_min = int(re.sub(PATTERNS['END_IN_ALPHA'], '', str(self.vddc_curve_range[str(curve_pts[0])]['SCLK'][0])))
sclk_max = int(re.sub(PATTERNS['END_IN_ALPHA'], '', str(self.vddc_curve_range[str(curve_pts[0])]['SCLK'][1])))
if curve_pts[1] < sclk_min or curve_pts[1] > sclk_max:
return False
vddc_min = int(re.sub(PATTERNS['END_IN_ALPHA'], '', str('650mV')))
vddc_max = int(re.sub(PATTERNS['END_IN_ALPHA'], '', str(self.vddc_curve_range[str(curve_pts[0])]['VOLT'][1])))
if curve_pts[2] < vddc_min or curve_pts[2] > vddc_max:
return False
return True
def is_valid_pstate_list_str(self, ps_str: str, clk_name: str) -> bool:
"""
Check if the given p-states are valid for the given clock.
:param ps_str: String of comma separated pstate numbers
:param clk_name: The target clock name
:return: True if valid
"""
if ps_str == '':
return True
if not re.fullmatch(PATTERNS['VALID_PS_STR'], ps_str):
return False
ps_list = self.prm.mclk_mask.split(',') if clk_name == 'MCLK' else self.prm.sclk_mask.split(',')
for ps_val in ps_str.split():
if ps_val not in ps_list:
return False
return True
def get_current_ppm_mode(self) -> Union[None, List[Union[int, str]]]:
"""
Read GPU ppm definitions and current settings from driver files.
:return: ppm state
:rtype: list
"""
if self.prm.vendor != GpuItem.GPU_Vendor.AMD:
return None
if self.prm.power_dpm_force.lower() == 'auto':
return [-1, 'AUTO']
ppm_item = self.prm.ppm.split('-')
return [int(ppm_item[0]), ppm_item[1]]
def read_gpu_ppm_table(self) -> None:
"""
Read the ppm table.
"""
if self.prm.vendor != GpuItem.GPU_Vendor.AMD:
return
if not self.prm.readable or self.prm.gpu_type in [GpuItem.GPU_Type.Legacy, GpuItem.GPU_Type.Unsupported]:
return
file_path = os.path.join(self.prm.card_path, 'pp_power_profile_mode')
if not os.path.isfile(file_path):
print('Error getting power profile modes: {}'.format(file_path), file=sys.stderr)
sys.exit(-1)
with open(file_path) as card_file:
for line in card_file:
linestr = line.strip()
# Check for mode name: begins with '[ ]+[0-9].*'
if re.fullmatch(r'[ ]+[0-9].*', line[0:3]):
linestr = re.sub(r'[ ]*[*]*:', ' ', linestr)
line_items = linestr.split()
LOGGER.debug('PPM line: %s', linestr)
if len(line_items) < 2:
print('Error: invalid ppm: {}'.format(linestr), file=sys.stderr)
continue
LOGGER.debug('Valid ppm line: %s', linestr)
self.ppm_modes[line_items[0]] = line_items[1:]
self.ppm_modes['-1'] = ['AUTO', 'Auto']
rdata = self.read_gpu_sensor('power_dpm_force', vendor=GpuItem.GPU_Vendor.AMD, sensor_type='DEVICE')
if rdata is False:
print('Error: card file does not exist: {}'.format(file_path), file=sys.stderr)
LOGGER.debug('Card file does not exist: %s', file_path)
self.prm.readable = False
else:
self.set_params_value('power_dpm_force', rdata)
def read_gpu_pstates(self) -> None:
"""
Read GPU pstate definitions and parameter ranges from driver files.
Set card type based on pstate configuration
"""
if self.prm.vendor != GpuItem.GPU_Vendor.AMD:
return
if not self.prm.readable or self.prm.gpu_type in [GpuItem.GPU_Type.Legacy,
GpuItem.GPU_Type.Unsupported,
GpuItem.GPU_Type.APU]:
return
range_mode = False
file_path = os.path.join(self.prm.card_path, 'pp_od_clk_voltage')
if not os.path.isfile(file_path):
print('Error getting p-states: {}'.format(file_path), file=sys.stderr)
self.prm.readable = False
return
with open(file_path) as card_file:
for line in card_file:
line = line.strip()
if re.fullmatch('OD_.*:$', line):
if re.fullmatch('OD_.CLK:$', line):
clk_name = line.strip()
elif re.fullmatch('OD_VDDC_CURVE:$', line):
clk_name = line.strip()
elif re.fullmatch('OD_RANGE:$', line):
clk_name = ''
range_mode = True
continue
line = re.sub(r'@', ' ', line)
lineitems: List[any] = line.split()
lineitems_len = len(lineitems)
if self.prm.gpu_type in [self.GPU_Type.Undefined, self.GPU_Type.Supported]:
if len(lineitems) == 3:
self.prm.gpu_type = self.GPU_Type.PStates
elif len(lineitems) == 2:
self.prm.gpu_type = self.GPU_Type.CurvePts
else:
print('Error: Invalid pstate entry length {} for{}: '.format(lineitems_len,
os.path.join(self.prm.card_path, 'pp_od_clk_voltage')), file=sys.stderr)
LOGGER.debug('Invalid line length for pstate line item: %s', line)
continue
if not range_mode:
lineitems[0] = int(re.sub(':', '', lineitems[0]))
if self.prm.gpu_type in [self.GPU_Type.PStatesNE, self.GPU_Type.PStates]:
if clk_name == 'OD_SCLK:':
self.sclk_state[lineitems[0]] = [lineitems[1], lineitems[2]]
elif clk_name == 'OD_MCLK:':
self.mclk_state[lineitems[0]] = [lineitems[1], lineitems[2]]
else:
# Type GPU_Type.CurvePts
if clk_name == 'OD_SCLK:':
self.sclk_state[lineitems[0]] = [lineitems[1], '-']
elif clk_name == 'OD_MCLK:':
self.mclk_state[lineitems[0]] = [lineitems[1], '-']
elif clk_name == 'OD_VDDC_CURVE:':
self.vddc_curve[lineitems[0]] = [lineitems[1], lineitems[2]]
else:
if lineitems[0] == 'SCLK:':
self.prm.sclk_f_range = [lineitems[1], lineitems[2]]
elif lineitems[0] == 'MCLK:':
self.prm.mclk_f_range = [lineitems[1], lineitems[2]]
elif lineitems[0] == 'VDDC:':
self.prm.vddc_range = [lineitems[1], lineitems[2]]
elif re.fullmatch('VDDC_CURVE_.*', line):
if len(lineitems) == 3:
index = re.sub(r'VDDC_CURVE_.*\[', '', lineitems[0])
index = re.sub(r'\].*', '', index)
param = re.sub(r'VDDC_CURVE_', '', lineitems[0])
param = re.sub(r'\[[0-9]\]:', '', param)
LOGGER.debug('Curve: index: %s param: %s, val1 %s, val2: %s',
index, param, lineitems[1], lineitems[2])
if index in self.vddc_curve_range.keys():
self.vddc_curve_range[index].update({param: [lineitems[1], lineitems[2]]})
else:
self.vddc_curve_range[index] = {}
self.vddc_curve_range[index].update({param: [lineitems[1], lineitems[2]]})
else:
print('Error: Invalid CURVE entry: {}'.format(file_path), file=sys.stderr)
def read_gpu_sensor(self, parameter: str, vendor: GpuEnum = GPU_Vendor.AMD,
sensor_type: str = 'HWMON') -> Union[None, bool, int, str, tuple, list, dict]:
"""
Read sensor for the given parameter name. Process per sensor_details dict using the specified
vendor name and sensor_type.
:param parameter: GpuItem parameter name (AMD)
:param vendor: GPU vendor name enum object
:param sensor_type: GPU sensor name (HWMON or DEVICE)
:return: Value from reading sensor.
"""
if vendor in [self.GPU_Vendor.AMD, self.GPU_Vendor.PCIE]:
return self.read_gpu_sensor_generic(parameter, vendor, sensor_type)
if vendor == self.GPU_Vendor.NVIDIA:
return self.read_gpu_sensor_nv(parameter)
print('Error: Invalid vendor [{}]'.format(vendor))
return None
def read_gpu_sensor_nv(self, parameter: str) -> Union[None, bool, int, str, tuple, list, dict]:
"""
Function to read a single sensor from NV GPU.
:param parameter: Target parameter for reading
:return: read results
"""
if parameter in self.read_disabled:
return False
cmd_str = '{} -i {} --query-gpu={} --format=csv,noheader,nounits'.format(
env.GUT_CONST.cmd_nvidia_smi, self.prm.pcie_id, parameter)
LOGGER.debug('NV command:\n%s', cmd_str)
nsmi_item = None
try:
nsmi_item = subprocess.check_output(shlex.split(cmd_str), shell=False).decode().split('\n')
LOGGER.debug('NV raw query response: [%s]', nsmi_item)
except (subprocess.CalledProcessError, OSError) as except_err:
LOGGER.debug('NV query %s error: [%s]', nsmi_item, except_err)
self.read_disabled.append(parameter)
return False
return_item = nsmi_item[0].strip() if nsmi_item else None
LOGGER.debug('NV query result: [%s]', return_item)
return return_item
def read_gpu_sensor_generic(self, parameter: str, vendor: GpuEnum = GPU_Vendor.AMD,
sensor_type: str = 'HWMON') -> Union[None, bool, int, str, tuple, list, dict]:
"""
Read sensor for the given parameter name. Process per sensor_details dict using the specified
vendor name and sensor_type.
:param parameter: GpuItem parameter name (AMD)
:param vendor: GPU vendor name enum object
:param sensor_type: GPU sensor name (HWMON or DEVICE)
:return: Value from reading sensor.
"""
if self.prm.gpu_type in [GpuItem.GPU_Type.Unsupported] and parameter != 'id':
return None
if not self.prm.readable and parameter != 'id':
return None
if sensor_type not in self._sensor_details[vendor].keys():
print('Error: Invalid sensor_type [{}]'.format(sensor_type))
return None
sensor_dict = self._sensor_details[vendor][sensor_type]
if parameter not in sensor_dict.keys():
print('Error: Invalid parameter [{}]'.format(parameter))
return None
if parameter in self.read_disabled:
return None
device_sensor_path = self.prm.card_path if self.prm.card_path else self.prm.sys_card_path
LOGGER.debug('sensor path set to [%s]', device_sensor_path)
sensor_path = self.prm.hwmon_path if sensor_type == 'HWMON' else device_sensor_path
values = []
ret_value = []
ret_dict = {}
target_sensor = sensor_dict[parameter]
if target_sensor['type'] == self.SensorType.InputLabelX:
sensor_files = glob.glob(os.path.join(sensor_path, target_sensor['sensor'][0]))
else:
sensor_files = target_sensor['sensor']
for sensor_file in sensor_files:
file_path = os.path.join(sensor_path, sensor_file)
if os.path.isfile(file_path):
try:
with open(file_path) as hwmon_file:
if target_sensor['type'] in [self.SensorType.SingleStringSelect, self.SensorType.MLSS]:
lines = hwmon_file.readlines()
for line in lines:
values.append(line.strip())
else:
values.append(hwmon_file.readline().strip())
if target_sensor['type'] == self.SensorType.InputLabelX:
if '_input' in file_path:
file_path = file_path.replace('_input', '_label')
elif '_crit' in file_path:
file_path = file_path.replace('_crit', '_label')
else:
print('Error in sensor label pair: {}'.format(target_sensor))
if os.path.isfile(file_path):
with open(file_path) as hwmon_file:
values.append(hwmon_file.readline().strip())
else:
values.append(os.path.basename(sensor_file))
except OSError as err:
LOGGER.debug('Exception [%s]: Can not read HW file: %s', err, file_path)
self.read_disabled.append(parameter)
return False
else:
LOGGER.debug('HW file does not exist: %s', file_path)
self.read_disabled.append(parameter)
return False
if target_sensor['type'] == self.SensorType.SingleParam:
if target_sensor['cf'] == 1:
return int(values[0])
return int(values[0]) * target_sensor['cf']
if target_sensor['type'] == self.SensorType.InputLabel:
ret_value.append(int(values[0]) * target_sensor['cf'])
ret_value.append(values[1])
return tuple(ret_value)
if target_sensor['type'] in [self.SensorType.MLSS, self.SensorType.MLMS]:
return values
if target_sensor['type'] == self.SensorType.MinMax:
ret_value.append(int(int(values[0]) * target_sensor['cf']))
ret_value.append(int(int(values[1]) * target_sensor['cf']))
return tuple(ret_value)
if target_sensor['type'] == self.SensorType.InputLabelX:
for i in range(0, len(values), 2):
ret_dict.update({values[i+1]: int(values[i]) * target_sensor['cf']})
return ret_dict
if target_sensor['type'] == self.SensorType.SingleStringSelect:
for item in values:
if '*' in item:
return item
return None
if target_sensor['type'] == self.SensorType.SingleString:
return values[0]
raise ValueError('Invalid sensor type: {}'.format(target_sensor['type']))
def read_gpu_sensor_set(self, data_type: Enum = SensorSet.All) -> bool:
"""
Read GPU sensor data from HWMON and DEVICE sensors using the sensor set defined
by data_type.
:param data_type: Specifies the sensor set: Dynamic, Static, Info, State, All Monitor
"""
if self.prm.vendor == self.GPU_Vendor.AMD:
return self.read_gpu_sensor_set_amd(data_type)
if self.prm.vendor == self.GPU_Vendor.NVIDIA:
return self.read_gpu_sensor_set_nv(data_type)
return False
def read_gpu_sensor_set_nv(self, data_type: Enum = SensorSet.All) -> bool:
"""
Use the nvidia_smi tool to query GPU parameters.
:param data_type: specifies the set of sensors to read
:return: True if successful, else False and card will have read disabled
"""
if data_type not in self.nv_query_items.keys():
raise TypeError('Invalid SensorSet value: [{}]'.format(data_type))
sensor_dict = GpuItem.nv_query_items[data_type]
nsmi_items = []
query_list = [item for sublist in sensor_dict.values() for item in sublist]
query_list = [item for item in query_list if item not in self.read_disabled]
if self.validated_sensors:
qry_string = ','.join(query_list)
cmd_str = '{} -i {} --query-gpu={} --format=csv,noheader,nounits'.format(
env.GUT_CONST.cmd_nvidia_smi, self.prm.pcie_id, qry_string)
LOGGER.debug('NV command:\n%s', cmd_str)
try:
nsmi_items = subprocess.check_output(shlex.split(cmd_str), shell=False).decode().split('\n')
LOGGER.debug('NV query (single-call) result: [%s]', nsmi_items)
except (subprocess.CalledProcessError, OSError) as except_err:
LOGGER.debug('NV query %s error: [%s]', nsmi_items, except_err)
return False
if nsmi_items:
nsmi_items = nsmi_items[0].split(',')
nsmi_items = [item.strip() for item in nsmi_items]
else:
# Read sensors one at a time if SensorSet.All has not been validated
if data_type == GpuItem.SensorSet.All:
self.validated_sensors = True
for query_item in query_list:
query_data = self.read_gpu_sensor_nv(query_item)
nsmi_items.append(query_data)
LOGGER.debug('NV query (each-call) query item [%s], result: [%s]', query_item, query_data)
if not nsmi_items:
LOGGER.debug('NV query (each-call) failed for all sensors, disabling read for card [%s]',
self.prm.card_num)
self.prm.readable = False
return False
results = dict(zip(query_list, nsmi_items))
LOGGER.debug('NV query result: %s', results)
# Populate GpuItem data from results dictionary
for param_name, sensor_list in sensor_dict.items():
if param_name == 'power_cap_range':
if results['power.min_limit'] and re.fullmatch(PATTERNS['IS_FLOAT'], results['power.min_limit']):
power_min = float(results['power.min_limit'])
else:
power_min = results['power.min_limit']
if results['power.max_limit'] and re.fullmatch(PATTERNS['IS_FLOAT'], results['power.max_limit']):
power_max = float(results['power.max_limit'])
else:
power_max = results['power.max_limit']
self.prm.power_cap_range = [power_min, power_max]
elif param_name == 'power':
if results['power.draw'] and re.fullmatch(PATTERNS['IS_FLOAT'], results['power.draw']):
power = float(results['power.draw'])
else:
power = None
self.set_params_value('power', power)
elif param_name == 'pstates':
pstate_str = re.sub(PATTERNS['ALPHA'], '', results['pstate'])
pstate = int(pstate_str) if pstate_str.isnumeric() else None
self.prm['sclk_ps'][0] = pstate
self.prm['mclk_ps'][0] = pstate
elif param_name in ['temperatures', 'voltages', 'frequencies', 'frequencies_max']:
self.prm[param_name] = {}
for sn_k in sensor_list:
if sn_k not in results: continue
if results[sn_k] and re.fullmatch(PATTERNS['IS_FLOAT'], results[sn_k]):
param_val = float(results[sn_k])
else:
param_val = None
self.prm[param_name].update({sn_k: param_val})
elif re.fullmatch(PATTERNS['GPUMEMTYPE'], param_name):
for sn_k in sensor_list:
if sn_k not in results: continue
mem_value = int(results[sn_k]) if results[sn_k].isnumeric else None
self.prm[param_name] = mem_value / 1024.0
self.set_memory_usage()
elif param_name == 'fan_speed':
sn_k = sensor_list[0]
if re.fullmatch(PATTERNS['IS_FLOAT'], results[sn_k]):
self.prm[param_name] = float(results[sn_k])
self.prm.fan_pwm = self.prm[param_name]
elif param_name == 'link_spd':
self.prm.link_spd = 'GEN{}'.format(results['pcie.link.gen.current'])
elif param_name == 'model':
self.prm.model = results['name']
self.prm.model_display = self.prm.model_device_decode
if results['name'] and len(results['name']) < len(self.prm.model_device_decode):
self.prm.model_display = results['name']
elif len(sensor_list) == 1:
sn_k = sensor_list[0]
if re.fullmatch(PATTERNS['IS_FLOAT'], results[sn_k]):
self.prm[param_name] = float(results[sn_k])
elif not results[sn_k]:
self.prm[param_name] = None
self.prm[param_name] = results[sn_k]
return True
def read_gpu_sensor_set_amd(self, data_type: Enum = SensorSet.All) -> bool:
"""
Read GPU sensor data from HWMON and DEVICE sensors using the sensor set defined
by data_type.
:param data_type: Specifies the sensor set: Dynamic, Static, Info, State, All Monitor
"""
if not self.prm.readable:
return False
return_status = False
param_list = self.sensor_sets[data_type]
for sensor_type, param_names in param_list.items():
for param in param_names:
LOGGER.debug('Processing parameter: %s', param)
rdata = self.read_gpu_sensor(param, vendor=self.prm.vendor, sensor_type=sensor_type)
if rdata is False:
if param != 'unique_id':
LOGGER.debug('Error reading parameter: %s disabling for %s', param, self.prm.card_num)
print('Warning: Error reading parameter: {}, disabling for this GPU: {}'.format(param,
self.prm.card_num))
elif rdata is None:
LOGGER.debug('Read data [%s], Invalid or disabled parameter: %s', rdata, param)
else:
LOGGER.debug('Valid data [%s] for parameter: %s', rdata, param)
self.set_params_value(param, rdata)
return_status = True
return return_status
def print_ppm_table(self) -> None:
"""
Print human friendly table of ppm parameters.
"""
if self.prm.vendor != GpuItem.GPU_Vendor.AMD:
return
if not self.prm.readable or self.prm.gpu_type in [GpuItem.GPU_Type.Legacy, GpuItem.GPU_Type.Unsupported]:
LOGGER.debug('PPM for card number %s not readable.', self.prm.card_num)
return
pre = ' '
print('{}: {}'.format(self._GPU_Param_Labels['card_num'], self.prm.card_num))
print('{}{}: {}'.format(pre, self._GPU_Param_Labels['model'], self.prm.model))
print('{}{}: {}'.format(pre, self._GPU_Param_Labels['card_path'], self.prm.card_path))
print('{}{}: {}'.format(pre, self._GPU_Param_Labels['gpu_type'], self.prm.gpu_type.name))
print('{}{}: {}'.format(pre, self._GPU_Param_Labels['power_dpm_force'], self.prm.power_dpm_force))
print('{}{}{}'.format(pre, '', '#'.ljust(50, '#')))
file_path = os.path.join(self.prm.card_path, 'pp_power_profile_mode')
with open(file_path, 'r') as file_ptr:
lines = file_ptr.readlines()
for line in lines:
print(' {}'.format(line.strip('\n')))
def print_pstates(self) -> None:
"""
Print human friendly table of p-states.
"""
if self.prm.vendor != GpuItem.GPU_Vendor.AMD:
return
if not self.prm.readable or self.prm.gpu_type in [GpuItem.GPU_Type.Legacy, GpuItem.GPU_Type.Unsupported]:
LOGGER.debug('P-states for card number %s not readable.', self.prm.card_num)
return
pre = ' '
print('{}: {}'.format(self._GPU_Param_Labels['card_num'], self.prm.card_num))
print('{}{}: {}'.format(pre, self._GPU_Param_Labels['model'], self.prm.model))
print('{}{}: {}'.format(pre, self._GPU_Param_Labels['card_path'], self.prm.card_path))
print('{}{}: {}'.format(pre, self._GPU_Param_Labels['gpu_type'], self.prm.gpu_type.name))
# DPM States
if self.prm.gpu_type == self.GPU_Type.CurvePts:
print('{}{}{}'.format(pre, '', '#'.ljust(50, '#')))
print('{}DPM States:'.format(pre))
print('{}SCLK: {:<17} MCLK:'.format(pre, ' '))
for ps_num, ps_freq in self.sclk_dpm_state.items():
print('{} {:>1}: {:<8} '.format(pre, ps_num, ps_freq), end='')
if ps_num in self.mclk_dpm_state.keys():
print('{:3>}: {:<8}'.format(ps_num, self.mclk_dpm_state[ps_num]))
else:
print('')
# pp_od_clk_voltage states
print('{}{}{}'.format(pre, '', '#'.ljust(50, '#')))
print('{}PP OD States:'.format(pre))
print('{}SCLK: {:<17} MCLK:'.format(pre, ' '))
for ps_num, ps_vals in self.sclk_state.items():
print('{} {:>1}: {:<8} {:<8} '.format(pre, ps_num, ps_vals[0], ps_vals[1]), end='')
if ps_num in self.mclk_state.keys():
print('{:3>}: {:<8} {:<8}'.format(ps_num, self.mclk_state[ps_num][0], self.mclk_state[ps_num][1]))
else:
print('')
if self.prm.gpu_type == self.GPU_Type.CurvePts:
# Curve points
print('{}{}{}'.format(pre, '', '#'.ljust(50, '#')))
print('{}VDDC_CURVE:'.format(pre))
for vc_index, vc_vals in self.vddc_curve.items():
print('{} {}: {}'.format(pre, vc_index, vc_vals))
print('')
def print(self, short: bool = False, clflag: bool = False) -> None:
"""
Display ls like listing function for GPU parameters.
:param clflag: Display clinfo data if True
:param short: Display short listing
"""
pre = ''
for param_name, param_label in self._GPU_Param_Labels.items():
if short:
if param_name not in self.short_list:
continue
if self.prm.vendor == GpuItem.GPU_Vendor.NVIDIA:
if param_name in self.NV_Skip_List:
continue
elif self.prm.vendor == GpuItem.GPU_Vendor.AMD:
if param_name in self.AMD_Skip_List:
continue
if self.prm.gpu_type == self.GPU_Type.APU:
if param_name in self._fan_item_list:
continue
if 'Range' in param_label:
continue
if self.prm.gpu_type == self.GPU_Type.Legacy:
if param_name in self.LEGACY_Skip_List:
continue
if not self.prm.readable:
if param_name not in self.get_nc_params_list():
continue
pre = '' if param_name == 'card_num' else ' '
if re.search(r'sep[0-9]', param_name):
print('{}{}'.format(pre, param_label.ljust(50, param_label)))
continue
if param_name == 'unique_id':
if self.prm.unique_id is None:
continue
if self.prm.gpu_type == self.GPU_Type.CurvePts and param_name == 'vddc_range':
continue
if isinstance(self.get_params_value(param_name), float):
print('{}{}: {:.3f}'.format(pre, param_label, self.get_params_value(param_name)))
elif isinstance(self.get_params_value(param_name), dict):
param_dict = self.get_params_value(param_name)
print('{}{}: {}'.format(pre, param_label, {key: param_dict[key] for key in sorted(param_dict)}))
elif self.get_params_value(param_name) == '':
print('{}{}: {}'.format(pre, param_label, None))
else:
print('{}{}: {}'.format(pre, param_label, self.get_params_value(param_name)))
if clflag and self.prm.compute:
for param_name, param_label in self._GPU_CLINFO_Labels.items():
if re.search(r'sep[0-9]', param_name):
print('{}{}'.format(pre, param_label.ljust(50, param_label)))
continue
print('{}: {}'.format(param_label, self.get_clinfo_value(param_name)))
print('')
def get_plot_data(self) -> dict:
"""
Return a dictionary of dynamic gpu parameters used by gpu-plot to populate a df.
:return: Dictionary of GPU state info for plot data.
"""
gpu_state = {'Time': str(self.energy['tn'].strftime(env.GUT_CONST.TIME_FORMAT)),
'Card#': int(self.prm.card_num)}
for table_item in self.table_parameters:
gpu_state_str = str(re.sub(PATTERNS['MHz'], '', str(self.get_params_value(table_item)))).strip()
if gpu_state_str == 'nan':
gpu_state[table_item] = np_nan
elif gpu_state_str.isnumeric():
gpu_state[table_item] = int(gpu_state_str)
elif re.fullmatch(PATTERNS['IS_FLOAT'], gpu_state_str):
gpu_state[table_item] = float(gpu_state_str)
elif gpu_state_str == '' or gpu_state_str == '-1' or gpu_state_str == 'NA' or gpu_state_str is None:
gpu_state[table_item] = 'NA'
else:
gpu_state[table_item] = gpu_state_str
return gpu_state
GpuDict = Dict[str, GpuItem]
class GpuList:
"""
A list of GpuItem indexed with uuid. It also contains a table of parameters used for tabular printouts
"""
def __init__(self) -> None:
self.list: GpuDict = {}
self.opencl_map = {}
self.amd_featuremask = None
self.amd_wattman = False
self.amd_writable = False
self.nv_readwritable = False
def __repr__(self) -> dict:
return self.list
def __str__(self) -> str:
return 'GPU_List: Number of GPUs: {}'.format(self.num_gpus())
def __getitem__(self, uuid: str) -> GpuItem:
if uuid in self.list:
return self.list[uuid]
raise KeyError('KeyError: invalid uuid: {}'.format(uuid))
def __setitem__(self, uuid: str, value: GpuItem) -> None:
self.list[uuid] = value
def __iter__(self) -> Generator[GpuItem, None, None]:
for value in self.list.values():
yield value
def items(self) -> Generator[Union[str, GpuItem], None, None]:
"""
Get uuid, gpu pairs from a GpuList object.
:return: uuid, gpu pair
"""
for key, value in self.list.items():
yield key, value
def uuids(self) -> Generator[str, None, None]:
"""
Get uuids of the GpuList object.
:return: uuids from the GpuList object.
"""
for key in self.list:
yield key
def gpus(self) -> Generator[GpuItem, None, None]:
"""
Get GpuItems from a GpuList object.
:return: GpuUItem
"""
return self.__iter__()
def add(self, gpu_item: GpuItem) -> None:
"""
Add given GpuItem to the GpuList.
:param gpu_item: Item to be added
"""
self[gpu_item.prm.uuid] = gpu_item
LOGGER.debug('Added GPU Item %s to GPU List', gpu_item.prm.uuid)
def get_pcie_map(self) -> dict:
"""
Get mapping of card number to pcie address as dict.
:return: dict of num: pcieid
"""
pcie_dict = {}
for gpu in self.gpus():
pcie_dict.update({gpu.prm.card_num: gpu.prm.pcie_id})
return pcie_dict
def wattman_status(self) -> str:
"""
Display Wattman status.
:return: Status string
"""
LOGGER.debug('AMD featuremask: %s', hex(self.amd_featuremask))
if self.amd_wattman:
return 'Wattman features enabled: {}'.format(hex(self.amd_featuremask))
return 'Wattman features not enabled: {}, See README file.'.format(hex(self.amd_featuremask))
@staticmethod
def table_param_labels() -> dict:
"""
Get dictionary of parameter labels to be used in table reports.
:return: Dictionary of table parameters/labels
"""
return GpuItem._table_param_labels
@staticmethod
def table_parameters() -> List[str]:
"""
Get list of parameters to be used in table reports.
:return: List of table parameters
"""
return GpuItem.table_parameters
@staticmethod
def get_gpu_pci_list() -> Union[List[str], None]:
"""
Use call to lspci to get a list of pci addresses of all GPUs.
:return: List of GPU pci addresses.
"""
pci_list = []
try:
lspci_output = subprocess.check_output(env.GUT_CONST.cmd_lspci, shell=False).decode().split('\n')
except (subprocess.CalledProcessError, OSError) as except_err:
print('Error [{}]: lspci failed to find GPUs'.format(except_err))
return None
for lspci_line in lspci_output:
if re.search(PATTERNS['PCI_GPU'], lspci_line):
LOGGER.debug('Found GPU pci: %s', lspci_line)
pciid = re.search(env.GUT_CONST.PATTERNS['PCI_ADD'], lspci_line)
if pciid:
pci_list.append(pciid.group(0))
return pci_list
def set_gpu_list(self, clinfo_flag: bool = False) -> bool:
"""
Use lspci to populate list of all installed GPUs.
:return: True on success
"""
if not env.GUT_CONST.cmd_lspci:
return False
if clinfo_flag:
self.read_gpu_opencl_data()
LOGGER.debug('OpenCL map: %s', self.opencl_map)
# Check AMD writability
try:
self.amd_featuremask = env.GUT_CONST.read_amdfeaturemask()
except FileNotFoundError:
self.amd_wattman = self.amd_writable = False
self.amd_wattman = self.amd_writable = (self.amd_featuremask == int(0xffff7fff) or
self.amd_featuremask == int(0xffffffff) or
self.amd_featuremask == int(0xfffd7fff))
# Check NV read/writability
if env.GUT_CONST.cmd_nvidia_smi:
self.nv_readwritable = True
pcie_ids = self.get_gpu_pci_list()
if not pcie_ids:
print('Error [{}]: lspci failed to find GPUs')
return False
LOGGER.debug('Found %s GPUs', len(pcie_ids))
for pcie_id in pcie_ids:
# Initial GPU Item
gpu_uuid = uuid4().hex
self.add(GpuItem(gpu_uuid))
LOGGER.debug('GPU: %s', pcie_id)
gpu_name = 'UNKNOWN'
driver_module = 'UNKNOWN'
card_path = ''
sys_card_path = ''
hwmon_path = ''
readable = writable = compute = False
gpu_type = GpuItem.GPU_Type.Undefined
vendor = GpuItem.GPU_Vendor.Undefined
opencl_device_version = None if clinfo_flag else 'UNKNOWN'
# Get more GPU details from lspci -k -s
cmd_str = '{} -k -s {}'.format(env.GUT_CONST.cmd_lspci, pcie_id)
try:
lspci_items = subprocess.check_output(shlex.split(cmd_str), shell=False).decode().split('\n')
except (subprocess.CalledProcessError, OSError) as except_err:
LOGGER.debug('Fatal error [%s]: Can not get GPU details with lspci.', except_err)
print('Fatal Error [{}]: Can not get GPU details with lspci'.format(except_err))
sys.exit(-1)
LOGGER.debug('lspci output items:\n %s', lspci_items)
# Get Long GPU Name
gpu_name_items = lspci_items[0].split(': ', maxsplit=1)
if len(gpu_name_items) >= 2:
gpu_name = gpu_name_items[1]
# Check for Fiji ProDuo
if re.search('Fiji', gpu_name):
if re.search(r'Radeon Pro Duo', lspci_items[1].split('[AMD/ATI]')[1]):
gpu_name = 'Radeon Fiji Pro Duo'
# Get GPU brand: AMD, INTEL, NVIDIA, ASPEED
if re.search(PATTERNS['AMD_GPU'], gpu_name):
vendor = GpuItem.GPU_Vendor.AMD
gpu_type = GpuItem.GPU_Type.Supported
if self.opencl_map:
if pcie_id in self.opencl_map.keys():
if 'device_version' in self.opencl_map[pcie_id].keys():
opencl_device_version = self.opencl_map[pcie_id]['device_version']
compute = True
else:
compute = True
if re.search(PATTERNS['NV_GPU'], gpu_name):
vendor = GpuItem.GPU_Vendor.NVIDIA
if env.GUT_CONST.cmd_nvidia_smi:
readable = True
gpu_type = GpuItem.GPU_Type.Supported
if self.opencl_map:
if pcie_id in self.opencl_map.keys():
if 'device_version' in self.opencl_map[pcie_id].keys():
opencl_device_version = self.opencl_map[pcie_id]['device_version']
compute = True
else:
compute = True
if re.search(PATTERNS['INTC_GPU'], gpu_name):
vendor = GpuItem.GPU_Vendor.INTEL
gpu_type = GpuItem.GPU_Type.Unsupported
if self.opencl_map:
if pcie_id in self.opencl_map.keys():
if 'device_version' in self.opencl_map[pcie_id].keys():
opencl_device_version = self.opencl_map[pcie_id]['device_version']
compute = True
else:
compute = not bool(re.search(r' 530', gpu_name))
if re.search(PATTERNS['ASPD_GPU'], gpu_name):
vendor = GpuItem.GPU_Vendor.ASPEED
gpu_type = GpuItem.GPU_Type.Unsupported
if re.search(PATTERNS['MTRX_GPU'], gpu_name):
vendor = GpuItem.GPU_Vendor.MATROX
gpu_type = GpuItem.GPU_Type.Unsupported
# Get Driver Name
for lspci_line in lspci_items:
if re.search(r'([kK]ernel)', lspci_line):
driver_module_items = lspci_line.split(': ')
if len(driver_module_items) >= 2:
driver_module = driver_module_items[1].strip()
# Get full card path
device_dirs = glob.glob(os.path.join(env.GUT_CONST.card_root, 'card?/device'))
# Match system device directory to pcie ID.
for device_dir in device_dirs:
sysfspath = str(Path(device_dir).resolve())
LOGGER.debug('sysfpath: %s\ndevice_dir: %s', sysfspath, device_dir)
if pcie_id == sysfspath[-7:]:
card_path = device_dir
sys_card_path = sysfspath
LOGGER.debug('card_path set to: %s', device_dir)
# No card path could be found. Set readable/writable to False and type to Unsupported
if not card_path:
LOGGER.debug('card_path not set for: %s', pcie_id)
LOGGER.debug('GPU[%s] type set to Unsupported', gpu_uuid)
gpu_type = GpuItem.GPU_Type.Unsupported
readable = writable = False
try_path = '/sys/devices/pci*:*/'
sys_pci_dirs = None
for _ in range(6):
search_path = os.path.join(try_path, '????:{}'.format(pcie_id))
sys_pci_dirs = glob.glob(search_path)
if sys_pci_dirs:
# Found a match
break
try_path = os.path.join(try_path, '????:??:??.?')
if not sys_pci_dirs:
LOGGER.debug('/sys/device file search found no match to pcie_id: %s', pcie_id)
else:
if len(sys_pci_dirs) > 1:
LOGGER.debug('/sys/device file search found multiple matches to pcie_id %s:\n%s',
pcie_id, sys_pci_dirs)
else:
LOGGER.debug('/sys/device file search found match to pcie_id %s:\n%s',
pcie_id, sys_pci_dirs)
sys_card_path = sys_pci_dirs[0]
# Get full hwmon path
if card_path:
LOGGER.debug('Card dir [%s] contents:\n%s', card_path, list(os.listdir(card_path)))
hw_file_srch = glob.glob(os.path.join(card_path, env.GUT_CONST.hwmon_sub) + '?')
LOGGER.debug('HW file search: %s', hw_file_srch)
if len(hw_file_srch) > 1:
print('More than one hwmon file found: {}'.format(hw_file_srch))
elif len(hw_file_srch) == 1:
hwmon_path = hw_file_srch[0]
LOGGER.debug('HW dir [%s] contents:\n%s', hwmon_path, list(os.listdir(hwmon_path)))
# Check AMD write capability
if vendor == GpuItem.GPU_Vendor.AMD and card_path:
pp_od_clk_voltage_file = os.path.join(card_path, 'pp_od_clk_voltage')
if os.path.isfile(pp_od_clk_voltage_file):
gpu_type = GpuItem.GPU_Type.Supported
readable = True
if self.amd_writable:
writable = True
elif os.path.isfile(os.path.join(card_path, 'power_dpm_state')):
if os.path.isfile(os.path.join(card_path, 'pp_dpm_mclk')) or GpuItem.is_apu(gpu_name):
readable = True
gpu_type = GpuItem.GPU_Type.APU
elif os.path.isfile(os.path.join(card_path, 'power_dpm_state')):
# if no pp_od_clk_voltage but has power_dpm_state, assume legacy, and disable some sensors
readable = True
gpu_type = GpuItem.GPU_Type.Legacy
if LOGGER.getEffectiveLevel() == logging.DEBUG:
# Write pp_od_clk_voltage details to debug LOGGER
if os.path.isfile(pp_od_clk_voltage_file):
with open(pp_od_clk_voltage_file, 'r') as file_ptr:
pp_od_file_details = file_ptr.read()
else:
pp_od_file_details = 'The file {} does not exist'.format(pp_od_clk_voltage_file)
LOGGER.debug('%s contents:\n%s', pp_od_clk_voltage_file, pp_od_file_details)
# Set GPU parameters
self[gpu_uuid].populate_prm_from_dict({'pcie_id': pcie_id, 'model': gpu_name,
'vendor': vendor,
'driver': driver_module, 'card_path': card_path,
'sys_card_path': sys_card_path, 'gpu_type': gpu_type,
'hwmon_path': hwmon_path, 'readable': readable,
'writable': writable, 'compute': compute,
'compute_platform': opencl_device_version})
LOGGER.debug('Card flags: readable: %s, writable: %s, type: %s',
readable, writable, self[gpu_uuid].prm.gpu_type)
# Read GPU ID
rdata = self[gpu_uuid].read_gpu_sensor('id', vendor=GpuItem.GPU_Vendor.PCIE, sensor_type='DEVICE')
if rdata:
self[gpu_uuid].set_params_value('id', rdata)
if clinfo_flag:
if pcie_id in self.opencl_map.keys():
self[gpu_uuid].populate_ocl(self.opencl_map[pcie_id])
return True
def read_gpu_opencl_data(self) -> bool:
"""
Use clinfo system call to get openCL details for relevant GPUs.
:return: Returns True if successful
.. todo:: Read of Intel pcie_id is not working.
"""
# Check access to clinfo command
if not env.GUT_CONST.cmd_clinfo:
print('OS Command [clinfo] not found. Use sudo apt-get install clinfo to install', file=sys.stderr)
return False
# Run the clinfo command
cmd = subprocess.Popen(shlex.split('{} --raw'.format(env.GUT_CONST.cmd_clinfo)), shell=False,
stdout=subprocess.PIPE)
# Clinfo Keywords and related opencl_map key.
ocl_keywords = {'CL_KERNEL_PREFERRED_WORK_GROUP_SIZE_MULTIPLE': 'prf_wg_multiple',
'CL_DEVICE_MAX_WORK_GROUP_SIZE': 'max_wg_size',
'CL_DEVICE_PREFERRED_WORK_GROUP_SIZE': 'prf_wg_size',
'CL_DEVICE_MAX_WORK_ITEM_SIZES': 'max_wi_sizes',
'CL_DEVICE_MAX_WORK_ITEM_DIMENSIONS': 'max_wi_dim',
'CL_DEVICE_MAX_MEM_ALLOC_SIZE': 'max_mem_allocation',
'CL_DEVICE_SIMD_INSTRUCTION_WIDTH': 'simd_ins_width',
'CL_DEVICE_SIMD_WIDTH': 'simd_width',
'CL_DEVICE_SIMD_PER_COMPUTE_UNIT': 'simd_per_cu',
'CL_DEVICE_MAX_COMPUTE_UNITS': 'max_cu',
'CL_DEVICE_NAME': 'device_name',
'CL_DEVICE_OPENCL_C_VERSION': 'opencl_version',
'CL_DRIVER_VERSION': 'driver_version',
'CL_DEVICE_VERSION': 'device_version'}
def init_temp_map() -> dict:
"""
Return an initialized clinfo dict.
:return: Initialized clinfo dict
"""
t_dict = {}
for temp_keys in ocl_keywords.values():
t_dict[temp_keys] = None
return t_dict
# Initialize dict variables
ocl_vendor = ocl_index = ocl_pcie_id = ocl_pcie_bus_id = ocl_pcie_slot_id = None
temp_map = init_temp_map()
# Read each line from clinfo --raw
for line in cmd.stdout:
linestr = line.decode('utf-8').strip()
if len(linestr) < 1:
continue
if linestr[0] != '[':
continue
line_items = linestr.split(maxsplit=2)
if len(line_items) != 3:
continue
cl_vendor, cl_index = tuple(re.sub(r'[\[\]]', '', line_items[0]).split('/'))
if cl_index == '*':
continue
if not ocl_index:
ocl_index = cl_index
ocl_vendor = cl_vendor
ocl_pcie_slot_id = ocl_pcie_bus_id = None
# If new cl_index, then update opencl_map
if cl_vendor != ocl_vendor or cl_index != ocl_index:
# Update opencl_map with dict variables when new index is encountered.
self.opencl_map.update({ocl_pcie_id: temp_map})
LOGGER.debug('cl_vendor: %s, cl_index: %s, pcie_id: %s',
ocl_vendor, ocl_index, self.opencl_map[ocl_pcie_id])
# Initialize dict variables
ocl_index = cl_index
ocl_vendor = cl_vendor
ocl_pcie_id = ocl_pcie_bus_id = ocl_pcie_slot_id = None
temp_map = init_temp_map()
param_str = line_items[1]
# Check item in clinfo_keywords
for clinfo_keyword, opencl_map_keyword in ocl_keywords.items():
if clinfo_keyword in param_str:
temp_map[opencl_map_keyword] = line_items[2].strip()
LOGGER.debug('openCL map %s: [%s]', clinfo_keyword, temp_map[opencl_map_keyword])
continue
# PCIe ID related clinfo_keywords
# Check for AMD pcie_id details
if 'CL_DEVICE_TOPOLOGY' in param_str:
ocl_pcie_id = (line_items[2].split()[1]).strip()
LOGGER.debug('AMD ocl_pcie_id [%s]', ocl_pcie_id)
continue
# Check for NV pcie_id details
if 'CL_DEVICE_PCI_BUS_ID_NV' in param_str:
ocl_pcie_bus_id = hex(int(line_items[2].strip()))
if ocl_pcie_slot_id is not None:
ocl_pcie_id = '{}:{}.0'.format(ocl_pcie_bus_id[2:].zfill(2), ocl_pcie_slot_id[2:].zfill(2))
ocl_pcie_slot_id = ocl_pcie_bus_id = None
LOGGER.debug('NV ocl_pcie_id [%s]', ocl_pcie_id)
continue
if 'CL_DEVICE_PCI_SLOT_ID_NV' in param_str:
ocl_pcie_slot_id = hex(int(line_items[2].strip()))
if ocl_pcie_bus_id is not None:
ocl_pcie_id = '{}:{}.0'.format(ocl_pcie_bus_id[2:].zfill(2), ocl_pcie_slot_id[2:].zfill(2))
ocl_pcie_slot_id = ocl_pcie_bus_id = None
LOGGER.debug('NV ocl_pcie_id [%s]', ocl_pcie_id)
continue
# Check for INTEL pcie_id details
# TODO don't know how to do this yet.
self.opencl_map.update({ocl_pcie_id: temp_map})
return True
def num_vendor_gpus(self, compatibility: Enum = GpuItem.GPU_Comp.ALL) -> Dict[str, int]:
"""
Return the count of GPUs by vendor. Counts total by default, but can also by rw, ronly, or wonly.
:param compatibility: Only count vendor GPUs if True.
:return: Dictionary of GPU counts
"""
try:
_ = compatibility.name
except AttributeError:
raise AttributeError('Error: {} not a valid compatibility name: [{}]'.format(
compatibility, GpuItem.GPU_Comp))
results_dict = {}
for gpu in self.gpus():
if compatibility == GpuItem.GPU_Comp.ReadWrite:
if not gpu.prm.readable or not gpu.prm.writable:
continue
if compatibility == GpuItem.GPU_Comp.ReadOnly:
if not gpu.prm.readable:
continue
if compatibility == GpuItem.GPU_Comp.WriteOnly:
if not gpu.prm.writable:
continue
if gpu.prm.vendor.name not in results_dict.keys():
results_dict.update({gpu.prm.vendor.name: 1})
else:
results_dict[gpu.prm.vendor.name] += 1
return results_dict
def num_gpus(self, vendor: Enum = GpuItem.GPU_Vendor.ALL) -> Dict[str, int]:
"""
Return the count of GPUs by total, rw, r-only or w-only.
:param vendor: Only count vendor GPUs of specific vendor or all vendors by default.
:return: Dictionary of GPU counts
"""
try:
vendor_name = vendor.name
except AttributeError:
raise AttributeError('Error: {} not a valid vendor name: [{}]'.format(vendor, GpuItem.GPU_Vendor))
results_dict = {'vendor': vendor_name, 'total': 0, 'rw': 0, 'r-only': 0, 'w-only': 0}
for gpu in self.gpus():
if vendor != GpuItem.GPU_Vendor.ALL:
if vendor != gpu.prm.vendor:
continue
if gpu.prm.readable and gpu.prm.writable:
results_dict['rw'] += 1
elif gpu.prm.readable:
results_dict['r-only'] += 1
elif gpu.prm.writable:
results_dict['w-only'] += 1
results_dict['total'] += 1
return results_dict
def list_gpus(self, vendor: Enum = GpuItem.GPU_Vendor.ALL,
compatibility: Enum = GpuItem.GPU_Comp.ALL) -> 'class GpuList':
"""
Return GPU_Item of GPUs. Contains all by default, but can be a subset with vendor and compatibility args.
Only one flag should be set.
:param vendor: Only count vendor GPUs or ALL by default.
:param compatibility: Only count GPUs with specified compatibility (all, readable, writable)
:return: GpuList of compatible GPUs
"""
try:
_ = compatibility.name
except AttributeError:
raise AttributeError('Error: {} not a valid compatibility name: [{}]'.format(
compatibility, GpuItem.GPU_Comp))
try:
_ = vendor.name
except AttributeError:
raise AttributeError('Error: {} not a valid vendor name: [{}]'.format(vendor, GpuItem.GPU_Vendor))
result_list = GpuList()
for uuid, gpu in self.items():
if vendor != GpuItem.GPU_Vendor.ALL:
if vendor != gpu.prm.vendor:
continue
if compatibility == GpuItem.GPU_Comp.Readable:
# Skip Legacy GPU type, since most parameters can not be read.
if gpu.prm.gpu_type != GpuItem.GPU_Type.Legacy:
if gpu.prm.readable:
result_list[uuid] = gpu
elif compatibility == GpuItem.GPU_Comp.Writable:
if gpu.prm.writable:
result_list[uuid] = gpu
else:
result_list[uuid] = gpu
return result_list
def read_gpu_ppm_table(self) -> None:
"""
Read GPU ppm data and populate GpuItem.
"""
for gpu in self.gpus():
if gpu.prm.readable:
gpu.read_gpu_ppm_table()
def print_ppm_table(self) -> None:
"""
Print the GpuItem ppm data.
"""
for gpu in self.gpus():
gpu.print_ppm_table()
def read_gpu_pstates(self) -> None:
"""
Read GPU p-state data and populate GpuItem.
"""
for gpu in self.gpus():
if gpu.prm.readable:
gpu.read_gpu_pstates()
def print_pstates(self) -> None:
"""
Print the GpuItem p-state data.
"""
for gpu in self.gpus():
gpu.print_pstates()
def read_gpu_sensor_set(self, data_type: Enum = GpuItem.SensorSet.All) -> None:
"""
Read sensor data from all GPUs in self.list.
:param data_type: Specifies the sensor set to use in the read.
"""
for gpu in self.gpus():
if gpu.prm.readable:
gpu.read_gpu_sensor_set(data_type)
# Printing Methods follow.
def print(self, short: bool = False, clflag: bool = False) -> None:
"""
Print all GpuItem.
:param short: If true, print short report
:param clflag: If true, print clinfo
"""
for gpu in self.gpus():
gpu.print(short=short, clflag=clflag)
def print_table(self, title: Union[str, None] = None) -> bool:
"""
Print table of parameters.
:return: True if success
"""
if self.num_gpus()['total'] < 1:
return False
if title:
print('\x1b[1;36m{}\x1b[0m'.format(title))
print('┌', '─'.ljust(13, '─'), sep='', end='')
for _ in self.gpus():
print('┬', '─'.ljust(16, '─'), sep='', end='')
print('┐')
print('│\x1b[1;36m' + 'Card #'.ljust(13, ' ') + '\x1b[0m', sep='', end='')
for gpu in self.gpus():
print('│\x1b[1;36mcard{:<12}\x1b[0m'.format(gpu.prm.card_num), end='')
print('│')
print('├', '─'.ljust(13, '─'), sep='', end='')
for _ in self.gpus():
print('┼', '─'.ljust(16, '─'), sep='', end='')
print('┤')
for table_item in self.table_parameters():
print('│\x1b[1;36m{:<13}\x1b[0m'.format(str(self.table_param_labels()[table_item])[:13]), end='')
for gpu in self.gpus():
data_value_raw = gpu.get_params_value(table_item)
if isinstance(data_value_raw, float):
data_value_raw = round(data_value_raw, 3)
print('│{:<16}'.format(str(data_value_raw)[:16]), end='')
print('│')
print('└', '─'.ljust(13, '─'), sep='', end='')
for _ in self.gpus():
print('┴', '─'.ljust(16, '─'), sep='', end='')
print('┘')
return True
def print_log_header(self, log_file_ptr: TextIO) -> bool:
"""
Print the log header.
:param log_file_ptr: File pointer for target output.
:return: True if success
"""
if self.num_gpus()['total'] < 1:
return False
# Print Header
print('Time|Card#', end='', file=log_file_ptr)
for table_item in self.table_parameters():
print('|{}'.format(table_item), end='', file=log_file_ptr)
print('', file=log_file_ptr)
return True
def print_log(self, log_file_ptr: TextIO) -> bool:
"""
Print the log data.
:param log_file_ptr: File pointer for target output.
:return: True if success
"""
if self.num_gpus()['total'] < 1:
return False
# Print Data
for gpu in self.gpus():
print('{}|{}'.format(gpu.energy['tn'].strftime(env.GUT_CONST.TIME_FORMAT), gpu.prm.card_num),
sep='', end='', file=log_file_ptr)
for table_item in self.table_parameters():
print('|{}'.format(re.sub(PATTERNS['MHz'], '', str(gpu.get_params_value(table_item)).strip())),
sep='', end='', file=log_file_ptr)
print('', file=log_file_ptr)
return True
def print_plot_header(self, log_file_ptr: IO[Union[str, bytes]]) -> bool:
"""
Print the plot header.
:param log_file_ptr: File pointer for target output.
:return: True if success
"""
if self.num_gpus()['total'] < 1:
return False
# Print Header
line_str_item = ['Time|Card#']
for table_item in self.table_parameters():
line_str_item.append('|' + table_item)
line_str_item.append('\n')
line_str = ''.join(line_str_item)
log_file_ptr.write(line_str.encode('utf-8'))
log_file_ptr.flush()
return True
def print_plot(self, log_file_ptr: IO[Union[str, bytes]]) -> bool:
"""
Print the plot data.
:param log_file_ptr: File pointer for target output.
:return: True on success
"""
if self.num_gpus()['total'] < 1:
return False
# Print Data
for gpu in self.gpus():
line_str_item = ['{}|{}'.format(str(gpu.energy['tn'].strftime(env.GUT_CONST.TIME_FORMAT)),
gpu.prm.card_num)]
for table_item in self.table_parameters():
line_str_item.append('|' + re.sub(PATTERNS['MHz'], '', str(gpu.get_params_value(table_item))).strip())
line_str_item.append('\n')
line_str = ''.join(line_str_item)
log_file_ptr.write(line_str.encode('utf-8'))
log_file_ptr.flush()
return True
def about() -> None:
"""
Print details about this module.
"""
print(__doc__)
print('Author: ', __author__)
print('Copyright: ', __copyright__)
print('Credits: ', *['\n {}'.format(item) for item in __credits__])
print('License: ', __license__)
print('Version: ', __version__)
print('Maintainer: ', __maintainer__)
print('Status: ', __status__)
sys.exit(0)
if __name__ == '__main__':
about()
