lxb
/
Tools_CheckGpus
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

update

master
lxbhahaha 1 year ago
parent
e8f36e728b
commit
1f81da00c5
1 changed files with 214 additions and 19 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 233
      check.py

233
check.py
View File

@ -89,6 +89,193 @@ def print_res(data_list):
res = f'{idx}: {status} - {gpu_name} - {diff_len_space}{used_mem} / {total_mem} MiB, GPU Util: {util_gpu} %'
print(res)
table_icon = {
'hline': '─',
'vline': '│',
'left':{
'up' : '┌',
'middle' : '├',
'bottom' : '└',
},
'mid':{
'up' : '┬',
'middle' : '┼',
'bottom' : '┴',
},
'right':{
'up' : '┐',
'middle' : '┤',
'bottom' : '┘',
},
}
def clamp_str(input_str, length, fill=False, fill_type='center', len_is=None):
if len_is:
ori_len = len_is
else:
ori_len = len(input_str)
if ori_len > length:
input_str = input_str[:length-1]
input_str += '…'
else:
diff = length - ori_len
if fill_type == 'center':
left = diff // 2
right = diff - left
elif fill_type == 'left':
left = 0
right = diff
elif fill_type == 'right':
left = diff
right = 0
tmp_list = []
tmp_list.append(' ' * left)
tmp_list.append(input_str)
tmp_list.append(' ' * right)
input_str = ''.join(tmp_list)
return input_str
def get_bar(bar_ratio, max_len, color=False):
assert 0 <= bar_ratio <= 1
res = []
used_len = int(bar_ratio * max_len)
res.append('|' * used_len)
res.append(' ' * (max_len-used_len))
return ''.join(res)
def print_table_res(data_list):
def print_line(line_type, width_list, spaces=[]):
assert line_type in ['up', 'middle', 'bottom']
str_list = []
for i, cell in enumerate(width_list):
# 开头
if i == 0:
if i in spaces:
str_list.append(table_icon['vline'])
else:
str_list.append(table_icon['left'][line_type])
# 中间
str_list.extend((' ' if i in spaces else table_icon['hline']) * cell)
# 右边
if i == len(width_list) - 1:
if i in spaces:
str_list.append(table_icon['vline'])
else:
str_list.append(table_icon['right'][line_type])
else:
if i in spaces and (i+1) in spaces:
str_list.append(table_icon['vline'])
elif i in spaces:
str_list.append(table_icon['left']['middle'])
elif (i + 1) in spaces:
str_list.append(table_icon['right']['middle'])
else:
str_list.append(table_icon['mid'][line_type])
print(''.join(str_list))
# print('TODO')
print(time.strftime("%Y%m-%d%H:%M:%S", time.localtime(time.time())))
cell_width_list = [10, 20, 24, 15]
len_last3 = cell_width_list[1] + cell_width_list[2] + cell_width_list[3] + 2
# 输出head ------------------------------------------
print_line('up', cell_width_list)
str_list = []
str_list.append(table_icon['vline'])
str_list.append(clamp_str('Title', cell_width_list[0], True))
str_list.append(table_icon['vline'])
str_list.append(clamp_str('Name, State, Temperature', cell_width_list[1], True))
str_list.append(table_icon['vline'])
str_list.append(clamp_str('Memory-Usage', cell_width_list[2], True))
str_list.append(table_icon['vline'])
str_list.append(clamp_str('GPU-Util', cell_width_list[3], True))
str_list.append(table_icon['vline'])
print(''.join(str_list))
# 输出内容 ------------------------------------------
for i, data in enumerate(data_list):
print_line('middle', cell_width_list)
title = data['server_data']['title']
info_list = data.get('info_list', None)
if info_list:
for j, info in enumerate(info_list):
str_list = []
str_list.append(table_icon['vline'])
str_list.append(clamp_str(' ', cell_width_list[0], True))
str_list.append(table_icon['vline'])
# 显卡名称
str_list.append(clamp_str(f" {info['idx']} : {info['gpu_name']}", cell_width_list[1], True, 'left'))
str_list.append(table_icon['vline'])
# 显存占用
used_mem = info['used_mem']
total_mem = info['total_mem']
mem_str = f'{used_mem} / {total_mem} MiB '
str_list.append(clamp_str(mem_str, cell_width_list[2], True, 'right'))
str_list.append(table_icon['vline'])
# 显卡利用率
util_gpu = info['util_gpu']
str_list.append(clamp_str(f"{util_gpu} % ", cell_width_list[3], True, 'right'))
str_list.append(table_icon['vline'])
print(''.join(str_list))
# 第二行
str_list = []
# 服务器标题
str_list.append(table_icon['vline'])
str_list.append(clamp_str(title if j==0 else ' ', cell_width_list[0], True))
str_list.append(table_icon['vline'])
# 占用情况
if used_mem < 500:
status = COLOR_GREEN + 'free' + END_COLOR
text_len = 5
elif used_mem / total_mem < 0.5:
status = COLOR_YELLOW + 'occupied' + END_COLOR
text_len = 9
else:
status = COLOR_RED + 'occupied' + END_COLOR
text_len = 9
str_list.append(clamp_str(f" {status}", cell_width_list[1]-5, True, 'left', text_len))
# 温度
str_list.append(clamp_str(f"{info['temperature']}C ", 5, True, 'right'))
str_list.append(table_icon['vline'])
# 显存进度条
str_list.append(get_bar(used_mem/total_mem, cell_width_list[2]))
str_list.append(table_icon['vline'])
# 利用率进度条
str_list.append(get_bar(util_gpu/100, cell_width_list[3]))
str_list.append(table_icon['vline'])
print(''.join(str_list))
# 下一张卡
if j != len(info_list)-1:
print_line('middle', cell_width_list, [0])
else:
str_list = []
str_list.append(table_icon['vline'])
str_list.append(clamp_str(title, cell_width_list[0], True))
str_list.append(table_icon['vline'])
str_list.append(clamp_str('erro', len_last3, True))
str_list.append(table_icon['vline'])
print(''.join(str_list))
print_line('bottom', cell_width_list)
def keep_check_one(server: dict, shared_data_list: list, server_idx: int, interval: float):
try:
# 建立SSH连接
@ -161,6 +348,7 @@ def realtime(args):
parser.add_argument('-n', type=float, default=2, help='服务器多久刷新一次')
parser.add_argument('-f', type=float, default=2, help='显示多久刷新一次')
parser.add_argument('-e', '--exclude', type=str, default='', help='不需要显示的服务器(title)用,分割')
parser.add_argument('-t', '--table', action='store_true', help='以表格形式绘制')
args = parser.parse_args(args)
except:
print('参数有误!')
@ -193,33 +381,40 @@ def realtime(args):
while True:
with data_list_lock:
os.system('cls' if os.name == 'nt' else 'clear')
print(time.strftime("%Y%m-%d%H:%M:%S", time.localtime(time.time())))
for data in data_list:
print(data['server_data']['title'] + ' ---------------')
info_list = data.get('info_list', None)
if info_list:
print_res(info_list)
# print(info_list)
else:
print('出错')
if args.table:
print_table_res(data_list)
else:
print(time.strftime("%Y%m-%d%H:%M:%S", time.localtime(time.time())))
for data in data_list:
print(data['server_data']['title'] + ' ---------------')
info_list = data.get('info_list', None)
if info_list:
print_res(info_list)
# print(info_list)
else:
print('出错')
time.sleep(args.f)
run_realtime = False
def check_all():
def check_all(show_type='list'):
# 加载json
with open(server_list_path, 'r') as f:
server_list = json.load(f)
# 处理每一个结果
for server_data in server_list:
print('* ' + server_data['title'] + ' --------------------')
try:
res = check_gpu_utilization(server_data)
print_res(res)
except:
print('连接出现问题.')
print()
if show_type == 'list':
# 处理每一个结果
for server_data in server_list:
print('* ' + server_data['title'] + ' --------------------')
try:
res = check_gpu_utilization(server_data)
print_res(res)
except:
print('连接出现问题.')
print()
elif show_type == 'table':
pass
# TODO
def print_server_list():
# 加载json

Write Preview
Loading…
Cancel
Save