Skip to content

P
pyrasite
Commit 48395120 authored by Luke Macken's avatar Luke Macken
Browse files
Options

Graph I/O and thread activity

parent 35f88d24
Hide whitespace changes
Inline Side-by-side
Showing with 170 additions and 107 deletions
pyrasite/tools/gui.py
View file @ 48395120
...@@ -31,6 +31,7 @@ import tokenize ...@@ -31,6 +31,7 @@ import tokenize
import threading import threading
from os.path import join, abspath, dirname from os.path import join, abspath, dirname
from random import randrange
from meliae import loader from meliae import loader
from gi.repository import GLib, GObject, Pango, Gtk, WebKit from gi.repository import GLib, GObject, Pango, Gtk, WebKit
...@@ -39,19 +40,32 @@ from pyrasite.utils import setup_logger, run, humanize_bytes ...@@ -39,19 +40,32 @@ from pyrasite.utils import setup_logger, run, humanize_bytes
log = logging.getLogger('pyrasite') log = logging.getLogger('pyrasite')
socket_families = dict([(getattr(socket, k), k) for k in dir(socket) POLL_INTERVAL = 1
if k.startswith('AF_')]) INTERVALS = 200
socket_types = dict([(getattr(socket, k), k) for k in dir(socket)
if k.startswith('SOCK_')])
POLL_INTERVAL = 3
cpu_intervals = [] cpu_intervals = []
cpu_details = '' cpu_details = ''
mem_intervals = [] mem_intervals = []
mem_details = '' mem_details = ''
write_intervals = []
read_intervals = []
read_count = read_bytes = write_count = write_bytes = 0 read_count = read_bytes = write_count = write_bytes = 0
# TODO: make sure we flush thread ids when we switch processes
thread_intervals = {}
thread_colors = {}
thread_totals = {}
# Prefer tango colors for our lines. Fall back to random ones.
tango = ['c4a000', 'ce5c00', '8f5902', '4e9a06', '204a87', '5c3566',
'a40000', '555753']
def get_color():
used = thread_colors.values()
for color in tango:
if color not in used:
return color
return "".join([hex(randrange(0, 255))[2:] for i in range(3)])
class Process(pyrasite.PyrasiteIPC, GObject.GObject): class Process(pyrasite.PyrasiteIPC, GObject.GObject):
""" """
...@@ -286,111 +300,114 @@ class PyrasiteWindow(Gtk.Window): ...@@ -286,111 +300,114 @@ class PyrasiteWindow(Gtk.Window):
def generate_description(self, proc, title): def generate_description(self, proc, title):
p = psutil.Process(proc.pid) p = psutil.Process(proc.pid)
io = p.get_io_counters()
self.info_html = """ self.info_html = """
<html><head> <html><head>
<style> <style>
body {font: normal 12px/150%% Arial, Helvetica, sans-serif;} body {font: normal 12px/150%% Arial, Helvetica, sans-serif;}
.grid table { border-collapse: collapse; text-align: left; width: 100%%; } .grid table { border-collapse: collapse; text-align: left; width: 100%%; }
.grid {font: normal 12px/150%% Arial, Helvetica, sans-serif; background: #fff; overflow: hidden; border: 1px solid #006699; -webkit-border-radius: 3px; border-radius: 3px; } .grid {
.grid table td, .grid table th { padding: 3px 10px; } font: normal 12px/150%% Arial, Helvetica, sans-serif;
.grid table thead th { background: #fff; overflow: hidden; border: 1px solid #2e3436;
background:-webkit-gradient( linear, left top, left bottom, color-stop(0.05, #006699), color-stop(1, #00557F) ); -webkit-border-radius: 3px; border-radius: 3px;
background-color:#006699; color:#FFFFFF; font-size: 15px; }
font-weight: bold; border-left: 1px solid #0070A8; } .grid table td, .grid table th { padding: 3px 10px; }
.grid table thead th:first-child { border: none; } .grid table thead th {
.grid table tbody td { color: #00557F; border-left: 1px solid #E1EEF4;font-size: 12px;font-weight: normal; } background:-webkit-gradient( linear, left top, left bottom, color-stop(0.05, #888a85), color-stop(1, #555753) );
.grid table tbody .alt td { background: #E1EEf4; color: #00557F; } background-color:#2e3436; color:#FFFFFF; font-size: 15px;
.grid table tbody td:first-child { border: none; } font-weight: bold; border-left: 1px solid #2e3436; }
</style> .grid table thead th:first-child { border: none; }
.grid table tbody td { color: #2e3436; border-left: 1px solid #2e3436;font-size: 12px;font-weight: normal; }
.grid table tbody .alt td { background: #d3d7cf; color: #2e3436; }
.grid table tbody td:first-child { border: none; }
</style>
</head> </head>
<body> <body>
<h2>%(title)s</h2>
<h2>%(title)s</h2> <div class="grid">
<div class="grid"> <table>
<table> <thead>
<thead> <tr><th width="50%%">CPU: <span id="cpu_details"/></th>
<tr><th>CPU: <span id="cpu_details"/></th> <th width="50%%">Memory: <span id="mem_details"/></th></tr>
<th>Memory: <span id="mem_details"/></th></tr> </thead>
</thead> <tbody>
<tbody> <tr><td><span id="cpu_graph" class="cpu_graph"></span></td>
<tr><td><span id="cpu_graph" class="cpu_graph"></span></td> <td><span id="mem_graph" class="mem_graph"></span></td></tr>
<td><span id="mem_graph" class="mem_graph"></span></td></tr> </tbody>
</tbody> </table>
</table> </div>
</div> <br/>
<div class="grid">
<table>
<thead>
<tr><th width="50%%">Read: <span id="read_details"/></th>
<th width="50%%">Write: <span id="write_details"/></th></tr>
</thead>
<tbody>
<tr><td><span id="read_graph"></span></td>
<td><span id="write_graph"></span></td></tr>
</tbody>
</table>
</div>
<br/>
<div class="grid">
<table>
<thead>
<tr><th>Threads</th></tr>
</thead>
<tbody>
<tr><td><span id="thread_graph"></span></td></tr>
</tbody>
</table>
</div>
<br/>
""" % dict(title = proc.title) """ % dict(title = proc.title)
self.info_html += """
<h3>I/O Counters</h3>
<div class="grid">
<table>
<thead><tr><th></th><th>Count</th><th>Size</th></tr></thead>
<tbody>
<tr><td>Read</td><td><span id="read_count">%s</span></td>
<td><span id="read_size">%s</span></td></tr>
<tr><td>Write</td><td><span id="write_count">%s</span></td>
<td><span id="write_size">%s</span></td></tr>
</tbody>
</table>
</div>
""" % (io.read_count, humanize_bytes(io.read_bytes),
io.write_count, humanize_bytes(io.write_bytes))
open_files = p.get_open_files() open_files = p.get_open_files()
if open_files: if open_files:
self.info_html += """ self.info_html += """
<h3>Open Files</h3>
<div class="grid"> <div class="grid">
<table> <table>
<thead><tr><th>fd</th><th>Path</th></tr></thead> <thead><tr><th>Open Files</th></tr></thead>
<tbody>%(open_files)s</tbody> <tbody>%(open_files)s</tbody>
</table> </table>
</div> </div>
<br/>
""" % dict( """ % dict(
open_files = ''.join(['<tr%s><td>%s</td><td>%s</td></tr>' % open_files = ''.join(['<tr%s><td>%s</td></tr>' %
(i % 2 and ' class="alt"' or '',f.fd,f.path) (i % 2 and ' class="alt"' or '', f.path)
for i, f in enumerate(open_files)])) for i, f in enumerate(open_files)]))
conns = p.get_connections() conns = p.get_connections()
if conns: if conns:
self.info_html += """ self.info_html += """
<h3>Connections</h3>
<div class="grid"> <div class="grid">
<table> <table>
<thead><tr><th>fd</th><th>Family</th><th>Type</th> <thead><tr><th colspan="4">Connections</th></tr></thead>
<th>Local</th><th>Remote</th><th>Status</th></tr></thead>
<tbody> <tbody>
""" """
for i, c in enumerate(conns): for i, conn in enumerate(conns):
if conn.type == socket.SOCK_STREAM:
type = 'TCP'
elif conn.type == socket.SOCK_DGRAM:
type = 'UDP'
else:
type = 'UNIX'
lip, lport = conn.local_address
if not conn.remote_address:
rip = rport = '*'
else:
rip, rport = conn.remote_address
self.info_html += """ self.info_html += """
<tr%s><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td> <tr%s><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>
<td>%s</td></tr> """ % (i % 2 and ' class="alt"' or '',
""" % (i % 2 and ' class="alt"' or '', c.fd, type, '%s:%s' % (lip, lport),
socket_families[c.family], socket_types[c.type], '%s:%s' % (rip, rport), conn.status)
':'.join(map(str, c.local_address)),
':'.join(map(str, c.remote_address)),
c.status)
self.info_html += """
</tbody></table></div>
"""
threads = p.get_threads()
if threads:
self.info_html += """ self.info_html += """
<h3>Threads</h3> </tbody></table></div>
<div class="grid">
<table>
<thead><tr><th>ID</th><th>User Time</th><th>System Time</th>
</tr></thead><tbody>
""" """
for i, thread in enumerate(threads):
self.info_html += """
<tr%s><td>%s</td><td>%s</td><td>%s</td></tr>
""" % (i % 2 and ' class="alt"' or '', thread.id,
thread.user_time, thread.system_time)
self.info_html += "</tbody></table></div>"
self.info_html += "</body></html>" self.info_html += "</body></html>"
...@@ -419,10 +436,6 @@ class PyrasiteWindow(Gtk.Window): ...@@ -419,10 +436,6 @@ class PyrasiteWindow(Gtk.Window):
self.details_view.load_string(self.details_html, "text/html", "utf-8", '#') self.details_view.load_string(self.details_html, "text/html", "utf-8", '#')
global cpu_intervals, mem_intervals, cpu_details, mem_details
cpu_intervals = [p.get_cpu_percent(interval=1.0)]
mem_intervals = [p.get_memory_info().rss]
if not self.resource_thread: if not self.resource_thread:
self.resource_thread = ResourceUsagePoller(proc.pid) self.resource_thread = ResourceUsagePoller(proc.pid)
#self.resource_thread.process = p #self.resource_thread.process = p
...@@ -431,19 +444,8 @@ class PyrasiteWindow(Gtk.Window): ...@@ -431,19 +444,8 @@ class PyrasiteWindow(Gtk.Window):
self.resource_thread.start() self.resource_thread.start()
self.resource_thread.process = p self.resource_thread.process = p
def poll_resource_usage(): GObject.timeout_add(100, self.inject_js)
self.info_view.execute_script(""" GObject.timeout_add(3500, self.render_resource_usage)
jQuery('#cpu_graph').sparkline(%s, {'height': 50});
jQuery('#mem_graph').sparkline(%s, {'height': 50, lineColor: '#f00',
fillColor: '#ffa', minSpotColor: false, maxSpotColor: false,
spotColor: '#77f', spotRadius: 3});
jQuery('#cpu_details').text('%s');
jQuery('#mem_details').text('%s');
""" % (cpu_intervals, mem_intervals, cpu_details, mem_details))
return True
GObject.timeout_add(500, self.inject_js)
GObject.timeout_add(3500, poll_resource_usage)
def inject_js(self): def inject_js(self):
log.debug("Injecting jQuery") log.debug("Injecting jQuery")
...@@ -455,6 +457,42 @@ class PyrasiteWindow(Gtk.Window): ...@@ -455,6 +457,42 @@ class PyrasiteWindow(Gtk.Window):
self.info_view.execute_script(sparkline.read()) self.info_view.execute_script(sparkline.read())
sparkline.close() sparkline.close()
def render_resource_usage(self):
"""Render our resource usage using jQuery+Sparklines in our WebKit view"""
global cpu_intervals, mem_intervals, cpu_details, mem_details
global read_intervals, write_intervals, read_bytes, write_bytes
script = """
jQuery('#cpu_graph').sparkline(%s, {'height': 75, 'width': 250, spotRadius: 3,
fillColor: '#73d216', lineColor: '#4e9a06'});
jQuery('#mem_graph').sparkline(%s, {'height': 75, 'width': 250,
lineColor: '#5c3566', fillColor: '#75507b', minSpotColor: false,
maxSpotColor: false, spotColor: '#f57900', spotRadius: 3});
jQuery('#cpu_details').text('%s');
jQuery('#mem_details').text('%s');
jQuery('#read_graph').sparkline(%s, {'height': 75, 'width': 250,
lineColor: '#a40000', fillColor: '#cc0000', minSpotColor: false,
maxSpotColor: false, spotColor: '#729fcf', spotRadius: 3});
jQuery('#write_graph').sparkline(%s, {'height': 75, 'width': 250,
lineColor: '#ce5c00', fillColor: '#f57900', minSpotColor: false,
maxSpotColor: false, spotColor: '#8ae234', spotRadius: 3});
jQuery('#read_details').text('%s');
jQuery('#write_details').text('%s');
""" % (cpu_intervals, mem_intervals, cpu_details, mem_details,
read_intervals, write_intervals, humanize_bytes(read_bytes),
humanize_bytes(write_bytes))
for i, thread in enumerate(thread_intervals):
script += """
jQuery('#thread_graph').sparkline(%s, {
%s'lineColor': '#%s', 'fillColor': false, 'spotRadius': 3,
'spotColor': '#%s'});
""" % (thread_intervals[thread], i != 0 and "'composite': true,"
or "'height': 75, 'width': 575,", thread_colors[thread],
thread_colors[thread])
self.info_view.execute_script(script)
return True
def update_progress(self, fraction, text=None): def update_progress(self, fraction, text=None):
if text: if text:
self.progress.set_text(text + '...') self.progress.set_text(text + '...')
...@@ -484,7 +522,7 @@ class PyrasiteWindow(Gtk.Window): ...@@ -484,7 +522,7 @@ class PyrasiteWindow(Gtk.Window):
self.generate_description(proc, title) self.generate_description(proc, title)
# Inject a reverse subshell # Inject a reverse subshell
self.update_progress(0.2, "Injecting backdoor") self.update_progress(0.2, "Injecting reverse connection")
if proc.title not in self.processes: if proc.title not in self.processes:
proc.connect() proc.connect()
self.processes[proc.title] = proc self.processes[proc.title] = proc
...@@ -755,11 +793,15 @@ class ResourceUsagePoller(threading.Thread): ...@@ -755,11 +793,15 @@ class ResourceUsagePoller(threading.Thread):
def run(self): def run(self):
global cpu_intervals, mem_intervals, cpu_details, mem_details global cpu_intervals, mem_intervals, cpu_details, mem_details
global read_count, read_bytes, write_count, write_bytes global read_count, read_bytes, write_count, write_bytes
global read_intervals, write_intervals, thread_intervals
while True: while True:
if self.process: if self.process:
if len(cpu_intervals) >= 100: if len(cpu_intervals) >= INTERVALS:
cpu_intervals = cpu_intervals[1:100] cpu_intervals = cpu_intervals[1:INTERVALS]
mem_intervals = mem_intervals[1:100] mem_intervals = mem_intervals[1:INTERVALS]
read_intervals = read_intervals[1:INTERVALS]
write_intervals = write_intervals[1:INTERVALS]
cpu_intervals.append( cpu_intervals.append(
self.process.get_cpu_percent(interval=POLL_INTERVAL)) self.process.get_cpu_percent(interval=POLL_INTERVAL))
mem_intervals.append(self.process.get_memory_info().rss) mem_intervals.append(self.process.get_memory_info().rss)
...@@ -771,16 +813,37 @@ class ResourceUsagePoller(threading.Thread): ...@@ -771,16 +813,37 @@ class ResourceUsagePoller(threading.Thread):
self.process.get_memory_percent(), self.process.get_memory_percent(),
humanize_bytes(meminfo.rss), humanize_bytes(meminfo.rss),
humanize_bytes(cputimes.system)) humanize_bytes(cputimes.system))
io = self.process.get_io_counters() io = self.process.get_io_counters()
read_since_last = io.read_bytes - read_bytes
read_intervals.append(read_since_last)
read_count = io.read_count read_count = io.read_count
read_bytes = humanize_bytes(io.read_bytes) read_bytes = io.read_bytes
write_since_last = io.write_bytes - write_bytes
write_intervals.append(write_since_last)
write_count = io.write_count write_count = io.write_count
write_bytes = humanize_bytes(io.write_bytes) write_bytes = io.write_bytes
for thread in self.process.get_threads():
if thread.id not in thread_intervals:
thread_intervals[thread.id] = []
thread_colors[thread.id] = get_color()
thread_totals[thread.id] = 0.0
if len(thread_intervals[thread.id]) >= INTERVALS:
thread_intervals[thread.id] = thread_intervals[thread.id][1:INTERVALS]
# FIXME: we should figure out some way to visually
# distinguish between user and system time.
total = thread.system_time + thread.user_time
amount_since = total - thread_totals[thread.id]
thread_intervals[thread.id].append(float('%.2f' % amount_since))
thread_totals[thread.id] = total
def main(): def main():
mainloop = GLib.MainLoop()
GObject.threads_init() GObject.threads_init()
mainloop = GLib.MainLoop()
window = PyrasiteWindow() window = PyrasiteWindow()
window.show() window.show()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7