Package logilab :: Package common :: Module proc
[frames] | no frames]

Source Code for Module logilab.common.proc

  1  # copyright 2003-2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved. 
  2  # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr 
  3  # 
  4  # This file is part of logilab-common. 
  5  # 
  6  # logilab-common is free software: you can redistribute it and/or modify it under 
  7  # the terms of the GNU Lesser General Public License as published by the Free 
  8  # Software Foundation, either version 2.1 of the License, or (at your option) any 
  9  # later version. 
 10  # 
 11  # logilab-common is distributed in the hope that it will be useful, but WITHOUT 
 12  # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 
 13  # FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more 
 14  # details. 
 15  # 
 16  # You should have received a copy of the GNU Lesser General Public License along 
 17  # with logilab-common.  If not, see <http://www.gnu.org/licenses/>. 
 18  """module providing: 
 19  * process information (linux specific: rely on /proc) 
 20  * a class for resource control (memory / time / cpu time) 
 21   
 22  This module doesn't work on windows platforms (only tested on linux) 
 23   
 24  :organization: Logilab 
 25   
 26   
 27   
 28  """ 
 29  __docformat__ = "restructuredtext en" 
 30   
 31  import os 
 32  import stat 
 33  from resource import getrlimit, setrlimit, RLIMIT_CPU, RLIMIT_AS 
 34  from signal import signal, SIGXCPU, SIGKILL, SIGUSR2, SIGUSR1 
 35  from threading import Timer, currentThread, Thread, Event 
 36  from time import time 
 37   
 38  from logilab.common.tree import Node 
 39   
40 -class NoSuchProcess(Exception): pass
41
42 -def proc_exists(pid):
43 """check the a pid is registered in /proc 44 raise NoSuchProcess exception if not 45 """ 46 if not os.path.exists('/proc/%s' % pid): 47 raise NoSuchProcess()
48 49 PPID = 3 50 UTIME = 13 51 STIME = 14 52 CUTIME = 15 53 CSTIME = 16 54 VSIZE = 22 55
56 -class ProcInfo(Node):
57 """provide access to process information found in /proc""" 58
59 - def __init__(self, pid):
60 self.pid = int(pid) 61 Node.__init__(self, self.pid) 62 proc_exists(self.pid) 63 self.file = '/proc/%s/stat' % self.pid 64 self.ppid = int(self.status()[PPID])
65
66 - def memory_usage(self):
67 """return the memory usage of the process in Ko""" 68 try : 69 return int(self.status()[VSIZE]) 70 except IOError: 71 return 0
72
73 - def lineage_memory_usage(self):
74 return self.memory_usage() + sum(child.lineage_memory_usage() for child in self.children)
75
76 - def time(self, children=0):
77 """return the number of jiffies that this process has been scheduled 78 in user and kernel mode""" 79 status = self.status() 80 time = int(status[UTIME]) + int(status[STIME]) 81 if children: 82 time += int(status[CUTIME]) + int(status[CSTIME]) 83 return time
84
85 - def status(self):
86 """return the list of fields found in /proc/<pid>/stat""" 87 return open(self.file).read().split()
88
89 - def name(self):
90 """return the process name found in /proc/<pid>/stat 91 """ 92 return self.status()[1].strip('()')
93
94 - def age(self):
95 """return the age of the process 96 """ 97 return os.stat(self.file)[stat.ST_MTIME]
98
99 -class ProcInfoLoader:
100 """manage process information""" 101
102 - def __init__(self):
103 self._loaded = {}
104
105 - def list_pids(self):
106 """return a list of existent process ids""" 107 for subdir in os.listdir('/proc'): 108 if subdir.isdigit(): 109 yield int(subdir)
110
111 - def load(self, pid):
112 """get a ProcInfo object for a given pid""" 113 pid = int(pid) 114 try: 115 return self._loaded[pid] 116 except KeyError: 117 procinfo = ProcInfo(pid) 118 procinfo.manager = self 119 self._loaded[pid] = procinfo 120 return procinfo
121 122
123 - def load_all(self):
124 """load all processes information""" 125 for pid in self.list_pids(): 126 try: 127 procinfo = self.load(pid) 128 if procinfo.parent is None and procinfo.ppid: 129 pprocinfo = self.load(procinfo.ppid) 130 pprocinfo.append(procinfo) 131 except NoSuchProcess: 132 pass
133 134 135 try:
136 - class ResourceError(BaseException):
137 """Error raise when resource limit is reached""" 138 limit = "Unknown Resource Limit"
139 except NameError:
140 - class ResourceError(Exception):
141 """Error raise when resource limit is reached""" 142 limit = "Unknown Resource Limit"
143 144
145 -class XCPUError(ResourceError):
146 """Error raised when CPU Time limit is reached""" 147 limit = "CPU Time"
148
149 -class LineageMemoryError(ResourceError):
150 """Error raised when the total amount of memory used by a process and 151 it's child is reached""" 152 limit = "Lineage total Memory"
153
154 -class TimeoutError(ResourceError):
155 """Error raised when the process is running for to much time""" 156 limit = "Real Time"
157 158 # Can't use subclass because the StandardError MemoryError raised 159 RESOURCE_LIMIT_EXCEPTION = (ResourceError, MemoryError) 160 161
162 -class MemorySentinel(Thread):
163 """A class checking a process don't use too much memory in a separated 164 daemonic thread 165 """
166 - def __init__(self, interval, memory_limit, gpid=os.getpid()):
167 Thread.__init__(self, target=self._run, name="Test.Sentinel") 168 self.memory_limit = memory_limit 169 self._stop = Event() 170 self.interval = interval 171 self.setDaemon(True) 172 self.gpid = gpid
173
174 - def stop(self):
175 """stop ap""" 176 self._stop.set()
177
178 - def _run(self):
179 pil = ProcInfoLoader() 180 while not self._stop.isSet(): 181 if self.memory_limit <= pil.load(self.gpid).lineage_memory_usage(): 182 os.killpg(self.gpid, SIGUSR1) 183 self._stop.wait(self.interval)
184 185
186 -class ResourceController:
187
188 - def __init__(self, max_cpu_time=None, max_time=None, max_memory=None, 189 max_reprieve=60):
190 if SIGXCPU == -1: 191 raise RuntimeError("Unsupported platform") 192 self.max_time = max_time 193 self.max_memory = max_memory 194 self.max_cpu_time = max_cpu_time 195 self._reprieve = max_reprieve 196 self._timer = None 197 self._msentinel = None 198 self._old_max_memory = None 199 self._old_usr1_hdlr = None 200 self._old_max_cpu_time = None 201 self._old_usr2_hdlr = None 202 self._old_sigxcpu_hdlr = None 203 self._limit_set = 0 204 self._abort_try = 0 205 self._start_time = None 206 self._elapse_time = 0
207
208 - def _hangle_sig_timeout(self, sig, frame):
209 raise TimeoutError()
210
211 - def _hangle_sig_memory(self, sig, frame):
212 if self._abort_try < self._reprieve: 213 self._abort_try += 1 214 raise LineageMemoryError("Memory limit reached") 215 else: 216 os.killpg(os.getpid(), SIGKILL)
217
218 - def _handle_sigxcpu(self, sig, frame):
219 if self._abort_try < self._reprieve: 220 self._abort_try += 1 221 raise XCPUError("Soft CPU time limit reached") 222 else: 223 os.killpg(os.getpid(), SIGKILL)
224
225 - def _time_out(self):
226 if self._abort_try < self._reprieve: 227 self._abort_try += 1 228 os.killpg(os.getpid(), SIGUSR2) 229 if self._limit_set > 0: 230 self._timer = Timer(1, self._time_out) 231 self._timer.start() 232 else: 233 os.killpg(os.getpid(), SIGKILL)
234
235 - def setup_limit(self):
236 """set up the process limit""" 237 assert currentThread().getName() == 'MainThread' 238 os.setpgrp() 239 if self._limit_set <= 0: 240 if self.max_time is not None: 241 self._old_usr2_hdlr = signal(SIGUSR2, self._hangle_sig_timeout) 242 self._timer = Timer(max(1, int(self.max_time) - self._elapse_time), 243 self._time_out) 244 self._start_time = int(time()) 245 self._timer.start() 246 if self.max_cpu_time is not None: 247 self._old_max_cpu_time = getrlimit(RLIMIT_CPU) 248 cpu_limit = (int(self.max_cpu_time), self._old_max_cpu_time[1]) 249 self._old_sigxcpu_hdlr = signal(SIGXCPU, self._handle_sigxcpu) 250 setrlimit(RLIMIT_CPU, cpu_limit) 251 if self.max_memory is not None: 252 self._msentinel = MemorySentinel(1, int(self.max_memory) ) 253 self._old_max_memory = getrlimit(RLIMIT_AS) 254 self._old_usr1_hdlr = signal(SIGUSR1, self._hangle_sig_memory) 255 as_limit = (int(self.max_memory), self._old_max_memory[1]) 256 setrlimit(RLIMIT_AS, as_limit) 257 self._msentinel.start() 258 self._limit_set += 1
259
260 - def clean_limit(self):
261 """reinstall the old process limit""" 262 if self._limit_set > 0: 263 if self.max_time is not None: 264 self._timer.cancel() 265 self._elapse_time += int(time())-self._start_time 266 self._timer = None 267 signal(SIGUSR2, self._old_usr2_hdlr) 268 if self.max_cpu_time is not None: 269 setrlimit(RLIMIT_CPU, self._old_max_cpu_time) 270 signal(SIGXCPU, self._old_sigxcpu_hdlr) 271 if self.max_memory is not None: 272 self._msentinel.stop() 273 self._msentinel = None 274 setrlimit(RLIMIT_AS, self._old_max_memory) 275 signal(SIGUSR1, self._old_usr1_hdlr) 276 self._limit_set -= 1
277