Package CedarBackup2 :: Package extend :: Module postgresql
[hide private]
[frames] | no frames]

Source Code for Module CedarBackup2.extend.postgresql

  1  # -*- coding: iso-8859-1 -*- 
  2  # vim: set ft=python ts=3 sw=3 expandtab: 
  3  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
  4  # 
  5  #              C E D A R 
  6  #          S O L U T I O N S       "Software done right." 
  7  #           S O F T W A R E 
  8  # 
  9  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
 10  # 
 11  # Copyright (c) 2006 Kenneth J. Pronovici. 
 12  # Copyright (c) 2006 Antoine Beaupre. 
 13  # All rights reserved. 
 14  # 
 15  # This program is free software; you can redistribute it and/or 
 16  # modify it under the terms of the GNU General Public License, 
 17  # Version 2, as published by the Free Software Foundation. 
 18  # 
 19  # This program is distributed in the hope that it will be useful, 
 20  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
 21  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 
 22  # 
 23  # Copies of the GNU General Public License are available from 
 24  # the Free Software Foundation website, http://www.gnu.org/. 
 25  # 
 26  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
 27  # 
 28  # Author   : Kenneth J. Pronovici <pronovic@ieee.org> 
 29  #            Antoine Beaupre <anarcat@koumbit.org> 
 30  # Language : Python (>= 2.3) 
 31  # Project  : Official Cedar Backup Extensions 
 32  # Revision : $Id: postgresql.py 570 2006-06-24 20:29:45Z pronovic $ 
 33  # Purpose  : Provides an extension to back up PostgreSQL databases. 
 34  # 
 35  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
 36  # This file was created with a width of 132 characters, and NO tabs. 
 37   
 38  ######################################################################## 
 39  # Module documentation 
 40  ######################################################################## 
 41   
 42  """ 
 43  Provides an extension to back up PostgreSQL databases. 
 44   
 45  This is a Cedar Backup extension used to back up PostgreSQL databases via the 
 46  Cedar Backup command line.  It requires a new configurations section 
 47  <postgresql> and is intended to be run either immediately before or immediately 
 48  after the standard collect action.  Aside from its own configuration, it 
 49  requires the options and collect configuration sections in the standard Cedar 
 50  Backup configuration file. 
 51   
 52  The backup is done via the C{pg_dump} or C{pg_dumpall} commands included with 
 53  the PostgreSQL product.  Output can be compressed using C{gzip} or C{bzip2}. 
 54  Administrators can configure the extension either to back up all databases or 
 55  to back up only specific databases.  The extension assumes that the current 
 56  user has passwordless access to the database since there is no easy way to pass 
 57  a password to the C{pg_dump} client. This can be accomplished using appropriate 
 58  voodoo in the C{pg_hda.conf} file. 
 59   
 60  Note that this code always produces a full backup.  There is currently no 
 61  facility for making incremental backups. 
 62   
 63  You should always make C{/etc/cback.conf} unreadble to non-root users once you 
 64  place postgresql configuration into it, since postgresql configuration will 
 65  contain information about available PostgreSQL databases and usernames. 
 66   
 67  Use of this extension I{may} expose usernames in the process listing (via 
 68  C{ps}) when the backup is running if the username is specified in the 
 69  configuration. 
 70   
 71  @author: Kenneth J. Pronovici <pronovic@ieee.org> 
 72  @author: Antoine Beaupre <anarcat@koumbit.org> 
 73  """ 
 74   
 75  ######################################################################## 
 76  # Imported modules 
 77  ######################################################################## 
 78   
 79  # System modules 
 80  import os 
 81  import logging 
 82  from gzip import GzipFile 
 83  from bz2 import BZ2File 
 84   
 85  # Cedar Backup modules 
 86  from CedarBackup2.xmlutil import createInputDom, addContainerNode, addStringNode, addBooleanNode 
 87  from CedarBackup2.xmlutil import readChildren, readFirstChild, readString, readStringList, readBoolean 
 88  from CedarBackup2.config import VALID_COLLECT_MODES, VALID_COMPRESS_MODES 
 89  from CedarBackup2.util import resolveCommand, executeCommand 
 90  from CedarBackup2.util import ObjectTypeList, changeOwnership 
 91   
 92   
 93  ######################################################################## 
 94  # Module-wide constants and variables 
 95  ######################################################################## 
 96   
 97  logger = logging.getLogger("CedarBackup2.log.extend.postgresql") 
 98  POSTGRESQLDUMP_COMMAND = [ "pg_dump", ] 
 99  POSTGRESQLDUMPALL_COMMAND = [ "pg_dumpall", ] 
100   
101   
102  ######################################################################## 
103  # PostgresqlConfig class definition 
104  ######################################################################## 
105   
106 -class PostgresqlConfig(object):
107 108 """ 109 Class representing PostgreSQL configuration. 110 111 The PostgreSQL configuration information is used for backing up PostgreSQL databases. 112 113 The following restrictions exist on data in this class: 114 115 - The compress mode must be one of the values in L{VALID_COMPRESS_MODES}. 116 - The 'all' flag must be 'Y' if no databases are defined. 117 - The 'all' flag must be 'N' if any databases are defined. 118 - Any values in the databases list must be strings. 119 120 @sort: __init__, __repr__, __str__, __cmp__, user, all, databases 121 """ 122
123 - def __init__(self, user=None, compressMode=None, all=None, databases=None):
124 """ 125 Constructor for the C{PostgresqlConfig} class. 126 127 @param user: User to execute backup as. 128 @param compressMode: Compress mode for backed-up files. 129 @param all: Indicates whether to back up all databases. 130 @param databases: List of databases to back up. 131 """ 132 self._user = None 133 self._compressMode = None 134 self._all = None 135 self._databases = None 136 self.user = user 137 self.compressMode = compressMode 138 self.all = all 139 self.databases = databases
140
141 - def __repr__(self):
142 """ 143 Official string representation for class instance. 144 """ 145 return "PostgresqlConfig(%s, %s, %s)" % (self.user, self.all, self.databases)
146
147 - def __str__(self):
148 """ 149 Informal string representation for class instance. 150 """ 151 return self.__repr__()
152
153 - def __cmp__(self, other):
154 """ 155 Definition of equals operator for this class. 156 @param other: Other object to compare to. 157 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other. 158 """ 159 if other is None: 160 return 1 161 if self._user != other._user: 162 if self._user < other._user: 163 return -1 164 else: 165 return 1 166 if self._compressMode != other._compressMode: 167 if self._compressMode < other._compressMode: 168 return -1 169 else: 170 return 1 171 if self._all != other._all: 172 if self._all < other._all: 173 return -1 174 else: 175 return 1 176 if self._databases != other._databases: 177 if self._databases < other._databases: 178 return -1 179 else: 180 return 1 181 return 0
182
183 - def _setUser(self, value):
184 """ 185 Property target used to set the user value. 186 """ 187 if value is not None: 188 if len(value) < 1: 189 raise ValueError("User must be non-empty string.") 190 self._user = value
191
192 - def _getUser(self):
193 """ 194 Property target used to get the user value. 195 """ 196 return self._user
197
198 - def _setCompressMode(self, value):
199 """ 200 Property target used to set the compress mode. 201 If not C{None}, the mode must be one of the values in L{VALID_COMPRESS_MODES}. 202 @raise ValueError: If the value is not valid. 203 """ 204 if value is not None: 205 if value not in VALID_COMPRESS_MODES: 206 raise ValueError("Compress mode must be one of %s." % VALID_COMPRESS_MODES) 207 self._compressMode = value
208
209 - def _getCompressMode(self):
210 """ 211 Property target used to get the compress mode. 212 """ 213 return self._compressMode
214
215 - def _setAll(self, value):
216 """ 217 Property target used to set the 'all' flag. 218 No validations, but we normalize the value to C{True} or C{False}. 219 """ 220 if value: 221 self._all = True 222 else: 223 self._all = False
224
225 - def _getAll(self):
226 """ 227 Property target used to get the 'all' flag. 228 """ 229 return self._all
230
231 - def _setDatabases(self, value):
232 """ 233 Property target used to set the databases list. 234 Either the value must be C{None} or each element must be a string. 235 @raise ValueError: If the value is not a string. 236 """ 237 if value is None: 238 self._databases = None 239 else: 240 for database in value: 241 if len(database) < 1: 242 raise ValueError("Each database must be a non-empty string.") 243 try: 244 saved = self._databases 245 self._databases = ObjectTypeList(basestring, "string") 246 self._databases.extend(value) 247 except Exception, e: 248 self._databases = saved 249 raise e
250
251 - def _getDatabases(self):
252 """ 253 Property target used to get the databases list. 254 """ 255 return self._databases
256 257 user = property(_getUser, _setUser, None, "User to execute backup as.") 258 compressMode = property(_getCompressMode, _setCompressMode, None, "Compress mode to be used for backed-up files.") 259 all = property(_getAll, _setAll, None, "Indicates whether to back up all databases.") 260 databases = property(_getDatabases, _setDatabases, None, "List of databases to back up.")
261 262 263 ######################################################################## 264 # LocalConfig class definition 265 ######################################################################## 266
267 -class LocalConfig(object):
268 269 """ 270 Class representing this extension's configuration document. 271 272 This is not a general-purpose configuration object like the main Cedar 273 Backup configuration object. Instead, it just knows how to parse and emit 274 PostgreSQL-specific configuration values. Third parties who need to read and 275 write configuration related to this extension should access it through the 276 constructor, C{validate} and C{addConfig} methods. 277 278 @note: Lists within this class are "unordered" for equality comparisons. 279 280 @sort: __init__, __repr__, __str__, __cmp__, postgresql, validate, addConfig 281 """ 282
283 - def __init__(self, xmlData=None, xmlPath=None, validate=True):
284 """ 285 Initializes a configuration object. 286 287 If you initialize the object without passing either C{xmlData} or 288 C{xmlPath} then configuration will be empty and will be invalid until it 289 is filled in properly. 290 291 No reference to the original XML data or original path is saved off by 292 this class. Once the data has been parsed (successfully or not) this 293 original information is discarded. 294 295 Unless the C{validate} argument is C{False}, the L{LocalConfig.validate} 296 method will be called (with its default arguments) against configuration 297 after successfully parsing any passed-in XML. Keep in mind that even if 298 C{validate} is C{False}, it might not be possible to parse the passed-in 299 XML document if lower-level validations fail. 300 301 @note: It is strongly suggested that the C{validate} option always be set 302 to C{True} (the default) unless there is a specific need to read in 303 invalid configuration from disk. 304 305 @param xmlData: XML data representing configuration. 306 @type xmlData: String data. 307 308 @param xmlPath: Path to an XML file on disk. 309 @type xmlPath: Absolute path to a file on disk. 310 311 @param validate: Validate the document after parsing it. 312 @type validate: Boolean true/false. 313 314 @raise ValueError: If both C{xmlData} and C{xmlPath} are passed-in. 315 @raise ValueError: If the XML data in C{xmlData} or C{xmlPath} cannot be parsed. 316 @raise ValueError: If the parsed configuration document is not valid. 317 """ 318 self._postgresql = None 319 self.postgresql = None 320 if xmlData is not None and xmlPath is not None: 321 raise ValueError("Use either xmlData or xmlPath, but not both.") 322 if xmlData is not None: 323 self._parseXmlData(xmlData) 324 if validate: 325 self.validate() 326 elif xmlPath is not None: 327 xmlData = open(xmlPath).read() 328 self._parseXmlData(xmlData) 329 if validate: 330 self.validate()
331
332 - def __repr__(self):
333 """ 334 Official string representation for class instance. 335 """ 336 return "LocalConfig(%s)" % (self.postgresql)
337
338 - def __str__(self):
339 """ 340 Informal string representation for class instance. 341 """ 342 return self.__repr__()
343
344 - def __cmp__(self, other):
345 """ 346 Definition of equals operator for this class. 347 Lists within this class are "unordered" for equality comparisons. 348 @param other: Other object to compare to. 349 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other. 350 """ 351 if other is None: 352 return 1 353 if self._postgresql != other._postgresql: 354 if self._postgresql < other._postgresql: 355 return -1 356 else: 357 return 1 358 return 0
359
360 - def _setPostgresql(self, value):
361 """ 362 Property target used to set the postgresql configuration value. 363 If not C{None}, the value must be a C{PostgresqlConfig} object. 364 @raise ValueError: If the value is not a C{PostgresqlConfig} 365 """ 366 if value is None: 367 self._postgresql = None 368 else: 369 if not isinstance(value, PostgresqlConfig): 370 raise ValueError("Value must be a C{PostgresqlConfig} object.") 371 self._postgresql = value
372
373 - def _getPostgresql(self):
374 """ 375 Property target used to get the postgresql configuration value. 376 """ 377 return self._postgresql
378 379 postgresql = property(_getPostgresql, _setPostgresql, None, "Postgresql configuration in terms of a C{PostgresqlConfig} object.") 380
381 - def validate(self):
382 """ 383 Validates configuration represented by the object. 384 385 The compress mode must be filled in. Then, if the 'all' flag 386 I{is} set, no databases are allowed, and if the 'all' flag is 387 I{not} set, at least one database is required. 388 389 @raise ValueError: If one of the validations fails. 390 """ 391 if self.postgresql is None: 392 raise ValueError("PostgreSQL section is required.") 393 if self.postgresql.compressMode is None: 394 raise ValueError("Compress mode value is required.") 395 if self.postgresql.all: 396 if self.postgresql.databases is not None and self.postgresql.databases != []: 397 raise ValueError("Databases cannot be specified if 'all' flag is set.") 398 else: 399 if self.postgresql.databases is None or len(self.postgresql.databases) < 1: 400 raise ValueError("At least one PostgreSQL database must be indicated if 'all' flag is not set.")
401
402 - def addConfig(self, xmlDom, parentNode):
403 """ 404 Adds a <postgresql> configuration section as the next child of a parent. 405 406 Third parties should use this function to write configuration related to 407 this extension. 408 409 We add the following fields to the document:: 410 411 user //cb_config/postgresql/user 412 compressMode //cb_config/postgresql/compress_mode 413 all //cb_config/postgresql/all 414 415 We also add groups of the following items, one list element per 416 item:: 417 418 database //cb_config/postgresql/database 419 420 @param xmlDom: DOM tree as from C{impl.createDocument()}. 421 @param parentNode: Parent that the section should be appended to. 422 """ 423 if self.postgresql is not None: 424 sectionNode = addContainerNode(xmlDom, parentNode, "postgresql") 425 addStringNode(xmlDom, sectionNode, "user", self.postgresql.user) 426 addStringNode(xmlDom, sectionNode, "compress_mode", self.postgresql.compressMode) 427 addBooleanNode(xmlDom, sectionNode, "all", self.postgresql.all) 428 if self.postgresql.databases is not None: 429 for database in self.postgresql.databases: 430 addStringNode(xmlDom, sectionNode, "database", database)
431
432 - def _parseXmlData(self, xmlData):
433 """ 434 Internal method to parse an XML string into the object. 435 436 This method parses the XML document into a DOM tree (C{xmlDom}) and then 437 calls a static method to parse the postgresql configuration section. 438 439 @param xmlData: XML data to be parsed 440 @type xmlData: String data 441 442 @raise ValueError: If the XML cannot be successfully parsed. 443 """ 444 (xmlDom, parentNode) = createInputDom(xmlData) 445 self._postgresql = LocalConfig._parsePostgresql(parentNode)
446
447 - def _parsePostgresql(parent):
448 """ 449 Parses a postgresql configuration section. 450 451 We read the following fields:: 452 453 user //cb_config/postgresql/user 454 compressMode //cb_config/postgresql/compress_mode 455 all //cb_config/postgresql/all 456 457 We also read groups of the following item, one list element per 458 item:: 459 460 databases //cb_config/postgresql/database 461 462 @param parent: Parent node to search beneath. 463 464 @return: C{PostgresqlConfig} object or C{None} if the section does not exist. 465 @raise ValueError: If some filled-in value is invalid. 466 """ 467 postgresql = None 468 section = readFirstChild(parent, "postgresql") 469 if section is not None: 470 postgresql = PostgresqlConfig() 471 postgresql.user = readString(section, "user") 472 postgresql.compressMode = readString(section, "compress_mode") 473 postgresql.all = readBoolean(section, "all") 474 postgresql.databases = readStringList(section, "database") 475 return postgresql
476 _parsePostgresql = staticmethod(_parsePostgresql)
477 478 479 ######################################################################## 480 # Public functions 481 ######################################################################## 482 483 ########################### 484 # executeAction() function 485 ########################### 486
487 -def executeAction(configPath, options, config):
488 """ 489 Executes the PostgreSQL backup action. 490 491 @param configPath: Path to configuration file on disk. 492 @type configPath: String representing a path on disk. 493 494 @param options: Program command-line options. 495 @type options: Options object. 496 497 @param config: Program configuration. 498 @type config: Config object. 499 500 @raise ValueError: Under many generic error conditions 501 @raise IOError: If a backup could not be written for some reason. 502 """ 503 logger.debug("Executing PostgreSQL extended action.") 504 if config.options is None or config.collect is None: 505 raise ValueError("Cedar Backup configuration is not properly filled in.") 506 local = LocalConfig(xmlPath=configPath) 507 if local.postgresql.all: 508 logger.info("Backing up all databases.") 509 _backupDatabase(config.collect.targetDir, local.postgresql.compressMode, local.postgresql.user, 510 config.options.backupUser, config.options.backupGroup, None) 511 if local.postgresql.databases is not None and local.postgresql.databases != []: 512 logger.debug("Backing up %d individual databases." % len(local.postgresql.databases)) 513 for database in local.postgresql.databases: 514 logger.info("Backing up database [%s]." % database) 515 _backupDatabase(config.collect.targetDir, local.postgresql.compressMode, local.postgresql.user, 516 config.options.backupUser, config.options.backupGroup, database) 517 logger.info("Executed the PostgreSQL extended action successfully.")
518
519 -def _backupDatabase(targetDir, compressMode, user, backupUser, backupGroup, database=None):
520 """ 521 Backs up an individual PostgreSQL database, or all databases. 522 523 This internal method wraps the public method and adds some functionality, 524 like figuring out a filename, etc. 525 526 @param targetDir: Directory into which backups should be written. 527 @param compressMode: Compress mode to be used for backed-up files. 528 @param user: User to use for connecting to the database. 529 @param backupUser: User to own resulting file. 530 @param backupGroup: Group to own resulting file. 531 @param database: Name of database, or C{None} for all databases. 532 533 @return: Name of the generated backup file. 534 535 @raise ValueError: If some value is missing or invalid. 536 @raise IOError: If there is a problem executing the PostgreSQL dump. 537 """ 538 (outputFile, filename) = _getOutputFile(targetDir, database, compressMode) 539 try: 540 backupDatabase(user, outputFile, database) 541 finally: 542 outputFile.close() 543 if not os.path.exists(filename): 544 raise IOError("Dump file [%s] does not seem to exist after backup completed." % filename) 545 changeOwnership(filename, backupUser, backupGroup)
546
547 -def _getOutputFile(targetDir, database, compressMode):
548 """ 549 Opens the output file used for saving the PostgreSQL dump. 550 551 The filename is either C{"postgresqldump.txt"} or 552 C{"postgresqldump-<database>.txt"}. The C{".gz"} or C{".bz2"} extension is 553 added if C{compress} is C{True}. 554 555 @param targetDir: Target directory to write file in. 556 @param database: Name of the database (if any) 557 @param compressMode: Compress mode to be used for backed-up files. 558 559 @return: Tuple of (Output file object, filename) 560 """ 561 if database is None: 562 filename = os.path.join(targetDir, "postgresqldump.txt") 563 else: 564 filename = os.path.join(targetDir, "postgresqldump-%s.txt" % database) 565 if compressMode == "gzip": 566 filename = "%s.gz" % filename 567 outputFile = GzipFile(filename, "w") 568 elif compressMode == "bzip2": 569 filename = "%s.bz2" % filename 570 outputFile = BZ2File(filename, "w") 571 else: 572 outputFile = open(filename, "w") 573 logger.debug("PostgreSQL dump file will be [%s]." % filename) 574 return (outputFile, filename)
575 576 577 ############################ 578 # backupDatabase() function 579 ############################ 580
581 -def backupDatabase(user, backupFile, database=None):
582 """ 583 Backs up an individual PostgreSQL database, or all databases. 584 585 This function backs up either a named local PostgreSQL database or all local 586 PostgreSQL databases, using the passed in user for connectivity. 587 This is I{always} a full backup. There is no facility for incremental 588 backups. 589 590 The backup data will be written into the passed-in back file. Normally, 591 this would be an object as returned from C{open()}, but it is possible to 592 use something like a C{GzipFile} to write compressed output. The caller is 593 responsible for closing the passed-in backup file. 594 595 @note: Typically, you would use the C{root} user to back up all databases. 596 597 @param user: User to use for connecting to the database. 598 @type user: String representing PostgreSQL username. 599 600 @param backupFile: File use for writing backup. 601 @type backupFile: Python file object as from C{open()} or C{file()}. 602 603 @param database: Name of the database to be backed up. 604 @type database: String representing database name, or C{None} for all databases. 605 606 @raise ValueError: If some value is missing or invalid. 607 @raise IOError: If there is a problem executing the PostgreSQL dump. 608 """ 609 args = [] 610 if user is not None: 611 args.append('-U') 612 args.append(user) 613 614 if database is None: 615 command = resolveCommand(POSTGRESQLDUMPALL_COMMAND) 616 else: 617 command = resolveCommand(POSTGRESQLDUMP_COMMAND) 618 args.append(database) 619 620 result = executeCommand(command, args, returnOutput=False, ignoreStderr=True, doNotLog=True, outputFile=backupFile)[0] 621 if result != 0: 622 if database is None: 623 raise IOError("Error [%d] executing PostgreSQL database dump for all databases." % result) 624 else: 625 raise IOError("Error [%d] executing PostgreSQL database dump for database [%s]." % (result, database))
626