1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39 """
40 Provides backup peer-related objects and utility functions.
41
42 @sort: LocalPeer, Remote Peer
43
44 @var DEF_COLLECT_INDICATOR: Name of the default collect indicator file.
45 @var DEF_STAGE_INDICATOR: Name of the default stage indicator file.
46
47 @author: Kenneth J. Pronovici <pronovic@ieee.org>
48 """
49
50
51
52
53
54
55
56 import os
57 import logging
58 import shutil
59 import sets
60 import re
61
62
63 from CedarBackup2.filesystem import FilesystemList
64 from CedarBackup2.util import resolveCommand, executeCommand
65 from CedarBackup2.util import splitCommandLine, encodePath
66
67
68
69
70
71
72 logger = logging.getLogger("CedarBackup2.log.peer")
73
74 DEF_RCP_COMMAND = [ "/usr/bin/scp", "-B", "-q", "-C" ]
75 DEF_RSH_COMMAND = [ "/usr/bin/ssh", ]
76 DEF_CBACK_COMMAND = "/usr/bin/cback"
77
78 DEF_COLLECT_INDICATOR = "cback.collect"
79 DEF_STAGE_INDICATOR = "cback.stage"
80
81 SU_COMMAND = [ "su" ]
82
83
84
85
86
87
89
90
91
92
93
94 """
95 Backup peer representing a local peer in a backup pool.
96
97 This is a class representing a local (non-network) peer in a backup pool.
98 Local peers are backed up by simple filesystem copy operations. A local
99 peer has associated with it a name (typically, but not necessarily, a
100 hostname) and a collect directory.
101
102 The public methods other than the constructor are part of a "backup peer"
103 interface shared with the C{RemotePeer} class.
104
105 @sort: __init__, stagePeer, checkCollectIndicator, writeStageIndicator,
106 _copyLocalDir, _copyLocalFile, name, collectDir
107 """
108
109
110
111
112
114 """
115 Initializes a local backup peer.
116
117 Note that the collect directory must be an absolute path, but does not
118 have to exist when the object is instantiated. We do a lazy validation
119 on this value since we could (potentially) be creating peer objects
120 before an ongoing backup completed.
121
122 @param name: Name of the backup peer
123 @type name: String, typically a hostname
124
125 @param collectDir: Path to the peer's collect directory
126 @type collectDir: String representing an absolute local path on disk
127
128 @raise ValueError: If the name is empty.
129 @raise ValueError: If collect directory is not an absolute path.
130 """
131 self._name = None
132 self._collectDir = None
133 self.name = name
134 self.collectDir = collectDir
135
136
137
138
139
140
142 """
143 Property target used to set the peer name.
144 The value must be a non-empty string and cannot be C{None}.
145 @raise ValueError: If the value is an empty string or C{None}.
146 """
147 if value is None or len(value) < 1:
148 raise ValueError("Peer name must be a non-empty string.")
149 self._name = value
150
152 """
153 Property target used to get the peer name.
154 """
155 return self._name
156
158 """
159 Property target used to set the collect directory.
160 The value must be an absolute path and cannot be C{None}.
161 It does not have to exist on disk at the time of assignment.
162 @raise ValueError: If the value is C{None} or is not an absolute path.
163 @raise ValueError: If a path cannot be encoded properly.
164 """
165 if value is None or not os.path.isabs(value):
166 raise ValueError("Collect directory must be an absolute path.")
167 self._collectDir = encodePath(value)
168
170 """
171 Property target used to get the collect directory.
172 """
173 return self._collectDir
174
175 name = property(_getName, _setName, None, "Name of the peer.")
176 collectDir = property(_getCollectDir, _setCollectDir, None, "Path to the peer's collect directory (an absolute local path).")
177
178
179
180
181
182
183 - def stagePeer(self, targetDir, ownership=None, permissions=None):
184 """
185 Stages data from the peer into the indicated local target directory.
186
187 The collect and target directories must both already exist before this
188 method is called. If passed in, ownership and permissions will be
189 applied to the files that are copied.
190
191 @note: The caller is responsible for checking that the indicator exists,
192 if they care. This function only stages the files within the directory.
193
194 @note: If you have user/group as strings, call the L{util.getUidGid} function
195 to get the associated uid/gid as an ownership tuple.
196
197 @param targetDir: Target directory to write data into
198 @type targetDir: String representing a directory on disk
199
200 @param ownership: Owner and group that the staged files should have
201 @type ownership: Tuple of numeric ids C{(uid, gid)}
202
203 @param permissions: Permissions that the staged files should have
204 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}).
205
206 @return: Number of files copied from the source directory to the target directory.
207
208 @raise ValueError: If collect directory is not a directory or does not exist
209 @raise ValueError: If target directory is not a directory, does not exist or is not absolute.
210 @raise ValueError: If a path cannot be encoded properly.
211 @raise IOError: If there were no files to stage (i.e. the directory was empty)
212 @raise IOError: If there is an IO error copying a file.
213 @raise OSError: If there is an OS error copying or changing permissions on a file
214 """
215 targetDir = encodePath(targetDir)
216 if not os.path.isabs(targetDir):
217 logger.debug("Target directory [%s] not an absolute path." % targetDir)
218 raise ValueError("Target directory must be an absolute path.")
219 if not os.path.exists(self.collectDir) or not os.path.isdir(self.collectDir):
220 logger.debug("Collect directory [%s] is not a directory or does not exist on disk." % self.collectDir)
221 raise ValueError("Collect directory is not a directory or does not exist on disk.")
222 if not os.path.exists(targetDir) or not os.path.isdir(targetDir):
223 logger.debug("Target directory [%s] is not a directory or does not exist on disk." % targetDir)
224 raise ValueError("Target directory is not a directory or does not exist on disk.")
225 count = LocalPeer._copyLocalDir(self.collectDir, targetDir, ownership, permissions)
226 if count == 0:
227 raise IOError("Did not copy any files from local peer.")
228 return count
229
231 """
232 Checks the collect indicator in the peer's staging directory.
233
234 When a peer has completed collecting its backup files, it will write an
235 empty indicator file into its collect directory. This method checks to
236 see whether that indicator has been written. We're "stupid" here - if
237 the collect directory doesn't exist, you'll naturally get back C{False}.
238
239 If you need to, you can override the name of the collect indicator file
240 by passing in a different name.
241
242 @param collectIndicator: Name of the collect indicator file to check
243 @type collectIndicator: String representing name of a file in the collect directory
244
245 @return: Boolean true/false depending on whether the indicator exists.
246 @raise ValueError: If a path cannot be encoded properly.
247 """
248 collectIndicator = encodePath(collectIndicator)
249 if collectIndicator is None:
250 return os.path.exists(os.path.join(self.collectDir, DEF_COLLECT_INDICATOR))
251 else:
252 return os.path.exists(os.path.join(self.collectDir, collectIndicator))
253
255 """
256 Writes the stage indicator in the peer's staging directory.
257
258 When the master has completed collecting its backup files, it will write
259 an empty indicator file into the peer's collect directory. The presence
260 of this file implies that the staging process is complete.
261
262 If you need to, you can override the name of the stage indicator file by
263 passing in a different name.
264
265 @note: If you have user/group as strings, call the L{util.getUidGid}
266 function to get the associated uid/gid as an ownership tuple.
267
268 @param stageIndicator: Name of the indicator file to write
269 @type stageIndicator: String representing name of a file in the collect directory
270
271 @param ownership: Owner and group that the indicator file should have
272 @type ownership: Tuple of numeric ids C{(uid, gid)}
273
274 @param permissions: Permissions that the indicator file should have
275 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}).
276
277 @raise ValueError: If collect directory is not a directory or does not exist
278 @raise ValueError: If a path cannot be encoded properly.
279 @raise IOError: If there is an IO error creating the file.
280 @raise OSError: If there is an OS error creating or changing permissions on the file
281 """
282 stageIndicator = encodePath(stageIndicator)
283 if not os.path.exists(self.collectDir) or not os.path.isdir(self.collectDir):
284 logger.debug("Collect directory [%s] is not a directory or does not exist on disk." % self.collectDir)
285 raise ValueError("Collect directory is not a directory or does not exist on disk.")
286 if stageIndicator is None:
287 fileName = os.path.join(self.collectDir, DEF_STAGE_INDICATOR)
288 else:
289 fileName = os.path.join(self.collectDir, stageIndicator)
290 LocalPeer._copyLocalFile(None, fileName, ownership, permissions)
291
292
293
294
295
296
297 - def _copyLocalDir(sourceDir, targetDir, ownership=None, permissions=None):
298 """
299 Copies files from the source directory to the target directory.
300
301 This function is not recursive. Only the files in the directory will be
302 copied. Ownership and permissions will be left at their default values
303 if new values are not specified. The source and target directories are
304 allowed to be soft links to a directory, but besides that soft links are
305 ignored.
306
307 @note: If you have user/group as strings, call the L{util.getUidGid}
308 function to get the associated uid/gid as an ownership tuple.
309
310 @param sourceDir: Source directory
311 @type sourceDir: String representing a directory on disk
312
313 @param targetDir: Target directory
314 @type targetDir: String representing a directory on disk
315
316 @param ownership: Owner and group that the copied files should have
317 @type ownership: Tuple of numeric ids C{(uid, gid)}
318
319 @param permissions: Permissions that the staged files should have
320 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}).
321
322 @return: Number of files copied from the source directory to the target directory.
323
324 @raise ValueError: If source or target is not a directory or does not exist.
325 @raise ValueError: If a path cannot be encoded properly.
326 @raise IOError: If there is an IO error copying the files.
327 @raise OSError: If there is an OS error copying or changing permissions on a files
328 """
329 filesCopied = 0
330 sourceDir = encodePath(sourceDir)
331 targetDir = encodePath(targetDir)
332 for fileName in os.listdir(sourceDir):
333 sourceFile = os.path.join(sourceDir, fileName)
334 targetFile = os.path.join(targetDir, fileName)
335 LocalPeer._copyLocalFile(sourceFile, targetFile, ownership, permissions)
336 filesCopied += 1
337 return filesCopied
338 _copyLocalDir = staticmethod(_copyLocalDir)
339
340 - def _copyLocalFile(sourceFile=None, targetFile=None, ownership=None, permissions=None, overwrite=True):
341 """
342 Copies a source file to a target file.
343
344 If the source file is C{None} then the target file will be created or
345 overwritten as an empty file. If the target file is C{None}, this method
346 is a no-op. Attempting to copy a soft link or a directory will result in
347 an exception.
348
349 @note: If you have user/group as strings, call the L{util.getUidGid}
350 function to get the associated uid/gid as an ownership tuple.
351
352 @note: We will not overwrite a target file that exists when this method
353 is invoked. If the target already exists, we'll raise an exception.
354
355 @param sourceFile: Source file to copy
356 @type sourceFile: String representing a file on disk, as an absolute path
357
358 @param targetFile: Target file to create
359 @type targetFile: String representing a file on disk, as an absolute path
360
361 @param ownership: Owner and group that the copied should have
362 @type ownership: Tuple of numeric ids C{(uid, gid)}
363
364 @param permissions: Permissions that the staged files should have
365 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}).
366
367 @param overwrite: Indicates whether it's OK to overwrite the target file.
368 @type overwrite: Boolean true/false.
369
370 @raise ValueError: If the passed-in source file is not a regular file.
371 @raise ValueError: If a path cannot be encoded properly.
372 @raise IOError: If the target file already exists.
373 @raise IOError: If there is an IO error copying the file
374 @raise OSError: If there is an OS error copying or changing permissions on a file
375 """
376 targetFile = encodePath(targetFile)
377 sourceFile = encodePath(sourceFile)
378 if targetFile is None:
379 return
380 if not overwrite:
381 if os.path.exists(targetFile):
382 raise IOError("Target file [%s] already exists." % targetFile)
383 if sourceFile is None:
384 open(targetFile, "w").write("")
385 else:
386 if os.path.isfile(sourceFile) and not os.path.islink(sourceFile):
387 shutil.copy(sourceFile, targetFile)
388 else:
389 logger.debug("Source [%s] is not a regular file." % sourceFile)
390 raise ValueError("Source is not a regular file.")
391 if ownership is not None:
392 os.chown(targetFile, ownership[0], ownership[1])
393 if permissions is not None:
394 os.chmod(targetFile, permissions)
395 _copyLocalFile = staticmethod(_copyLocalFile)
396
397
398
399
400
401
403
404
405
406
407
408 """
409 Backup peer representing a remote peer in a backup pool.
410
411 This is a class representing a remote (networked) peer in a backup pool.
412 Remote peers are backed up using an rcp-compatible copy command. A remote
413 peer has associated with it a name (which must be a valid hostname), a
414 collect directory, a working directory and a copy method (an rcp-compatible
415 command).
416
417 You can also set an optional local user value. This username will be used
418 as the local user for any remote copies that are required. It can only be
419 used if the root user is executing the backup. The root user will C{su} to
420 the local user and execute the remote copies as that user.
421
422 The copy method is associated with the peer and not with the actual request
423 to copy, because we can envision that each remote host might have a
424 different connect method.
425
426 The public methods other than the constructor are part of a "backup peer"
427 interface shared with the C{LocalPeer} class.
428
429 @sort: __init__, stagePeer, checkCollectIndicator, writeStageIndicator,
430 executeRemoteCommand, executeManagedAction, _getDirContents,
431 _copyRemoteDir, _copyRemoteFile, _pushLocalFile, name, collectDir,
432 remoteUser, rcpCommand, rshCommand, cbackCommand
433 """
434
435
436
437
438
439 - def __init__(self, name=None, collectDir=None, workingDir=None, remoteUser=None,
440 rcpCommand=None, localUser=None, rshCommand=None, cbackCommand=None):
441 """
442 Initializes a remote backup peer.
443
444 @note: If provided, each command will eventually be parsed into a list of
445 strings suitable for passing to C{util.executeCommand} in order to avoid
446 security holes related to shell interpolation. This parsing will be
447 done by the L{util.splitCommandLine} function. See the documentation for
448 that function for some important notes about its limitations.
449
450 @param name: Name of the backup peer
451 @type name: String, must be a valid DNS hostname
452
453 @param collectDir: Path to the peer's collect directory
454 @type collectDir: String representing an absolute path on the remote peer
455
456 @param workingDir: Working directory that can be used to create temporary files, etc.
457 @type workingDir: String representing an absolute path on the current host.
458
459 @param remoteUser: Name of the Cedar Backup user on the remote peer
460 @type remoteUser: String representing a username, valid via remote shell to the peer
461
462 @param localUser: Name of the Cedar Backup user on the current host
463 @type localUser: String representing a username, valid on the current host
464
465 @param rcpCommand: An rcp-compatible copy command to use for copying files from the peer
466 @type rcpCommand: String representing a system command including required arguments
467
468 @param rshCommand: An rsh-compatible copy command to use for remote shells to the peer
469 @type rshCommand: String representing a system command including required arguments
470
471 @param cbackCommand: A chack-compatible command to use for executing managed actions
472 @type cbackCommand: String representing a system command including required arguments
473
474 @raise ValueError: If collect directory is not an absolute path
475 """
476 self._name = None
477 self._collectDir = None
478 self._workingDir = None
479 self._remoteUser = None
480 self._localUser = None
481 self._rcpCommand = None
482 self._rcpCommandList = None
483 self._rshCommand = None
484 self._rshCommandList = None
485 self._cbackCommand = None
486 self.name = name
487 self.collectDir = collectDir
488 self.workingDir = workingDir
489 self.remoteUser = remoteUser
490 self.localUser = localUser
491 self.rcpCommand = rcpCommand
492 self.rshCommand = rshCommand
493 self.cbackCommand = cbackCommand
494
495
496
497
498
499
501 """
502 Property target used to set the peer name.
503 The value must be a non-empty string and cannot be C{None}.
504 @raise ValueError: If the value is an empty string or C{None}.
505 """
506 if value is None or len(value) < 1:
507 raise ValueError("Peer name must be a non-empty string.")
508 self._name = value
509
511 """
512 Property target used to get the peer name.
513 """
514 return self._name
515
517 """
518 Property target used to set the collect directory.
519 The value must be an absolute path and cannot be C{None}.
520 It does not have to exist on disk at the time of assignment.
521 @raise ValueError: If the value is C{None} or is not an absolute path.
522 @raise ValueError: If the value cannot be encoded properly.
523 """
524 if value is not None:
525 if not os.path.isabs(value):
526 raise ValueError("Collect directory must be an absolute path.")
527 self._collectDir = encodePath(value)
528
530 """
531 Property target used to get the collect directory.
532 """
533 return self._collectDir
534
536 """
537 Property target used to set the working directory.
538 The value must be an absolute path and cannot be C{None}.
539 @raise ValueError: If the value is C{None} or is not an absolute path.
540 @raise ValueError: If the value cannot be encoded properly.
541 """
542 if value is not None:
543 if not os.path.isabs(value):
544 raise ValueError("Working directory must be an absolute path.")
545 self._workingDir = encodePath(value)
546
548 """
549 Property target used to get the working directory.
550 """
551 return self._workingDir
552
554 """
555 Property target used to set the remote user.
556 The value must be a non-empty string and cannot be C{None}.
557 @raise ValueError: If the value is an empty string or C{None}.
558 """
559 if value is None or len(value) < 1:
560 raise ValueError("Peer remote user must be a non-empty string.")
561 self._remoteUser = value
562
564 """
565 Property target used to get the remote user.
566 """
567 return self._remoteUser
568
570 """
571 Property target used to set the local user.
572 The value must be a non-empty string if it is not C{None}.
573 @raise ValueError: If the value is an empty string.
574 """
575 if value is not None:
576 if len(value) < 1:
577 raise ValueError("Peer local user must be a non-empty string.")
578 self._localUser = value
579
581 """
582 Property target used to get the local user.
583 """
584 return self._localUser
585
587 """
588 Property target to set the rcp command.
589
590 The value must be a non-empty string or C{None}. Its value is stored in
591 the two forms: "raw" as provided by the client, and "parsed" into a list
592 suitable for being passed to L{util.executeCommand} via
593 L{util.splitCommandLine}.
594
595 However, all the caller will ever see via the property is the actual
596 value they set (which includes seeing C{None}, even if we translate that
597 internally to C{DEF_RCP_COMMAND}). Internally, we should always use
598 C{self._rcpCommandList} if we want the actual command list.
599
600 @raise ValueError: If the value is an empty string.
601 """
602 if value is None:
603 self._rcpCommand = None
604 self._rcpCommandList = DEF_RCP_COMMAND
605 else:
606 if len(value) >= 1:
607 self._rcpCommand = value
608 self._rcpCommandList = splitCommandLine(self._rcpCommand)
609 else:
610 raise ValueError("The rcp command must be a non-empty string.")
611
613 """
614 Property target used to get the rcp command.
615 """
616 return self._rcpCommand
617
619 """
620 Property target to set the rsh command.
621
622 The value must be a non-empty string or C{None}. Its value is stored in
623 the two forms: "raw" as provided by the client, and "parsed" into a list
624 suitable for being passed to L{util.executeCommand} via
625 L{util.splitCommandLine}.
626
627 However, all the caller will ever see via the property is the actual
628 value they set (which includes seeing C{None}, even if we translate that
629 internally to C{DEF_RSH_COMMAND}). Internally, we should always use
630 C{self._rshCommandList} if we want the actual command list.
631
632 @raise ValueError: If the value is an empty string.
633 """
634 if value is None:
635 self._rshCommand = None
636 self._rshCommandList = DEF_RSH_COMMAND
637 else:
638 if len(value) >= 1:
639 self._rshCommand = value
640 self._rshCommandList = splitCommandLine(self._rshCommand)
641 else:
642 raise ValueError("The rsh command must be a non-empty string.")
643
645 """
646 Property target used to get the rsh command.
647 """
648 return self._rshCommand
649
651 """
652 Property target to set the cback command.
653
654 The value must be a non-empty string or C{None}. Unlike the other
655 command, this value is only stored in the "raw" form provided by the
656 client.
657
658 @raise ValueError: If the value is an empty string.
659 """
660 if value is None:
661 self._cbackCommand = None
662 else:
663 if len(value) >= 1:
664 self._cbackCommand = value
665 else:
666 raise ValueError("The cback command must be a non-empty string.")
667
669 """
670 Property target used to get the cback command.
671 """
672 return self._cbackCommand
673
674 name = property(_getName, _setName, None, "Name of the peer (a valid DNS hostname).")
675 collectDir = property(_getCollectDir, _setCollectDir, None, "Path to the peer's collect directory (an absolute local path).")
676 workingDir = property(_getWorkingDir, _setWorkingDir, None, "Path to the peer's working directory (an absolute local path).")
677 remoteUser = property(_getRemoteUser, _setRemoteUser, None, "Name of the Cedar Backup user on the remote peer.")
678 localUser = property(_getLocalUser, _setLocalUser, None, "Name of the Cedar Backup user on the current host.")
679 rcpCommand = property(_getRcpCommand, _setRcpCommand, None, "An rcp-compatible copy command to use for copying files.")
680 rshCommand = property(_getRshCommand, _setRshCommand, None, "An rsh-compatible command to use for remote shells to the peer.")
681 cbackCommand = property(_getCbackCommand, _setCbackCommand, None, "A chack-compatible command to use for executing managed actions.")
682
683
684
685
686
687
688 - def stagePeer(self, targetDir, ownership=None, permissions=None):
689 """
690 Stages data from the peer into the indicated local target directory.
691
692 The target directory must already exist before this method is called. If
693 passed in, ownership and permissions will be applied to the files that
694 are copied.
695
696 @note: The returned count of copied files might be inaccurate if some of
697 the copied files already existed in the staging directory prior to the
698 copy taking place. We don't clear the staging directory first, because
699 some extension might also be using it.
700
701 @note: If you have user/group as strings, call the L{util.getUidGid} function
702 to get the associated uid/gid as an ownership tuple.
703
704 @note: Unlike the local peer version of this method, an I/O error might
705 or might not be raised if the directory is empty. Since we're using a
706 remote copy method, we just don't have the fine-grained control over our
707 exceptions that's available when we can look directly at the filesystem,
708 and we can't control whether the remote copy method thinks an empty
709 directory is an error.
710
711 @param targetDir: Target directory to write data into
712 @type targetDir: String representing a directory on disk
713
714 @param ownership: Owner and group that the staged files should have
715 @type ownership: Tuple of numeric ids C{(uid, gid)}
716
717 @param permissions: Permissions that the staged files should have
718 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}).
719
720 @return: Number of files copied from the source directory to the target directory.
721
722 @raise ValueError: If target directory is not a directory, does not exist or is not absolute.
723 @raise ValueError: If a path cannot be encoded properly.
724 @raise IOError: If there were no files to stage (i.e. the directory was empty)
725 @raise IOError: If there is an IO error copying a file.
726 @raise OSError: If there is an OS error copying or changing permissions on a file
727 """
728 targetDir = encodePath(targetDir)
729 if not os.path.isabs(targetDir):
730 logger.debug("Target directory [%s] not an absolute path." % targetDir)
731 raise ValueError("Target directory must be an absolute path.")
732 if not os.path.exists(targetDir) or not os.path.isdir(targetDir):
733 logger.debug("Target directory [%s] is not a directory or does not exist on disk." % targetDir)
734 raise ValueError("Target directory is not a directory or does not exist on disk.")
735 count = RemotePeer._copyRemoteDir(self.remoteUser, self.localUser, self.name,
736 self._rcpCommand, self._rcpCommandList,
737 self.collectDir, targetDir,
738 ownership, permissions)
739 if count == 0:
740 raise IOError("Did not copy any files from local peer.")
741 return count
742
744 """
745 Checks the collect indicator in the peer's staging directory.
746
747 When a peer has completed collecting its backup files, it will write an
748 empty indicator file into its collect directory. This method checks to
749 see whether that indicator has been written. If the remote copy command
750 fails, we return C{False} as if the file weren't there.
751
752 If you need to, you can override the name of the collect indicator file
753 by passing in a different name.
754
755 @note: Apparently, we can't count on all rcp-compatible implementations
756 to return sensible errors for some error conditions. As an example, the
757 C{scp} command in Debian 'woody' returns a zero (normal) status even when
758 it can't find a host or if the login or path is invalid. Because of
759 this, the implementation of this method is rather convoluted.
760
761 @param collectIndicator: Name of the collect indicator file to check
762 @type collectIndicator: String representing name of a file in the collect directory
763
764 @return: Boolean true/false depending on whether the indicator exists.
765 @raise ValueError: If a path cannot be encoded properly.
766 """
767 try:
768 if collectIndicator is None:
769 sourceFile = os.path.join(self.collectDir, DEF_COLLECT_INDICATOR)
770 targetFile = os.path.join(self.workingDir, DEF_COLLECT_INDICATOR)
771 else:
772 collectIndicator = encodePath(collectIndicator)
773 sourceFile = os.path.join(self.collectDir, collectIndicator)
774 targetFile = os.path.join(self.workingDir, collectIndicator)
775 logger.debug("Fetch remote [%s] into [%s]." % (sourceFile, targetFile))
776 if os.path.exists(targetFile):
777 try:
778 os.remove(targetFile)
779 except:
780 raise Exception("Error: collect indicator [%s] already exists!" % targetFile)
781 try:
782 RemotePeer._copyRemoteFile(self.remoteUser, self.localUser, self.name,
783 self._rcpCommand, self._rcpCommandList,
784 sourceFile, targetFile,
785 overwrite=False)
786 if os.path.exists(targetFile):
787 return True
788 else:
789 return False
790 except Exception, e:
791 logger.info("Failed looking for collect indicator: %s" % e)
792 return False
793 finally:
794 if os.path.exists(targetFile):
795 try:
796 os.remove(targetFile)
797 except: pass
798
800 """
801 Writes the stage indicator in the peer's staging directory.
802
803 When the master has completed collecting its backup files, it will write
804 an empty indicator file into the peer's collect directory. The presence
805 of this file implies that the staging process is complete.
806
807 If you need to, you can override the name of the stage indicator file by
808 passing in a different name.
809
810 @note: If you have user/group as strings, call the L{util.getUidGid} function
811 to get the associated uid/gid as an ownership tuple.
812
813 @param stageIndicator: Name of the indicator file to write
814 @type stageIndicator: String representing name of a file in the collect directory
815
816 @raise ValueError: If a path cannot be encoded properly.
817 @raise IOError: If there is an IO error creating the file.
818 @raise OSError: If there is an OS error creating or changing permissions on the file
819 """
820 stageIndicator = encodePath(stageIndicator)
821 if stageIndicator is None:
822 sourceFile = os.path.join(self.workingDir, DEF_STAGE_INDICATOR)
823 targetFile = os.path.join(self.collectDir, DEF_STAGE_INDICATOR)
824 else:
825 sourceFile = os.path.join(self.workingDir, DEF_STAGE_INDICATOR)
826 targetFile = os.path.join(self.collectDir, stageIndicator)
827 try:
828 if not os.path.exists(sourceFile):
829 open(sourceFile, "w").write("")
830 RemotePeer._pushLocalFile(self.remoteUser, self.localUser, self.name,
831 self._rcpCommand, self._rcpCommandList,
832 sourceFile, targetFile)
833 finally:
834 if os.path.exists(sourceFile):
835 try:
836 os.remove(sourceFile)
837 except: pass
838
840 """
841 Executes a command on the peer via remote shell.
842
843 @param command: Command to execute
844 @type command: String command-line suitable for use with rsh.
845
846 @raise IOError: If there is an error executing the command on the remote peer.
847 """
848 RemotePeer._executeRemoteCommand(self.remoteUser, self.localUser,
849 self.name, self._rshCommand,
850 self._rshCommandList, command)
851
853 """
854 Executes a managed action on this peer.
855
856 @param action: Name of the action to execute.
857 @param fullBackup: Whether a full backup should be executed.
858
859 @raise IOError: If there is an error executing the action on the remote peer.
860 """
861 try:
862 command = RemotePeer._buildCbackCommand(self.cbackCommand, action, fullBackup)
863 self.executeRemoteCommand(command)
864 except IOError, e:
865 logger.info(e)
866 raise IOError("Failed to execute action [%s] on managed client [%s]." % (action, self.name))
867
868
869
870
871
872
873 - def _getDirContents(path):
874 """
875 Returns the contents of a directory in terms of a Set.
876
877 The directory's contents are read as a L{FilesystemList} containing only
878 files, and then the list is converted into a C{sets.Set} object for later
879 use.
880
881 @param path: Directory path to get contents for
882 @type path: String representing a path on disk
883
884 @return: Set of files in the directory
885 @raise ValueError: If path is not a directory or does not exist.
886 """
887 contents = FilesystemList()
888 contents.excludeDirs = True
889 contents.excludeLinks = True
890 contents.addDirContents(path)
891 return sets.Set(contents)
892 _getDirContents = staticmethod(_getDirContents)
893
894 - def _copyRemoteDir(remoteUser, localUser, remoteHost, rcpCommand, rcpCommandList,
895 sourceDir, targetDir, ownership=None, permissions=None):
896 """
897 Copies files from the source directory to the target directory.
898
899 This function is not recursive. Only the files in the directory will be
900 copied. Ownership and permissions will be left at their default values
901 if new values are not specified. Behavior when copying soft links from
902 the collect directory is dependent on the behavior of the specified rcp
903 command.
904
905 @note: The returned count of copied files might be inaccurate if some of
906 the copied files already existed in the staging directory prior to the
907 copy taking place. We don't clear the staging directory first, because
908 some extension might also be using it.
909
910 @note: If you have user/group as strings, call the L{util.getUidGid} function
911 to get the associated uid/gid as an ownership tuple.
912
913 @note: We don't have a good way of knowing exactly what files we copied
914 down from the remote peer, unless we want to parse the output of the rcp
915 command (ugh). We could change permissions on everything in the target
916 directory, but that's kind of ugly too. Instead, we use Python's set
917 functionality to figure out what files were added while we executed the
918 rcp command. This isn't perfect - for instance, it's not correct if
919 someone else is messing with the directory at the same time we're doing
920 the remote copy - but it's about as good as we're going to get.
921
922 @note: Apparently, we can't count on all rcp-compatible implementations
923 to return sensible errors for some error conditions. As an example, the
924 C{scp} command in Debian 'woody' returns a zero (normal) status even
925 when it can't find a host or if the login or path is invalid. We try
926 to work around this by issuing C{IOError} if we don't copy any files from
927 the remote host.
928
929 @param remoteUser: Name of the Cedar Backup user on the remote peer
930 @type remoteUser: String representing a username, valid via the copy command
931
932 @param localUser: Name of the Cedar Backup user on the current host
933 @type localUser: String representing a username, valid on the current host
934
935 @param remoteHost: Hostname of the remote peer
936 @type remoteHost: String representing a hostname, accessible via the copy command
937
938 @param rcpCommand: An rcp-compatible copy command to use for copying files from the peer
939 @type rcpCommand: String representing a system command including required arguments
940
941 @param rcpCommandList: An rcp-compatible copy command to use for copying files
942 @type rcpCommandList: Command as a list to be passed to L{util.executeCommand}
943
944 @param sourceDir: Source directory
945 @type sourceDir: String representing a directory on disk
946
947 @param targetDir: Target directory
948 @type targetDir: String representing a directory on disk
949
950 @param ownership: Owner and group that the copied files should have
951 @type ownership: Tuple of numeric ids C{(uid, gid)}
952
953 @param permissions: Permissions that the staged files should have
954 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}).
955
956 @return: Number of files copied from the source directory to the target directory.
957
958 @raise ValueError: If source or target is not a directory or does not exist.
959 @raise IOError: If there is an IO error copying the files.
960 """
961 beforeSet = RemotePeer._getDirContents(targetDir)
962 if localUser is not None:
963 try:
964 if os.getuid() != 0:
965 raise IOError("Only root can remote copy as another user.")
966 except AttributeError: pass
967 actualCommand = "%s %s@%s:%s/* %s" % (rcpCommand, remoteUser, remoteHost, sourceDir, targetDir)
968 command = resolveCommand(SU_COMMAND)
969 result = executeCommand(command, [localUser, "-c", actualCommand])[0]
970 if result != 0:
971 raise IOError("Error (%d) copying files from remote host as local user [%s]." % (result, localUser))
972 else:
973 copySource = "%s@%s:%s/*" % (remoteUser, remoteHost, sourceDir)
974 command = resolveCommand(rcpCommandList)
975 result = executeCommand(command, [copySource, targetDir])[0]
976 if result != 0:
977 raise IOError("Error (%d) copying files from remote host." % result)
978 afterSet = RemotePeer._getDirContents(targetDir)
979 if len(afterSet) == 0:
980 raise IOError("Did not copy any files from remote peer.")
981 differenceSet = afterSet.difference(beforeSet)
982 if len(differenceSet) == 0:
983 raise IOError("Apparently did not copy any new files from remote peer.")
984 for targetFile in differenceSet:
985 if ownership is not None:
986 os.chown(targetFile, ownership[0], ownership[1])
987 if permissions is not None:
988 os.chmod(targetFile, permissions)
989 return len(differenceSet)
990 _copyRemoteDir = staticmethod(_copyRemoteDir)
991
992 - def _copyRemoteFile(remoteUser, localUser, remoteHost,
993 rcpCommand, rcpCommandList,
994 sourceFile, targetFile, ownership=None,
995 permissions=None, overwrite=True):
996 """
997 Copies a remote source file to a target file.
998
999 @note: Internally, we have to go through and escape any spaces in the
1000 source path with double-backslash, otherwise things get screwed up. It
1001 doesn't seem to be required in the target path. I hope this is portable
1002 to various different rcp methods, but I guess it might not be (all I have
1003 to test with is OpenSSH).
1004
1005 @note: If you have user/group as strings, call the L{util.getUidGid} function
1006 to get the associated uid/gid as an ownership tuple.
1007
1008 @note: We will not overwrite a target file that exists when this method
1009 is invoked. If the target already exists, we'll raise an exception.
1010
1011 @note: Apparently, we can't count on all rcp-compatible implementations
1012 to return sensible errors for some error conditions. As an example, the
1013 C{scp} command in Debian 'woody' returns a zero (normal) status even when
1014 it can't find a host or if the login or path is invalid. We try to work
1015 around this by issuing C{IOError} the target file does not exist when
1016 we're done.
1017
1018 @param remoteUser: Name of the Cedar Backup user on the remote peer
1019 @type remoteUser: String representing a username, valid via the copy command
1020
1021 @param remoteHost: Hostname of the remote peer
1022 @type remoteHost: String representing a hostname, accessible via the copy command
1023
1024 @param localUser: Name of the Cedar Backup user on the current host
1025 @type localUser: String representing a username, valid on the current host
1026
1027 @param rcpCommand: An rcp-compatible copy command to use for copying files from the peer
1028 @type rcpCommand: String representing a system command including required arguments
1029
1030 @param rcpCommandList: An rcp-compatible copy command to use for copying files
1031 @type rcpCommandList: Command as a list to be passed to L{util.executeCommand}
1032
1033 @param sourceFile: Source file to copy
1034 @type sourceFile: String representing a file on disk, as an absolute path
1035
1036 @param targetFile: Target file to create
1037 @type targetFile: String representing a file on disk, as an absolute path
1038
1039 @param ownership: Owner and group that the copied should have
1040 @type ownership: Tuple of numeric ids C{(uid, gid)}
1041
1042 @param permissions: Permissions that the staged files should have
1043 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}).
1044
1045 @param overwrite: Indicates whether it's OK to overwrite the target file.
1046 @type overwrite: Boolean true/false.
1047
1048 @raise IOError: If the target file already exists.
1049 @raise IOError: If there is an IO error copying the file
1050 @raise OSError: If there is an OS error changing permissions on the file
1051 """
1052 if not overwrite:
1053 if os.path.exists(targetFile):
1054 raise IOError("Target file [%s] already exists." % targetFile)
1055 if localUser is not None:
1056 try:
1057 if os.getuid() != 0:
1058 raise IOError("Only root can remote copy as another user.")
1059 except AttributeError: pass
1060 actualCommand = "%s %s@%s:%s %s" % (rcpCommand, remoteUser, remoteHost, sourceFile.replace(" ", "\\ "), targetFile)
1061 command = resolveCommand(SU_COMMAND)
1062 result = executeCommand(command, [localUser, "-c", actualCommand])[0]
1063 if result != 0:
1064 raise IOError("Error (%d) copying [%s] from remote host as local user [%s]." % (result, sourceFile, localUser))
1065 else:
1066 copySource = "%s@%s:%s" % (remoteUser, remoteHost, sourceFile.replace(" ", "\\ "))
1067 command = resolveCommand(rcpCommandList)
1068 result = executeCommand(command, [copySource, targetFile])[0]
1069 if result != 0:
1070 raise IOError("Error (%d) copying [%s] from remote host." % (result, sourceFile))
1071 if not os.path.exists(targetFile):
1072 raise IOError("Apparently unable to copy file from remote host.")
1073 if ownership is not None:
1074 os.chown(targetFile, ownership[0], ownership[1])
1075 if permissions is not None:
1076 os.chmod(targetFile, permissions)
1077 _copyRemoteFile = staticmethod(_copyRemoteFile)
1078
1079 - def _pushLocalFile(remoteUser, localUser, remoteHost,
1080 rcpCommand, rcpCommandList,
1081 sourceFile, targetFile, overwrite=True):
1082 """
1083 Copies a local source file to a remote host.
1084
1085 @note: We will not overwrite a target file that exists when this method
1086 is invoked. If the target already exists, we'll raise an exception.
1087
1088 @note: Internally, we have to go through and escape any spaces in the
1089 source and target paths with double-backslash, otherwise things get
1090 screwed up. I hope this is portable to various different rcp methods,
1091 but I guess it might not be (all I have to test with is OpenSSH).
1092
1093 @note: If you have user/group as strings, call the L{util.getUidGid} function
1094 to get the associated uid/gid as an ownership tuple.
1095
1096 @param remoteUser: Name of the Cedar Backup user on the remote peer
1097 @type remoteUser: String representing a username, valid via the copy command
1098
1099 @param localUser: Name of the Cedar Backup user on the current host
1100 @type localUser: String representing a username, valid on the current host
1101
1102 @param remoteHost: Hostname of the remote peer
1103 @type remoteHost: String representing a hostname, accessible via the copy command
1104
1105 @param rcpCommand: An rcp-compatible copy command to use for copying files from the peer
1106 @type rcpCommand: String representing a system command including required arguments
1107
1108 @param rcpCommandList: An rcp-compatible copy command to use for copying files
1109 @type rcpCommandList: Command as a list to be passed to L{util.executeCommand}
1110
1111 @param sourceFile: Source file to copy
1112 @type sourceFile: String representing a file on disk, as an absolute path
1113
1114 @param targetFile: Target file to create
1115 @type targetFile: String representing a file on disk, as an absolute path
1116
1117 @param overwrite: Indicates whether it's OK to overwrite the target file.
1118 @type overwrite: Boolean true/false.
1119
1120 @raise IOError: If there is an IO error copying the file
1121 @raise OSError: If there is an OS error changing permissions on the file
1122 """
1123 if not overwrite:
1124 if os.path.exists(targetFile):
1125 raise IOError("Target file [%s] already exists." % targetFile)
1126 if localUser is not None:
1127 try:
1128 if os.getuid() != 0:
1129 raise IOError("Only root can remote copy as another user.")
1130 except AttributeError: pass
1131 actualCommand = '%s "%s" "%s@%s:%s"' % (rcpCommand, sourceFile, remoteUser, remoteHost, targetFile)
1132 command = resolveCommand(SU_COMMAND)
1133 result = executeCommand(command, [localUser, "-c", actualCommand])[0]
1134 if result != 0:
1135 raise IOError("Error (%d) copying [%s] to remote host as local user [%s]." % (result, sourceFile, localUser))
1136 else:
1137 copyTarget = "%s@%s:%s" % (remoteUser, remoteHost, targetFile.replace(" ", "\\ "))
1138 command = resolveCommand(rcpCommandList)
1139 result = executeCommand(command, [sourceFile.replace(" ", "\\ "), copyTarget])[0]
1140 if result != 0:
1141 raise IOError("Error (%d) copying [%s] to remote host." % (result, sourceFile))
1142 _pushLocalFile = staticmethod(_pushLocalFile)
1143
1144 - def _executeRemoteCommand(remoteUser, localUser, remoteHost, rshCommand, rshCommandList, remoteCommand):
1145 """
1146 Executes a command on the peer via remote shell.
1147
1148 @param remoteUser: Name of the Cedar Backup user on the remote peer
1149 @type remoteUser: String representing a username, valid on the remote host
1150
1151 @param localUser: Name of the Cedar Backup user on the current host
1152 @type localUser: String representing a username, valid on the current host
1153
1154 @param remoteHost: Hostname of the remote peer
1155 @type remoteHost: String representing a hostname, accessible via the copy command
1156
1157 @param rshCommand: An rsh-compatible copy command to use for remote shells to the peer
1158 @type rshCommand: String representing a system command including required arguments
1159
1160 @param rshCommandList: An rsh-compatible copy command to use for remote shells to the peer
1161 @type rshCommandList: Command as a list to be passed to L{util.executeCommand}
1162
1163 @param remoteCommand: The command to be executed on the remote host
1164 @type remoteCommand: String command-line, with no special shell characters ($, <, etc.)
1165
1166 @raise IOError: If there is an error executing the remote command
1167 """
1168 actualCommand = "%s %s@%s '%s'" % (rshCommand, remoteUser, remoteHost, remoteCommand)
1169 if localUser is not None:
1170 try:
1171 if os.getuid() != 0:
1172 raise IOError("Only root can remote shell as another user.")
1173 except AttributeError: pass
1174 command = resolveCommand(SU_COMMAND)
1175 result = executeCommand(command, [localUser, "-c", actualCommand])[0]
1176 if result != 0:
1177 raise IOError("Command failed [su -c %s \"%s\"]" % (localUser, actualCommand))
1178 else:
1179 command = resolveCommand(rshCommandList)
1180 result = executeCommand(command, ["%s@%s" % (remoteUser, remoteHost), "%s" % remoteCommand])[0]
1181 if result != 0:
1182 raise IOError("Command failed [%s]" % (actualCommand))
1183 _executeRemoteCommand = staticmethod(_executeRemoteCommand)
1184
1186 """
1187 Builds a Cedar Backup command line for the named action.
1188
1189 @note: If the cback command is None, then DEF_CBACK_COMMAND is used.
1190
1191 @param cbackCommand: cback command to execute, including required options
1192 @param action: Name of the action to execute.
1193 @param fullBackup: Whether a full backup should be executed.
1194
1195 @return: String suitable for passing to L{_executeRemoteCommand} as remoteCommand.
1196 @raise ValueError: If action is None.
1197 """
1198 if action is None:
1199 raise ValueError("Action cannot be None.")
1200 if cbackCommand is None:
1201 cbackCommand = DEF_CBACK_COMMAND
1202 if fullBackup:
1203 return "%s --full %s" % (cbackCommand, action)
1204 else:
1205 return "%s %s" % (cbackCommand, action)
1206 _buildCbackCommand = staticmethod(_buildCbackCommand)
1207