1
2
3
4 """
5 Version Control System functionality.
6 """
7
8 import re
9 import os
10 import sys
11 import glob
12 import tarfile
13 import tempfile
14 import commands
15
16 from moap.util import util, log
17
19 """
20 Returns a sorted list of VCS names that moap can work with.
21 """
22 moduleNames = util.getPackageModules('moap.vcs', ignore=['vcs', ])
23 modules = [util.namedModule('moap.vcs.%s' % s) for s in moduleNames]
24 names = [m.VCSClass.name for m in modules]
25 names.sort()
26 return names
27
29 """
30 Detect which version control system is being used in the source tree.
31
32 @returns: an instance of a subclass of L{VCS}, or None.
33 """
34 log.debug('vcs', 'detecting VCS in %s' % path)
35 if not path:
36 path = os.getcwd()
37 systems = util.getPackageModules('moap.vcs', ignore=['vcs', ])
38 log.debug('vcs', 'trying vcs modules %r' % systems)
39
40 for s in systems:
41 m = util.namedModule('moap.vcs.%s' % s)
42
43 try:
44 ret = m.detect(path)
45 except AttributeError:
46 sys.stderr.write('moap.vcs.%s is missing detect()\n' % s)
47 continue
48
49 if ret:
50 try:
51 o = m.VCSClass(path)
52 except AttributeError:
53 sys.stderr.write('moap.vcs.%s is missing VCSClass()\n' % s)
54 continue
55
56 log.debug('vcs', 'detected VCS %s' % s)
57
58 return o
59 log.debug('vcs', 'did not find %s' % s)
60
61 return None
62
63
64 -class VCS(log.Loggable):
65 """
66 @ivar path: the path to the top of the source tree
67 @ivar meta: paths that contain VCS metadata
68 @type meta: list of str
69 """
70 name = 'Some Version Control System'
71 logCategory = 'VCS'
72
73 path = None
74 meta = None
75
80
82 """
83 Get a list of paths newly added under the given path and relative to it.
84
85 @param path: the path under which to check for files
86 @type path: str
87
88 @returns: list of paths
89 @rtype: list of str
90 """
91 log.info('vcs',
92 "subclass %r should implement getAdded" % self.__class__)
93
95 """
96 Get a list of deleted paths under the given path and relative to it.
97
98 @param path: the path under which to check for files
99 @type path: str
100
101 @returns: list of paths
102 @rtype: list of str
103 """
104 log.info('vcs',
105 "subclass %r should implement getDeleted" % self.__class__)
106
108 """
109 Get a list of ignored paths under the given path and relative to it.
110
111 @param path: the path under which to check for files
112 @type path: str
113
114 @returns: list of paths
115 @rtype: list of str
116 """
117 raise NotImplementedError, \
118 'subclass %s should implement getIgnored' % self.__class__
119
121 """
122 Get a list of unknown paths under the given path and relative to it.
123
124 @param path: the path under which to check for files
125 @type path: str
126
127 @returns: list of paths
128 @rtype: list of str
129 """
130 raise NotImplementedError, \
131 'subclass %s should implement getUnknown' % self.__class__
132
133 - def ignore(self, paths, commit=True):
134 """
135 Make the VCS ignore the given list of paths.
136
137 @param paths: list of paths, relative to the checkout directory
138 @type paths: list of str
139 @param commit: if True, commit the ignore updates.
140 @type commit: boolean
141 """
142 raise NotImplementedError, \
143 'subclass %s should implement ignore' % self.__class__
144
145 - def commit(self, paths, message):
146 """
147 Commit the given list of paths, with the given message.
148 Note that depending on the VCS, parents that were just added
149 may need to be commited as well.
150
151 @type paths: list
152 @type message: str
153
154 @rtype: bool
155 """
156
158 """
159 Given the list of paths, create a dict of parentPath -> [child, ...]
160 If the path is in the root of the repository, parentPath will be ''
161
162 @rtype: dict of str -> list of str
163 """
164 result = {}
165
166 if not paths:
167 return result
168
169 for p in paths:
170
171 if p.endswith(os.path.sep): p = p[:-1]
172 base = os.path.basename(p)
173 dirname = os.path.dirname(p)
174 if not dirname in result.keys():
175 result[dirname] = []
176 result[dirname].append(base)
177
178 return result
179
180 - def diff(self, path):
181 """
182 Return a diff for the given path.
183
184 The diff should not end in a newline; an empty diff should
185 be an empty string.
186
187 The diff should also be relative to the working directory; no
188 absolute paths.
189
190 @rtype: str
191 @returns: the diff
192 """
193 raise NotImplementedError, \
194 'subclass %s should implement diff' % self.__class__
195
197 """
198 Return an re matcher object that will expand to the file being
199 changed.
200
201 The default implementation works for CVS and SVN.
202 """
203 return re.compile('^Index: (\S+)$')
204
206 """
207 Get a list of changes for the given path and subpaths.
208
209 @type diff: str
210 @param diff: the diff to use instead of a local vcs diff
211 (only useful for testing)
212
213 @returns: dict of path -> list of (oldLine, oldCount, newLine, newCount)
214 """
215 if not diff:
216 self.debug('getting changes from diff in %s' % path)
217 diff = self.diff(path)
218
219 changes = {}
220 fileMatcher = self.getFileMatcher()
221
222
223
224
225 changeMatcher = re.compile(
226 '^\@\@\s+'
227 '(-)(\d+),?(\d*)'
228 '\s+'
229 '(\+)(\d+),?(\d*)'
230 '\s+\@\@'
231 )
232
233 lines = diff.rstrip('\n').split("\n")
234 self.debug('diff is %d lines' % len(lines))
235 for i in range(len(lines)):
236 fm = fileMatcher.search(lines[i])
237 if fm:
238
239 path = fm.expand('\\1')
240 self.debug('Found file %s with deltas on line %d' % (
241 path, i + 1))
242 changes[path] = []
243 i += 1
244 while i < len(lines) and not fileMatcher.search(lines[i]):
245 self.log('Looking at line %d for file match' % (i + 1))
246 m = changeMatcher.search(lines[i])
247 if m:
248 self.debug('Found change on line %d' % (i + 1))
249 oldLine = int(m.expand('\\2'))
250
251 c = m.expand('\\3')
252 if not c: c = '1'
253 oldCount = int(c)
254 newLine = int(m.expand('\\5'))
255 c = m.expand('\\6')
256 if not c: c = '1'
257 newCount = int(c)
258 i += 1
259
260
261
262
263
264 block = []
265 while i < len(lines) \
266 and not changeMatcher.search(lines[i]) \
267 and not fileMatcher.search(lines[i]):
268 block.append(lines[i])
269 i += 1
270
271
272 self.log('Found change block of %d lines at line %d' % (
273 len(block), i - len(block) + 1))
274
275 for line in block:
276
277
278 if line[0] == ' ':
279 oldLine += 1
280 newLine += 1
281 oldCount -= 1
282 newCount -= 1
283 else:
284 break
285
286 block.reverse()
287 for line in block:
288
289
290 if line and line[0] == ' ':
291 oldCount -= 1
292 newCount -= 1
293 else:
294 break
295
296 changes[path].append(
297 (oldLine, oldCount, newLine, newCount))
298
299
300 i -= 1
301
302 i += 1
303
304 log.debug('vcs', '%d files changed' % len(changes.keys()))
305 return changes
306
308 """
309 Get a list of property changes for the given path and subpaths.
310 These are metadata changes to files, not content changes.
311
312 @rtype: dict of str -> list of str
313 @returns: dict of path -> list of property names
314 """
315 log.info('vcs',
316 "subclass %r should implement getPropertyChanges" % self.__class__)
317
319 """
320 Update the given path to the latest version.
321 """
322 raise NotImplementedError, \
323 'subclass %s should implement update' % self.__class__
324
326 """
327 Return shell commands necessary to do a fresh checkout of the current
328 checkout into a directory called 'checkout'.
329
330 @returns: newline-terminated string of commands.
331 @rtype: str
332 """
333 raise NotImplementedError, \
334 'subclass %s should implement getCheckoutCommands' % self.__class__
335
337 """
338 Back up the given VCS checkout into an archive.
339
340 This stores all unignored files, as well as a checkout command and
341 a diff, so the working directory can be fully restored.
342
343 The archive will contain:
344 - a subdirectory called unignored
345 - a file called diff
346 - an executable file called checkout.sh
347
348 @raises VCSBackupException: if for some reason it can't guarantee
349 a correct backup
350 """
351 mode = 'w:'
352 if archive.endswith('.gz'):
353 mode = 'w:gz'
354 if archive.endswith('.bz2'):
355 mode = 'w:bz2'
356
357
358
359
360
361
362
363 tar = tarfile.TarFile.open(name=archive, mode=mode)
364
365
366 (fd, diffpath) = tempfile.mkstemp(prefix='moap.backup.diff.')
367 diff = self.diff('')
368 if diff:
369 os.write(fd, diff + '\n')
370 os.close(fd)
371 tar.add(diffpath, arcname='diff')
372
373
374 (fd, checkoutpath) = tempfile.mkstemp(prefix='moap.backup.checkout.')
375 os.write(fd, "#!/bin/sh\n" + self.getCheckoutCommands())
376 os.close(fd)
377 os.chmod(checkoutpath, 0755)
378 tar.add(checkoutpath, arcname='checkout.sh')
379
380
381 tar.add(self.path, 'unignored', recursive=False)
382
383 unignoreds = self.getUnknown(self.path)
384
385 for rel in unignoreds:
386 abspath = os.path.join(self.path, rel)
387 self.debug('Adding unignored path %s', rel)
388 tar.add(abspath, 'unignored/' + rel)
389 tar.close()
390 os.unlink(diffpath)
391 os.unlink(checkoutpath)
392
393
394 restoreDir = tempfile.mkdtemp(prefix="moap.test.restore.")
395 os.rmdir(restoreDir)
396 self.restore(archive, restoreDir)
397
398 diff = self.diffCheckout(restoreDir)
399 if diff:
400 msg = "Unexpected diff output between %s and %s:\n%s" % (
401 self.path, restoreDir, diff)
402 self.debug(msg)
403 raise VCSBackupException(msg)
404 else:
405 self.debug('No important difference between '
406 'extracted archive and original directory')
407
408 os.system('rm -rf %s' % restoreDir)
409
411 """
412 Restore from the given archive to the given path.
413 """
414 self.debug('Restoring from archive %s to path %s' % (
415 archive, path))
416
417 if os.path.exists(path):
418 raise VCSException('path %s already exists')
419
420 oldPath = os.getcwd()
421
422
423 tar = tarfile.TarFile.open(name=archive)
424 try:
425 tar.extractall(path)
426 except AttributeError:
427
428 self.debug('Restoring by using tar directly')
429 os.system('mkdir -p %s' % path)
430 if archive.endswith('.gz'):
431 os.system('cd %s; tar xzf %s' % (path, archive))
432 elif archive.endswith('.bz2'):
433 os.system('cd %s; tar xjf %s' % (path, archive))
434 else:
435 raise AssertionError("Don't know how to handle %s" % archive)
436
437
438 os.chdir(path)
439 status, output = commands.getstatusoutput('./checkout.sh')
440 if status:
441 raise VCSException('checkout failed with status %r: %r' % (
442 status, output))
443 os.unlink('checkout.sh')
444
445
446 os.chdir('checkout')
447
448 os.system('patch -p0 < ../diff > /dev/null')
449 os.chdir('..')
450 os.unlink('diff')
451
452
453
454 for path in glob.glob('checkout/*') + glob.glob('checkout/.*'):
455 os.rename(path, os.path.basename(path))
456 os.rmdir('checkout')
457
458
459
460 for path in glob.glob('unignored/*') + glob.glob('unignored/.*'):
461
462
463
464 cmd = 'mv %s %s' % (path, os.path.basename(path))
465 self.debug(cmd)
466 os.system(cmd)
467 os.rmdir('unignored')
468
469 os.chdir(oldPath)
470
472 """
473 Diff our checkout to the given checkout directory.
474
475 Only complains about diffs in files we're interested in, which are
476 tracked or unignored files.
477 """
478 options = ""
479 if self.meta:
480 metaPattern = [s.replace('.', '\.') for s in self.meta]
481 options = "-x ".join([''] + metaPattern)
482 cmd = 'diff -aur %s %s %s 2>&1' % (options, self.path, checkoutDir)
483 self.debug('diffCheckout: running %s' % cmd)
484 output = commands.getoutput(cmd)
485 lines = output.split('\n')
486
487
488
489 d = {}
490 for path in self.getIgnored(self.path):
491 d[path] = True
492
493 matcher = re.compile('Only in (.*): (.*)$')
494
495 def isIgnored(line):
496
497 m = matcher.search(line)
498 if m:
499 path = os.path.join(m.expand("\\1"), m.expand("\\2"))
500 if path in d.keys():
501 self.debug('Removing ignored path %s from diff' % path)
502 return True
503
504 return False
505
506 lines = [l for l in lines if isIgnored(l)]
507
508 return "\n".join(lines)
509
511 """
512 Generic exception for a failed VCS operation.
513 """
514 pass
515
517 'The VCS cannot back up the working directory.'
518