diff --git a/etc/utils/add_version.py b/etc/utils/add_version.py index 8f2b2abc..e1840d21 100755 --- a/etc/utils/add_version.py +++ b/etc/utils/add_version.py @@ -17,6 +17,8 @@ """ +from __future__ import absolute_import, print_function + import sys import re @@ -25,12 +27,12 @@ def find_version(path): for line in f: index = line.find('GMVAULT_VERSION="') if index > -1: - print(line[index+17:-2]) + print((line[index+17:-2])) return line[index+17:-2] raise Exception("Cannot find GMVAULT_VERSION in %s\n" % (path)) -VERSION_PATTERN = r'###GMVAULTVERSION###' +VERSION_PATTERN = r'###GMVAULTVERSION###' VERSION_RE = re.compile(VERSION_PATTERN) def add_version(a_input, a_output, a_version): @@ -43,7 +45,7 @@ def add_version(a_input, a_output, a_version): if __name__ == '__main__': if len(sys.argv) < 4: - print("Error: need more parameters for %s." % (sys.argv[0])) + print(("Error: need more parameters for %s." % (sys.argv[0]))) print("Usage: add_version.py input_path output_path version.") exit(-1) diff --git a/etc/utils/find_version.py b/etc/utils/find_version.py index cf8bdeab..1d804623 100755 --- a/etc/utils/find_version.py +++ b/etc/utils/find_version.py @@ -17,6 +17,8 @@ """ +from __future__ import absolute_import, print_function + import sys @@ -25,7 +27,7 @@ def find_version(path): for line in f: index = line.find('GMVAULT_VERSION = "') if index > -1: - print(line[index+19:-2]) + print((line[index+19:-2])) res = line[index+19:-2] return res.strip() diff --git a/etc/utils/flask_stats.py b/etc/utils/flask_stats.py index 79239e10..56c8a699 100644 --- a/etc/utils/flask_stats.py +++ b/etc/utils/flask_stats.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import from flask import Flask import scrapping diff --git a/etc/utils/mem-profiling-tools/dowser/__init__.py b/etc/utils/mem-profiling-tools/dowser/__init__.py index fa562b17..2123986b 100644 --- a/etc/utils/mem-profiling-tools/dowser/__init__.py +++ b/etc/utils/mem-profiling-tools/dowser/__init__.py @@ -1,8 +1,9 @@ +from __future__ import absolute_import import cgi import gc import os localDir = os.path.join(os.getcwd(), os.path.dirname(__file__)) -from StringIO import StringIO +from io import BytesIO import sys import threading import time @@ -13,9 +14,11 @@ import cherrypy -import reftree +from . import reftree +from six import iteritems + def get_repr(obj, limit=250): return cgi.escape(reftree.get_repr(obj, limit)) @@ -44,10 +47,10 @@ def template(name, **params): class Root: - + period = 5 maxhistory = 300 - + def __init__(self): self.history = {} self.samples = 0 @@ -55,16 +58,16 @@ def __init__(self): cherrypy.engine.subscribe('exit', self.stop) self.runthread = threading.Thread(target=self.start) self.runthread.start() - + def start(self): self.running = True while self.running: self.tick() time.sleep(self.period) - + def tick(self): gc.collect() - + typecounts = {} for obj in gc.get_objects(): objtype = type(obj) @@ -72,34 +75,34 @@ def tick(self): typecounts[objtype] += 1 else: typecounts[objtype] = 1 - - for objtype, count in typecounts.iteritems(): + + for objtype, count in iteritems(typecounts): typename = objtype.__module__ + "." + objtype.__name__ if typename not in self.history: self.history[typename] = [0] * self.samples self.history[typename].append(count) - + samples = self.samples + 1 - + # Add dummy entries for any types which no longer exist - for typename, hist in self.history.iteritems(): + for typename, hist in iteritems(self.history): diff = samples - len(hist) if diff > 0: hist.extend([0] * diff) - + # Truncate history to self.maxhistory if samples > self.maxhistory: - for typename, hist in self.history.iteritems(): + for typename, hist in iteritems(self.history): hist.pop(0) else: self.samples = samples - + def stop(self): self.running = False - + def index(self, floor=0): rows = [] - typenames = self.history.keys() + typenames = list(self.history.keys()) typenames.sort() for typename in typenames: hist = self.history[typename] @@ -117,7 +120,7 @@ def index(self, floor=0): rows.append(row) return template("graphs.html", output="\n".join(rows)) index.exposed = True - + def chart(self, typename): """Return a sparkline chart of the given type.""" data = self.history[typename] @@ -128,28 +131,28 @@ def chart(self, typename): draw.line([(i, int(height - (v * scale))) for i, v in enumerate(data)], fill="#009900") del draw - - f = StringIO() + + f = BytesIO() im.save(f, "PNG") result = f.getvalue() - + cherrypy.response.headers["Content-Type"] = "image/png" return result chart.exposed = True - + def trace(self, typename, objid=None): gc.collect() - + if objid is None: rows = self.trace_all(typename) else: rows = self.trace_one(typename, objid) - + return template("trace.html", output="\n".join(rows), typename=cgi.escape(typename), objid=str(objid or '')) trace.exposed = True - + def trace_all(self, typename): rows = [] for obj in gc.get_objects(): @@ -160,7 +163,7 @@ def trace_all(self, typename): if not rows: rows = ["

The type you requested was not found.

"] return rows - + def trace_one(self, typename, objid): rows = [] objid = int(objid) @@ -181,7 +184,7 @@ def trace_one(self, typename, objid): (k, get_repr(v))) del v rows.append('') - + # Referrers rows.append('

Referrers (Parents)

') rows.append('

Show the ' @@ -193,7 +196,7 @@ def trace_one(self, typename, objid): if parentid: rows.append("

%s

" % parentrepr) rows.append('
') - + # Referents rows.append('

Referents (Children)

') for child in gc.get_referents(obj): @@ -203,10 +206,10 @@ def trace_one(self, typename, objid): if not rows: rows = ["

The object you requested was not found.

"] return rows - + def tree(self, typename, objid): gc.collect() - + rows = [] objid = int(objid) all_objs = gc.get_objects() @@ -218,17 +221,17 @@ def tree(self, typename, objid): "of the correct type."] else: rows.append('
') - + tree = ReferrerTree(obj) tree.ignore(all_objs) for depth, parentid, parentrepr in tree.walk(maxresults=1000): rows.append(parentrepr) - + rows.append('
') break if not rows: rows = ["

The object you requested was not found.

"] - + params = {'output': "\n".join(rows), 'typename': cgi.escape(typename), 'objid': str(objid), @@ -254,17 +257,17 @@ def tree(self, typename, objid): class ReferrerTree(reftree.Tree): - + ignore_modules = True - + def _gen(self, obj, depth=0): if self.maxdepth and depth >= self.maxdepth: yield depth, 0, "---- Max depth reached ----" raise StopIteration - + if isinstance(obj, ModuleType) and self.ignore_modules: raise StopIteration - + refs = gc.get_referrers(obj) refiter = iter(refs) self.ignore(refs, refiter) @@ -274,16 +277,16 @@ def _gen(self, obj, depth=0): if (isinstance(ref, FrameType) and ref.f_code.co_filename in (thisfile, self.filename)): continue - + # Exclude all functions and classes from this module or reftree. mod = getattr(ref, "__module__", "") if "dowser" in mod or "reftree" in mod or mod == '__main__': continue - + # Exclude all parents in our ignore list. if id(ref) in self._ignore: continue - + # Yield the (depth, id, repr) of our object. yield depth, 0, '%s
' % (" " * depth) if id(ref) in self.seen: @@ -291,21 +294,21 @@ def _gen(self, obj, depth=0): else: self.seen[id(ref)] = None yield depth, id(ref), self.get_repr(ref, obj) - + for parent in self._gen(ref, depth + 1): yield parent yield depth, 0, '%s
' % (" " * depth) - + def get_repr(self, obj, referent=None): """Return an HTML tree block describing the given object.""" objtype = type(obj) typename = objtype.__module__ + "." + objtype.__name__ prettytype = typename.replace("__builtin__.", "") - + name = getattr(obj, "__name__", "") if name: prettytype = "%s %r" % (prettytype, name) - + key = "" if referent: key = self.get_refkey(obj, referent) @@ -315,14 +318,14 @@ def get_repr(self, obj, referent=None): % (url("/trace/%s/%s" % (typename, id(obj))), id(obj), prettytype, key, get_repr(obj, 100)) ) - + def get_refkey(self, obj, referent): """Return the dict key or attribute name of obj which refers to referent.""" if isinstance(obj, dict): - for k, v in obj.iteritems(): + for k, v in iteritems(obj): if v is referent: return " (via its %r key)" % k - + for k in dir(obj) + ['__dict__']: if getattr(obj, k, None) is referent: return " (via its %r attribute)" % k diff --git a/etc/utils/mem-profiling-tools/dowser/reftree.py b/etc/utils/mem-profiling-tools/dowser/reftree.py index d0aec28e..2e4dd042 100644 --- a/etc/utils/mem-profiling-tools/dowser/reftree.py +++ b/etc/utils/mem-profiling-tools/dowser/reftree.py @@ -1,36 +1,39 @@ +from __future__ import absolute_import, print_function + import gc import sys from types import FrameType +import six class Tree: - + def __init__(self, obj): self.obj = obj self.filename = sys._getframe().f_code.co_filename self._ignore = {} - + def ignore(self, *objects): for obj in objects: self._ignore[id(obj)] = None - + def ignore_caller(self): f = sys._getframe() # = this function cur = f.f_back # = the function that called us (probably 'walk') self.ignore(cur, cur.f_builtins, cur.f_locals, cur.f_globals) caller = f.f_back # = the 'real' caller self.ignore(caller, caller.f_builtins, caller.f_locals, caller.f_globals) - + def walk(self, maxresults=100, maxdepth=None): """Walk the object tree, ignoring duplicates and circular refs.""" self.seen = {} self.ignore(self, self.__dict__, self.obj, self.seen, self._ignore) - + # Ignore the calling frame, its builtins, globals and locals self.ignore_caller() - + self.maxdepth = maxdepth count = 0 for result in self._gen(self.obj): @@ -39,12 +42,12 @@ def walk(self, maxresults=100, maxdepth=None): if maxresults and count >= maxresults: yield 0, 0, "==== Max results reached ====" raise StopIteration - + def print_tree(self, maxresults=100, maxdepth=None): """Walk the object tree, pretty-printing each branch.""" self.ignore_caller() for depth, refid, rep in self.walk(maxresults, maxdepth): - print ("%9d" % refid), (" " * depth * 2), rep + print(("%9d" % refid), (" " * depth * 2), rep) def _repr_container(obj): @@ -64,25 +67,25 @@ def repr_frame(obj): def get_repr(obj, limit=250): typename = getattr(type(obj), "__name__", None) handler = globals().get("repr_%s" % typename, repr) - + try: result = handler(obj) except: result = "unrepresentable object: %r" % sys.exc_info()[1] - + if len(result) > limit: result = result[:limit] + "..." - + return result class ReferentTree(Tree): - + def _gen(self, obj, depth=0): if self.maxdepth and depth >= self.maxdepth: yield depth, 0, "---- Max depth reached ----" raise StopIteration - + for ref in gc.get_referents(obj): if id(ref) in self._ignore: continue @@ -92,18 +95,18 @@ def _gen(self, obj, depth=0): else: self.seen[id(ref)] = None yield depth, id(ref), get_repr(ref) - + for child in self._gen(ref, depth + 1): yield child class ReferrerTree(Tree): - + def _gen(self, obj, depth=0): if self.maxdepth and depth >= self.maxdepth: yield depth, 0, "---- Max depth reached ----" raise StopIteration - + refs = gc.get_referrers(obj) refiter = iter(refs) self.ignore(refs, refiter) @@ -112,7 +115,7 @@ def _gen(self, obj, depth=0): if isinstance(ref, FrameType): if ref.f_code.co_filename == self.filename: continue - + if id(ref) in self._ignore: continue elif id(ref) in self.seen: @@ -121,23 +124,23 @@ def _gen(self, obj, depth=0): else: self.seen[id(ref)] = None yield depth, id(ref), get_repr(ref) - + for parent in self._gen(ref, depth + 1): yield parent class CircularReferents(Tree): - + def walk(self, maxresults=100, maxdepth=None): """Walk the object tree, showing circular referents.""" self.stops = 0 self.seen = {} self.ignore(self, self.__dict__, self.seen, self._ignore) - + # Ignore the calling frame, its builtins, globals and locals self.ignore_caller() - + self.maxdepth = maxdepth count = 0 for result in self._gen(self.obj): @@ -146,15 +149,15 @@ def walk(self, maxresults=100, maxdepth=None): if maxresults and count >= maxresults: yield 0, 0, "==== Max results reached ====" raise StopIteration - + def _gen(self, obj, depth=0, trail=None): if self.maxdepth and depth >= self.maxdepth: self.stops += 1 raise StopIteration - + if trail is None: trail = [] - + for ref in gc.get_referents(obj): if id(ref) in self._ignore: continue @@ -162,21 +165,21 @@ def _gen(self, obj, depth=0, trail=None): continue else: self.seen[id(ref)] = None - + refrepr = get_repr(ref) if id(ref) == id(self.obj): yield trail + [refrepr,] - + for child in self._gen(ref, depth + 1, trail + [refrepr,]): yield child - + def print_tree(self, maxresults=100, maxdepth=None): """Walk the object tree, pretty-printing each branch.""" self.ignore_caller() for trail in self.walk(maxresults, maxdepth): - print trail + print(trail) if self.stops: - print "%s paths stopped because max depth reached" % self.stops + print("%s paths stopped because max depth reached" % self.stops) def count_objects(): @@ -184,7 +187,7 @@ def count_objects(): for obj in gc.get_objects(): objtype = type(obj) d[objtype] = d.get(objtype, 0) + 1 - d = [(v, k) for k, v in d.iteritems()] + d = [(v, k) for k, v in six.iteritems(d)] d.sort() return d diff --git a/etc/utils/mem-profiling-tools/memdebug.py b/etc/utils/mem-profiling-tools/memdebug.py index bdf412ad..58baaf11 100644 --- a/etc/utils/mem-profiling-tools/memdebug.py +++ b/etc/utils/mem-profiling-tools/memdebug.py @@ -1,5 +1,6 @@ # memdebug.py +from __future__ import absolute_import import cherrypy import dowser diff --git a/etc/utils/scrapping.py b/etc/utils/scrapping.py index 9593a761..7d419862 100755 --- a/etc/utils/scrapping.py +++ b/etc/utils/scrapping.py @@ -19,6 +19,8 @@ #quick and dirty scrapper to get number of downloads +from __future__ import absolute_import, print_function + import json import datetime import mechanize @@ -41,16 +43,16 @@ def get_from_bitbucket(): #body_tag = soup.body all_tables = soup.findAll('table') - table = soup.find(lambda tag: tag.name=='table' and tag.has_key('id') and tag['id']=="uploaded-files") + table = soup.find(lambda tag: tag.name=='table' and 'id' in tag and tag['id']=="uploaded-files") rows = table.findAll(lambda tag: tag.name=='tr') - res = {} + res = {} for row in rows: - + tds = row.findAll(lambda tag: tag.name == 'td') - #print("tds = %s\n" %(tds)) + #print("tds = %s\n" %(tds)) td_number = 0 name = None @@ -73,10 +75,10 @@ def get_from_bitbucket(): return res def get_from_pypi(url): - + res = {} - print("Get info from pypi (url= %s)\n" % (url)) + print(("Get info from pypi (url= %s)\n" % (url))) br = mechanize.Browser() br.open(url) @@ -93,12 +95,12 @@ def get_from_pypi(url): #print("rows = %s\n" %(rows)) for row in rows: - + tds = row.findAll(lambda tag: tag.name == 'td') - #print("tds = %s\n" %(tds)) + #print("tds = %s\n" %(tds)) - #ignore tds that are too small + #ignore tds that are too small if len(tds) < 6: #print("ignore td = %s\n" % (tds)) continue @@ -142,20 +144,20 @@ def get_stats(return_type): res.update(get_from_pypi("https://pypi.python.org/pypi/gmvault/1.8-beta")) res.update(get_from_pypi("https://pypi.python.org/pypi/gmvault/1.7-beta")) - #print("name , nb_downloads") + #print("name , nb_downloads") total = 0 win_total = 0 lin_total = 0 mac_total = 0 v17_total = 0 - v18_total = 0 - v181_total = 0 + v18_total = 0 + v181_total = 0 pypi_total = 0 src_total = 0 for key in res.keys(): #print("key= %s: (%s)\n" %(key, res[key])) if key.endswith(".exe"): - win_total += res[key] + win_total += res[key] elif "macosx" in key: mac_total += res[key] else: @@ -178,11 +180,11 @@ def get_stats(return_type): #print("%s, %s\n" % (key, res[key])) total += res[key] - total += TOTAL_PREVIOUS_VERSIONS + total += TOTAL_PREVIOUS_VERSIONS win_total += WIN_TOTAL_PREVIOUS_VERSIONS lin_total += LIN_TOTAL_PREVIOUS_VERSIONS mac_total += MAC_TOTAL_PREVIOUS_VERSIONS - pypi_total += PYPI_TOTAL_PREVIOUS_VERSIONS + pypi_total += PYPI_TOTAL_PREVIOUS_VERSIONS src_total += SRC_TOTAL_PREVIOUS_VERSIONS the_str = "" @@ -204,6 +206,6 @@ def get_stats(return_type): if __name__ == "__main__": - print(get_stats("JSON")) + print((get_stats("JSON"))) diff --git a/setup.py b/setup.py index f6f07f85..dcb406c5 100644 --- a/setup.py +++ b/setup.py @@ -16,6 +16,8 @@ along with this program. If not, see . """ +from __future__ import absolute_import, print_function + import os from setuptools import setup @@ -61,11 +63,11 @@ def find_version(path): author_email='guillaume.aubert@gmail.com', url='http://www.gmvault.org', license='AGPLv3', - packages=['gmv','gmv.conf', 'gmv.conf.utils'], - package_dir = {'gmv': './src/gmv'}, + packages=['gmv', 'gmv.conf', 'gmv.conf.utils'], + package_dir={'': 'src'}, scripts=['./etc/scripts/gmvault'], package_data={'': ['release-note.txt']}, include_package_data=True, #install_requires=['argparse', 'Logbook==0.4.1', 'IMAPClient==0.9.2','gdata==2.0.17'] - install_requires=['argparse', 'Logbook==0.10.1', 'IMAPClient==0.13', 'chardet==2.3.0'] + install_requires=['argparse', 'Logbook==0.10.1', 'IMAPClient==0.13', 'chardet==2.3.0', 'six'] ) diff --git a/src/gmv/blowfish.py b/src/gmv/blowfish.py index 34602089..51e19c16 100644 --- a/src/gmv/blowfish.py +++ b/src/gmv/blowfish.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- - + # blowfish.py # Copyright (C) 2002 Michael Gilfix # @@ -16,7 +16,7 @@ # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. # - + # This software was modified by Ivan Voras: CTR cipher mode of # operation was added, together with testing and example code. # These changes are (c) 2007./08. Ivan Voras @@ -24,7 +24,7 @@ # GPL or Artistic License, the same as the original module. # All disclaimers of warranty from the original module also # apply to these changes. - + # Further modifications by Neil Tallim to make use # of more modern Python practises and features, improving # performance and, in this maintainer's opinion, readability. @@ -33,18 +33,37 @@ # June 13, 2010, subject to the terms of the original module. """ Blowfish Encryption - + This module is a pure python implementation of Bruce Schneier's encryption scheme 'Blowfish'. Blowish is a 16-round Feistel Network cipher and offers substantial speed gains over DES. - + The key is a string of length anywhere between 64 and 448 bits, or equivalently 8 and 56 bytes. The encryption and decryption functions operate on 64-bit blocks, or 8-byte strings. """ +from __future__ import absolute_import, print_function + import array import struct - +from six.moves import range + + +def _ensure_bytearray(s): + """Convert bytes to bytearrays and complain otherwise""" + if not isinstance(s, bytearray): + if not isinstance(s, bytes): + raise TypeError('Expected object of type bytes or bytearray, got: ' + '{0}'.format(type(s))) + else: + s = bytearray(s) + return s + +def _chr(i): + """Python 2/3 compatible version of chr for byte strings""" + return bytes(bytearray((i,))) + + class Blowfish: """ Implements the encryption and decryption functionality of the Blowfish @@ -53,29 +72,31 @@ class Blowfish: # Key restrictions KEY_MIN_LEN = 8 #64 bits KEY_MAX_LEN = 56 #448 bits - + # Cipher directions ENCRYPT = 0 DECRYPT = 1 - + # For _round() - _MODULUS = 2L ** 32 - + _MODULUS = 2 ** 32 + # CTR constants _BLOCK_SIZE = 8 - + def __init__(self, key): """ Creates an instance of blowfish using 'key' as the encryption key. - + Key is a string of bytes, used to seed calculations. Once the instance of the object is created, the key is no longer necessary. """ + key = _ensure_bytearray(key) + if not self.KEY_MIN_LEN <= len(key) <= self.KEY_MAX_LEN: raise ValueError("Attempted to initialize Blowfish cipher with key of invalid length: %(len)i" % { 'len': len(key), }) - + self._p_boxes = array.array('I', [ 0x243F6A88, 0x85A308D3, 0x13198A2E, 0x03707344, 0xA4093822, 0x299F31D0, 0x082EFA98, 0xEC4E6C89, @@ -83,7 +104,7 @@ def __init__(self, key): 0xC0AC29B7, 0xC97C50DD, 0x3F84D5B5, 0xB5470917, 0x9216D5D9, 0x8979FB1B ]) - + self._s_boxes = ( array.array('I', [ 0xD1310BA6, 0x98DFB5AC, 0x2FFD72DB, 0xD01ADFB7, @@ -350,36 +371,36 @@ def __init__(self, key): 0xB74E6132, 0xCE77E25B, 0x578FDFE3, 0x3AC372E6 ]) ) - + # Cycle through the p-boxes and round-robin XOR the # key with the p-boxes key_len = len(key) index = 0 - for i in xrange(len(self._p_boxes)): + for i in range(len(self._p_boxes)): self._p_boxes[i] = self._p_boxes[i] ^ ( - (ord(key[index % key_len]) << 24) + - (ord(key[(index + 1) % key_len]) << 16) + - (ord(key[(index + 2) % key_len]) << 8) + - (ord(key[(index + 3) % key_len])) + (key[index % key_len] << 24) + + (key[(index + 1) % key_len] << 16) + + (key[(index + 2) % key_len] << 8) + + (key[(index + 3) % key_len]) ) index += 4 - + # For the chaining process l = r = 0 - + # Begin chain replacing the p-boxes - for i in xrange(0, len(self._p_boxes), 2): + for i in range(0, len(self._p_boxes), 2): (l, r) = self.cipher(l, r, self.ENCRYPT) self._p_boxes[i] = l self._p_boxes[i + 1] = r - + # Chain replace the s-boxes - for i in xrange(len(self._s_boxes)): - for j in xrange(0, len(self._s_boxes[i]), 2): + for i in range(len(self._s_boxes)): + for j in range(0, len(self._s_boxes[i]), 2): (l, r) = self.cipher(l, r, self.ENCRYPT) self._s_boxes[i][j] = l self._s_boxes[i][j + 1] = r - + def initCTR(self, iv=0): """ Initializes CTR engine for encryption or decryption. @@ -389,18 +410,18 @@ def initCTR(self, iv=0): 'target-len': self._BLOCK_SIZE, 'q-len': struct.calcsize("Q"), }) - + self._ctr_iv = iv self._calcCTRBuf() - + def cipher(self, xl, xr, direction): """ Encrypts a 64-bit block of data where xl is the upper 32 bits and xr is the lower 32-bits. - + 'direction' is the direction to apply the cipher, either ENCRYPT or DECRYPT class-constants. - + Returns a tuple of either encrypted or decrypted data of the left half and right half of the 64-bit block. """ @@ -421,70 +442,75 @@ def cipher(self, xl, xr, direction): xr = xr ^ self._p_boxes[1] xl = xl ^ self._p_boxes[0] return (xl, xr) - + def encrypt(self, data): """ Encrypt an 8-byte (64-bit) block of text where 'data' is an 8 byte string. - + Returns an 8-byte encrypted string. """ + data = _ensure_bytearray(data) + if not len(data) == 8: raise ValueError("Attempted to encrypt data of invalid block length: %(len)i" % { 'len': len(data), }) - + # Use big endianess since that's what everyone else uses - xl = (ord(data[3])) | (ord(data[2]) << 8) | (ord(data[1]) << 16) | (ord(data[0]) << 24) - xr = (ord(data[7])) | (ord(data[6]) << 8) | (ord(data[5]) << 16) | (ord(data[4]) << 24) - + xl = (data[3]) | (data[2] << 8) | (data[1] << 16) | (data[0] << 24) + xr = (data[7]) | (data[6] << 8) | (data[5] << 16) | (data[4] << 24) + (cl, cr) = self.cipher(xl, xr, self.ENCRYPT) - chars = ''.join ([ - chr((cl >> 24) & 0xFF), chr((cl >> 16) & 0xFF), chr((cl >> 8) & 0xFF), chr(cl & 0xFF), - chr((cr >> 24) & 0xFF), chr((cr >> 16) & 0xFF), chr((cr >> 8) & 0xFF), chr(cr & 0xFF) + chars = b''.join ([ + _chr((cl >> 24) & 0xFF), _chr((cl >> 16) & 0xFF), _chr((cl >> 8) & 0xFF), _chr(cl & 0xFF), + _chr((cr >> 24) & 0xFF), _chr((cr >> 16) & 0xFF), _chr((cr >> 8) & 0xFF), _chr(cr & 0xFF) ]) return chars - + def decrypt(self, data): """ Decrypt an 8 byte (64-bit) encrypted block of text, where 'data' is the 8-byte encrypted string. - + Returns an 8-byte string of plaintext. """ + data = _ensure_bytearray(data) + if not len(data) == 8: raise ValueError("Attempted to encrypt data of invalid block length: %(len)i" % { 'len': len(data), }) - + # Use big endianess since that's what everyone else uses - cl = (ord(data[3])) | (ord(data[2]) << 8) | (ord(data[1]) << 16) | (ord(data[0]) << 24) - cr = (ord(data[7])) | (ord(data[6]) << 8) | (ord(data[5]) << 16) | (ord(data[4]) << 24) - + cl = (data[3]) | (data[2] << 8) | (data[1] << 16) | (data[0] << 24) + cr = (data[7]) | (data[6] << 8) | (data[5] << 16) | (data[4] << 24) + (xl, xr) = self.cipher (cl, cr, self.DECRYPT) - return ''.join ([ - chr((xl >> 24) & 0xFF), chr((xl >> 16) & 0xFF), chr((xl >> 8) & 0xFF), chr(xl & 0xFF), - chr((xr >> 24) & 0xFF), chr((xr >> 16) & 0xFF), chr((xr >> 8) & 0xFF), chr(xr & 0xFF) + return b''.join ([ + _chr((xl >> 24) & 0xFF), _chr((xl >> 16) & 0xFF), _chr((xl >> 8) & 0xFF), _chr(xl & 0xFF), + _chr((xr >> 24) & 0xFF), _chr((xr >> 16) & 0xFF), _chr((xr >> 8) & 0xFF), _chr(xr & 0xFF) ]) - + def encryptCTR(self, data): """ Encrypts an arbitrary string and returns the encrypted string. - + This method can be called successively for multiple string blocks. """ - if not type(data) is str: + data = _ensure_bytearray(data) + if not isinstance(data, bytearray): raise TypeError("Only 8-bit strings are supported") - - return ''.join([chr(ord(ch) ^ self._nextCTRByte()) for ch in data]) - + + return b''.join([_chr(ch ^ self._nextCTRByte()) for ch in data]) + def decryptCTR(self, data): """ Decrypts a string encrypted with encryptCTR() and returns the original string. """ return self.encryptCTR(data) - + def _calcCTRBuf(self): """ Calculates one block of CTR keystream. @@ -492,80 +518,80 @@ def _calcCTRBuf(self): self._ctr_cks = self.encrypt(struct.pack("Q", self._ctr_iv)) # keystream block self._ctr_iv += 1 self._ctr_pos = 0 - + def _nextCTRByte(self): """ Returns one byte of CTR keystream. """ - b = ord(self._ctr_cks[self._ctr_pos]) + b = self._ctr_cks[self._ctr_pos] self._ctr_pos += 1 - + if self._ctr_pos >= len(self._ctr_cks): self._calcCTRBuf() return b - + def _round(self, xl): """ Performs an obscuring function on the 32-bit block of data, 'xl', which is the left half of the 64-bit block of data. - + Returns the 32-bit result as a long integer. """ # Perform all ops as longs then and out the last 32-bits to # obtain the integer - f = long(self._s_boxes[0][(xl & 0xFF000000) >> 24]) - f += long(self._s_boxes[1][(xl & 0x00FF0000) >> 16]) + f = int(self._s_boxes[0][(xl & 0xFF000000) >> 24]) + f += int(self._s_boxes[1][(xl & 0x00FF0000) >> 16]) f %= self._MODULUS - f ^= long(self._s_boxes[2][(xl & 0x0000FF00) >> 8]) - f += long(self._s_boxes[3][(xl & 0x000000FF)]) + f ^= int(self._s_boxes[2][(xl & 0x0000FF00) >> 8]) + f += int(self._s_boxes[3][(xl & 0x000000FF)]) f %= self._MODULUS return f & 0xFFFFFFFF - + # Sample usage ############## if __name__ == '__main__': import time - + def _demo(heading, source, encrypted, decrypted): """demo method """ - print heading - print "\tSource: %(source)s" % { + print(heading) + print("\tSource: %(source)s" % { 'source': source, - } - print "\tEncrypted: %(encrypted)s" % { + }) + print("\tEncrypted: %(encrypted)s" % { 'encrypted': encrypted, - } - print "\tDecrypted: %(decrypted)s" % { + }) + print("\tDecrypted: %(decrypted)s" % { 'decrypted': decrypted, - } - print - - key = 'This is a test key' + }) + print() + + key = b'This is a test key' cipher = Blowfish(key) - + # Encryption processing - (xl, xr) = (123456L, 654321L) + (xl, xr) = (123456, 654321) (cl, cr) = cipher.cipher(xl, xr, cipher.ENCRYPT) (dl, dr) = cipher.cipher(cl, cr, cipher.DECRYPT) _demo("Testing encryption", (xl, xr), (cl, cr), (dl, dr)) - + # Block processing - text = 'testtest' + text = b'testtest' crypted = cipher.encrypt(text) decrypted = cipher.decrypt(crypted) _demo("Testing block encrypt", text, repr(crypted), decrypted) - + # CTR ptocessing cipher.initCTR() - text = "The quick brown fox jumps over the lazy dog" + text = b"The quick brown fox jumps over the lazy dog" crypted = cipher.encryptCTR(text) cipher.initCTR() decrypted = cipher.decryptCTR(crypted) _demo("Testing CTR logic", text, repr(crypted), decrypted) - + # Test speed - print "Testing speed" - test_strings = [''.join(("The quick brown fox jumps over the lazy dog", str(i),)) for i in xrange(1000)] + print("Testing speed") + test_strings = [b''.join((b"The quick brown fox jumps over the lazy dog", bytes(i),)) for i in range(1000)] n = 0 t1 = time.time() while True: @@ -575,8 +601,8 @@ def _demo(heading, source, encrypted, decrypted): t2 = time.time() if t2 - t1 >= 5.0: break - print "%(count)i encryptions in %(time)0.1f seconds: %(throughput)0.1f enc/s" % { + print("%(count)i encryptions in %(time)0.1f seconds: %(throughput)0.1f enc/s" % { 'count': n, 'time': t2 - t1, 'throughput': n / (t2 - t1), - } + }) diff --git a/src/gmv/cmdline_utils.py b/src/gmv/cmdline_utils.py index e195cd76..95d9e911 100755 --- a/src/gmv/cmdline_utils.py +++ b/src/gmv/cmdline_utils.py @@ -17,39 +17,41 @@ ''' +from __future__ import absolute_import, print_function + import argparse import sys -import gmv.log_utils as log_utils +from . import log_utils LOG = log_utils.LoggerFactory.get_logger('cmdline_utils') class CmdLineParser(argparse.ArgumentParser): #pylint: disable=R0904 - """ + """ Added service to OptionParser. - - Comments regarding usability of the lib. + + Comments regarding usability of the lib. By default you want to print the default in the help if you had them so the default formatter should print them Also new lines are eaten in the epilogue strings. You would use an epilogue to show examples most of the time so you - want to have the possiblity to go to a new line. There should be a way to format the epilogue differently from the rest + want to have the possiblity to go to a new line. There should be a way to format the epilogue differently from the rest + + """ - """ - BOOL_TRUE = ['yes', 'true', '1'] BOOL_FALSE = ['no', 'false', '0'] BOOL_VALS = BOOL_TRUE + BOOL_FALSE - - def __init__(self, *args, **kwargs): - """ constructor """ + + def __init__(self, *args, **kwargs): + """ constructor """ argparse.ArgumentParser.__init__(self, *args, **kwargs) #pylint: disable=W0142 - - # I like my help option message better than the default... - #self.remove_option('-h') - #self.add_option('-h', '--help', action='help', help='Show this message and exit.') - - self.epilogue = None - - @classmethod + + # I like my help option message better than the default... + #self.remove_option('-h') + #self.add_option('-h', '--help', action='help', help='Show this message and exit.') + + self.epilogue = None + + @classmethod def convert_to_boolean(cls, val): """ Convert yes, True, true, YES to boolean True and @@ -62,65 +64,65 @@ def convert_to_boolean(cls, val): return False else: raise Exception("val %s should be in %s to be convertible to a boolean." % (val, cls.BOOL_VALS)) - - def print_help(self, out=sys.stderr): - """ - Print the help message, followed by the epilogue (if set), to the - specified output file. You can define an epilogue by setting the - ``epilogue`` field. - + + def print_help(self, out=sys.stderr): + """ + Print the help message, followed by the epilogue (if set), to the + specified output file. You can define an epilogue by setting the + ``epilogue`` field. + :param out: file desc where to write the usage message - - """ + + """ super(CmdLineParser, self).print_help(out) - if self.epilogue: - #print >> out, '\n%s' % textwrap.fill(self.epilogue, 100, replace_whitespace = False) - print >> out, '\n%s' % self.epilogue - out.flush() - - def show_usage(self, msg=None): + if self.epilogue: + #print >> out, '\n%s' % textwrap.fill(self.epilogue, 100, replace_whitespace = False) + print('\n%s' % self.epilogue, file=out) + out.flush() + + def show_usage(self, msg=None): """ - Print usage message + Print usage message + """ + self.die_with_usage(msg) + + def die_with_usage(self, msg=None, exit_code=2): + """ + Display a usage message and exit. + + :Parameters: + msg : str + If not set to ``None`` (the default), this message will be + displayed before the usage message + + exit_code : int + The process exit code. Defaults to 2. + """ + if msg != None: + print(msg, file=sys.stderr) + + self.print_help(sys.stderr) + sys.exit(exit_code) + + def error(self, msg): + """ + Overrides parent ``OptionParser`` class's ``error()`` method and + forces the full usage message on error. """ - self.die_with_usage(msg) - - def die_with_usage(self, msg=None, exit_code=2): - """ - Display a usage message and exit. - - :Parameters: - msg : str - If not set to ``None`` (the default), this message will be - displayed before the usage message - - exit_code : int - The process exit code. Defaults to 2. - """ - if msg != None: - print >> sys.stderr, msg - - self.print_help(sys.stderr) - sys.exit(exit_code) - - def error(self, msg): - """ - Overrides parent ``OptionParser`` class's ``error()`` method and - forces the full usage message on error. - """ self.die_with_usage("%s: error: %s\n" % (self.prog, msg)) - + def message(self, msg): """ - Print a message + Print a message """ - print("%s: %s\n" % (self.prog, msg)) - - + print(("%s: %s\n" % (self.prog, msg))) + + SYNC_HELP_EPILOGUE = """Examples: a) full synchronisation with email and password login -#> gmvault --email foo.bar@gmail.com --passwd vrysecrtpasswd +#> gmvault --email foo.bar@gmail.com --passwd vrysecrtpasswd b) full synchronisation for german users that have to use googlemail instead of gmail @@ -128,7 +130,7 @@ def message(self, msg): c) restrict synchronisation with an IMAP request -#> gmvault --imap-request 'Since 1-Nov-2011 Before 10-Nov-2011' --email foo.bar@gmail.com --passwd sosecrtpasswd +#> gmvault --imap-request 'Since 1-Nov-2011 Before 10-Nov-2011' --email foo.bar@gmail.com --passwd sosecrtpasswd """ @@ -137,12 +139,12 @@ def test_command_parser(): Test the command parser """ #parser = argparse.ArgumentParser() - - + + parser = CmdLineParser() - + subparsers = parser.add_subparsers(help='commands') - + # A sync command sync_parser = subparsers.add_parser('sync', formatter_class=argparse.ArgumentDefaultsHelpFormatter, \ help='synchronize with given gmail account') @@ -150,65 +152,65 @@ def test_command_parser(): sync_parser.add_argument('-l', '--email', action='store', dest='email', help='email to sync with') # sync typ sync_parser.add_argument('-t', '--type', action='store', default='full-sync', help='type of synchronisation') - + sync_parser.add_argument("-i", "--imap-server", metavar = "HOSTNAME", \ help="Gmail imap server hostname. (default: imap.gmail.com)",\ dest="host", default="imap.gmail.com") - + sync_parser.add_argument("-p", "--imap-port", metavar = "PORT", \ help="Gmail imap server port. (default: 993)",\ dest="port", default=993) - + sync_parser.set_defaults(verb='sync') - + sync_parser.epilogue = SYNC_HELP_EPILOGUE - + # A restore command restore_parser = subparsers.add_parser('restore', help='restore email to a given email account') restore_parser.add_argument('email', action='store', help='email to sync with') restore_parser.add_argument('--recursive', '-r', default=False, action='store_true', help='Remove the contents of the directory, too', ) - + restore_parser.set_defaults(verb='restore') - + # A config command config_parser = subparsers.add_parser('config', help='add/delete/modify properties in configuration') config_parser.add_argument('dirname', action='store', help='New directory to create') config_parser.add_argument('--read-only', default=False, action='store_true', help='Set permissions to prevent writing to the directory', ) - + config_parser.set_defaults(verb='config') - - - - + + + + # global help #print("================ Global Help (-h)================") sys.argv = ['gmvault.py'] - print(parser.parse_args()) - + print((parser.parse_args())) + #print("================ Global Help (--help)================") #sys.argv = ['gmvault.py', '--help'] #print(parser.parse_args()) - + #print("================ Sync Help (--help)================") #sys.argv = ['gmvault.py', 'sync', '-h'] #print(parser.parse_args()) - + #sys.argv = ['gmvault.py', 'sync', 'guillaume.aubert@gmail.com', '--type', 'quick-sync'] - + #print(parser.parse_args()) #print("options = %s\n" % (options)) #print("args = %s\n" % (args)) - + if __name__ == '__main__': - + test_command_parser() - - + + diff --git a/src/gmv/collections_utils.py b/src/gmv/collections_utils.py index 91d1210b..78a9faf4 100755 --- a/src/gmv/collections_utils.py +++ b/src/gmv/collections_utils.py @@ -16,7 +16,9 @@ along with this program. If not, see . ''' +from __future__ import absolute_import import collections +from six.moves import map ## {{{ http://code.activestate.com/recipes/576669/ (r18) class OrderedDict(dict, collections.MutableMapping): @@ -91,34 +93,34 @@ class Map(object): specific multimaps are subclassed. """ def __init__(self): self._dict = {} - + def __repr__(self): return "%s(%s)" % (self.__class__.__name__, repr(self._dict)) - + __str__ = __repr__ - + def __getitem__(self, key): return self._dict[key] - + def __setitem__(self, key, value): self._dict[key] = value - + def __delitem__(self, key): del self._dict[key] def __len__(self): return len(self._dict) - + def remove(self, key, value): #pylint: disable=W0613 '''remove key from Map''' del self._dict[key] - + def keys(self): '''returns list of keys''' - return self._dict.keys() - + return list(self._dict.keys()) + def dict(self): - """ Allows access to internal dictionary, if necessary. Caution: multimaps + """ Allows access to internal dictionary, if necessary. Caution: multimaps will break if keys are not associated with proper container.""" return self._dict @@ -127,13 +129,13 @@ class ListMultimap(Map): def __init__(self): super(ListMultimap, self).__init__() self._dict = collections.defaultdict(list) - + def __setitem__(self, key, value): self._dict[key].append(value) def __len__(self): return len(self._dict) - + def remove(self, key, value): '''Remove key''' self._dict[key].remove(value) @@ -143,13 +145,13 @@ class SetMultimap(Map): def __init__(self): super(SetMultimap, self).__init__() self._dict = collections.defaultdict(set) - + def __setitem__(self, key, value): self._dict[key].add(value) def __len__(self): return len(self._dict) - + def remove(self, key, value): '''remove key''' self._dict[key].remove(value) @@ -159,13 +161,13 @@ class DictMultimap(Map): def __init__(self): super(DictMultimap, self).__init__() self._dict = collections.defaultdict(dict) - + def __setitem__(self, key, value): self._dict[key][value] = True def __len__(self): return len(self._dict) - + def remove(self, key, value): """ remove key""" del self._dict[key][value] diff --git a/src/gmv/conf/conf_helper.py b/src/gmv/conf/conf_helper.py index 7efc79a2..9cf04e29 100755 --- a/src/gmv/conf/conf_helper.py +++ b/src/gmv/conf/conf_helper.py @@ -16,87 +16,89 @@ along with this program. If not, see . ''' +from __future__ import absolute_import, print_function + import sys import os import re import codecs import gmv.conf.exceptions as exceptions -import gmv.conf.utils.struct_parser as struct_parser +import gmv.conf.utils.struct_parser as struct_parser class ResourceError(Exception): """ - Base class for ressource exceptions + Base class for ressource exceptions """ def __init__(self, a_msg): - + super(ResourceError, self).__init__(a_msg) class Resource(object): """ Class read a ressource. - It can be read first from the Command Line, then from the ENV as an env variable and finally from a conf file + It can be read first from the Command Line, then from the ENV as an env variable and finally from a conf file """ - - def __init__(self, a_cli_argument=None, a_env_variable=None, a_conf_property=None): - """ + + def __init__(self, a_cli_argument=None, a_env_variable=None, a_conf_property=None): + """ Default Constructor. It is important to understand that there is precedence between the different ways to set the ressource: - get from the command line if defined otherwise get from the Env variable if defined otherwise get from the conf file otherwise error - + Args: a_cli_argument : The command line argument name a_env_variable : The env variable name used for this ressource a_conf_property: It should be a tuple containing two elements (group,property) """ - + self._cli_arg = a_cli_argument.lower() if a_cli_argument is not None else None self._env_var = a_env_variable.upper() if a_env_variable is not None else None - + if a_conf_property is not None: (self._conf_group, self._conf_property) = a_conf_property else: self._conf_group = None self._conf_property = None - + def set_cli_argument(self, a_cli_argument): """cli_argument setter""" self._cli_arg = a_cli_argument.lower() - + def set_env_variable(self, a_env_variable): """env_variable setter""" self._env_var = a_env_variable - + @classmethod def _get_srandardized_cli_argument(cls, a_tostrip): """ - remove -- or - from the command line argument and add a -- prefix to standardize the cli argument + remove -- or - from the command line argument and add a -- prefix to standardize the cli argument """ the_str = a_tostrip - + while the_str.startswith('-'): the_str = the_str[1:] - + return '--%s' % (the_str) - + def _get_value_from_command_line(self): """ internal method for extracting the value from the command line. All command line agruments must be lower case (unix style). To Do support short and long cli args. - + Returns: the Value if defined otherwise None """ - + # check precondition if self._cli_arg == None: return None - + the_s = Resource._get_srandardized_cli_argument(self._cli_arg) - + # look for cliArg in sys argv for arg in sys.argv: if arg.lower() == the_s: @@ -104,28 +106,28 @@ def _get_value_from_command_line(self): #print "i = %d, val = %s\n"%(i,sys.argv[i]) if len(sys.argv) <= i: # No more thing to read in the command line so quit - print "Resource: Commandline argument %s has no value\n" % (self._cli_arg) - return None + print("Resource: Commandline argument %s has no value\n" % (self._cli_arg)) + return None else: #print "i+1 = %d, val = %s\n"%(i+1,sys.argv[i+1]) return sys.argv[i+1] - + def _get_value_from_env(self): """ internal method for extracting the value from the env. All support ENV Variables should be in uppercase. - + Returns: the Value if defined otherwise None """ - + # precondition if self._env_var == None: return None - + return os.environ.get(self._env_var, None) - + def _get_from_conf(self): """ Try to read the info from the Configuration if possible @@ -133,24 +135,24 @@ def _get_from_conf(self): if self._conf_group and self._conf_property: if Conf.can_be_instanciated(): return Conf.get_instance().get(self._conf_group, self._conf_property) - + return None - - + + def get_value(self, a_raise_exception=True): """ Return the value of the Resource as a string. - get from the command line if defined otherwise get from the Env variable if defined otherwise get from the conf file otherwise error - + Arguments: aRaiseException: flag indicating if an exception should be raise if value not found Returns: value of the Resource as a String - + Raises: exception CTBTOError if the aRaiseExceptionOnError flag is activated """ - + # get a value using precedence rule 1) command-line, 2) ENV, 3) Conf val = self._get_value_from_command_line() if val is None: @@ -158,47 +160,47 @@ def get_value(self, a_raise_exception=True): if val is None: val = self._get_from_conf() if (val is None) and a_raise_exception: - + the_str = "Cannot find " add_nor = 0 - + if self._cli_arg is not None: the_str += "commandline argument %s" % (self._cli_arg) add_nor += 1 - + if self._env_var is not None: - + if add_nor > 0: the_str += ", nor " - + the_str += "the Env Variable %s" % (self._env_var) add_nor += 1 - + if self._conf_group is not None: if add_nor > 0: the_str += ", nor " - + the_str += "the Conf Group:[%s] and Property=%s" % (self._conf_group, self._conf_property) add_nor += 1 - + if add_nor == 0: the_str += " any defined commandline argument, nor any env variable or"\ " Conf group and properties. They are all None, fatal error" else: the_str += ". One of them should be defined" - + raise ResourceError(the_str) - + return val - + def _get(self, conv): """ Private _get method used to convert to the right expected type (int,float or boolean). Strongly inspired by ConfigParser.py - + Returns: value converted into the asked type - + Raises: exception ValueError if conversion issue """ @@ -207,10 +209,10 @@ def _get(self, conv): def get_value_as_int(self): """ Return the value as an int - + Returns: value converted into the asked type - + Raises: exception ValueError if conversion issue """ @@ -219,10 +221,10 @@ def get_value_as_int(self): def get_value_as_float(self): """ Return the value as a float - + Returns: value converted into the asked type - + Raises: exception ValueError if conversion issue """ @@ -234,16 +236,16 @@ def get_value_as_float(self): def get_value_as_boolean(self): """ Return the value as a boolean - + Returns: value converted into the asked type - + Raises: exception ValueError if conversion issue """ val = self.get_value() if val.lower() not in self._boolean_states: - raise ValueError, 'Not a boolean: %s' % val + raise ValueError('Not a boolean: %s' % val) return self._boolean_states[val.lower()] class MockConf(object): @@ -255,36 +257,36 @@ def __init__(self, use_resource=True): default constructor """ pass - + @classmethod def get(cls, section, option, default=None, fail_if_missing=False): #pylint: disable=W0613 """ get one option from a section. """ return default - + @classmethod def print_content(cls, substitute_values = True):#pylint: disable=W0613 """ print all the options variables substituted. - + :param a_substitue_vals: bool for substituting values :returns: the string containing all sections and variables """ - raise exceptions.Error("Not implemented in MockupConf") + raise exceptions.Error("Not implemented in MockupConf") @classmethod def items(cls, section):#pylint: disable=W0613 """ return all items from a section. Items is a list of tuples (option,value) - + Args: section. The section where to find the option - + Returns: a list of tuples (option,value) - + Raises: exception NoSectionError if the section cannot be found """ - raise exceptions.Error("Not implemented in MockupConf") - + raise exceptions.Error("Not implemented in MockupConf") + @classmethod def getint(cls, section, option, default=0, fail_if_missing=False):#pylint: disable=W0613 """Return the int value of the option. @@ -293,7 +295,7 @@ def getint(cls, section, option, default=0, fail_if_missing=False):#pylint: disa @classmethod def getfloat(cls, section, option, default=0, fail_if_missing=False):#pylint: disable=W0613 - """Return the float value of the option. + """Return the float value of the option. Default value is 0, None value can't be used as default value""" return default @@ -301,12 +303,12 @@ def getfloat(cls, section, option, default=0, fail_if_missing=False):#pylint: di def getboolean(cls, section, option, default=False, fail_if_missing=False):#pylint: disable=W0613 """get bool value """ return default - + @classmethod def get_list(cls, section, option, default=None, fail_if_missing=False):#pylint: disable=W0613 """ get a list of string, int """ return default - + @classmethod def getlist(cls, section, option, default=None, fail_if_missing=False):#pylint: disable=W0613 """ Deprecated, use get_list instead""" @@ -316,89 +318,89 @@ def getlist(cls, section, option, default=None, fail_if_missing=False):#pylint: def getdict(cls, section, option, default=None, fail_if_missing=False):#pylint: disable=W0613 """ Deprecated, use get_dict instead""" return cls.get_dict(section, option, default, fail_if_missing) - - + + @classmethod def get_dict(cls, section, option, default=None, fail_if_missing=False):#pylint: disable=W0613 """ get a dict """ return default - + class Conf(object): """ Configuration Object with a several features: - + * get configuration info in different types * support for import * support for variables in configuration file * support for default values in all accessors * integrated with the resources object offering to get the configuration from an env var, a commandline option or the conf - * to be done : support for blocs, list comprehension and dict comprehension, json + * to be done : support for blocs, list comprehension and dict comprehension, json * to be done : define resources in the conf using the [Resource] group with A= { ENV:TESTVAR, CLI:--testvar, VAL:1.234 } - + """ # command line and env resource stuff CLINAME = "--conf_file" - ENVNAME = "CONF_FILE" - + ENVNAME = "CONF_FILE" + #class member _instance = None - + _CLIGROUP = "CLI" _ENVGROUP = "ENV" _MAX_INCLUDE_DEPTH = 10 - + @classmethod def get_instance(cls): """ singleton method """ if cls._instance == None: cls._instance = Conf() return cls._instance - + @classmethod def can_be_instanciated(cls): - """Class method used by the Resource to check that the Conf can be instantiated. - - These two objects have a special contract as they are strongly coupled. + """Class method used by the Resource to check that the Conf can be instantiated. + + These two objects have a special contract as they are strongly coupled. A Resource can use the Conf to check for a Resource and the Conf uses a Resource to read Conf filepath. - + :returns: True if the Conf file has got a file. - + :except Error: Base Conf Error - + """ #No conf info passed to the resource so the Resource will not look into the conf (to avoid recursive search) the_res = Resource(cls.CLINAME, cls.ENVNAME) - + filepath = the_res.get_value(a_raise_exception=False) - + if (filepath is not None) and os.path.exists(filepath): return True - + return False - - + + def __init__(self, use_resource=True): """ Constructor """ - + # create resource for the conf file self._conf_resource = Resource(Conf.CLINAME, Conf.ENVNAME) - + # list of sections self._sections = {} - + self._configuration_file_path = None - - # create config object - if use_resource: + + # create config object + if use_resource: self._load_config() def _load_config(self, a_file=None): """ _load the configuration file """ - try: + try: # get it from a Resource if not files are passed if a_file is None: - a_file = self._conf_resource.get_value() + a_file = self._conf_resource.get_value() if a_file is None: raise exceptions.Error("Conf. Error, need a configuration file path") @@ -409,29 +411,29 @@ def _load_config(self, a_file=None): # memorize conf file path self._configuration_file_path = a_file - except Exception, exce: - print "Can't read the config file %s" % a_file - print "Current executing from dir = %s\n" % os.getcwd() + except Exception as exce: + print("Can't read the config file %s" % a_file) + print("Current executing from dir = %s\n" % os.getcwd()) raise exce def get_conf_file_path(self): """return conf_file_path""" return self._configuration_file_path if self._configuration_file_path != None else "unknown" - + def sections(self): """Return a list of section names, excluding [DEFAULT]""" # self._sections will never have [DEFAULT] in it - return self._sections.keys() - + return list(self._sections.keys()) + @classmethod def _get_defaults(cls, section, option, default, fail_if_missing): """ To manage defaults. Args: default. The default value to return if fail_if_missing is False fail_if_missing. Throw an exception when the option is not found and fail_if_missing is true - + Returns: default if fail_if_missing is False - + Raises: exception NoOptionError if fail_if_missing is True """ @@ -442,26 +444,26 @@ def _get_defaults(cls, section, option, default, fail_if_missing): return str(default) else: return None - + def get(self, section, option, default=None, fail_if_missing=False): """ get one option from a section. - + return the default if it is not found and if fail_if_missing is False, otherwise return NoOptionError - + :param section: Section where to find the option :type section: str :param option: Option to get :param default: Default value to return if fail_if_missing is False :param fail_if_missing: Will throw an exception when the option is not found and fail_if_missing is true - + :returns: the option as a string - + :except NoOptionError: Raised only when fail_is_missing set to True - + """ # all options are kept in lowercase opt = self.optionxform(option) - + if section not in self._sections: #check if it is a ENV section dummy = None @@ -477,17 +479,17 @@ def get(self, section, option, default=None, fail_if_missing=False): return self._replace_vars(self._sections[section][opt], "%s[%s]" % (section, option), - 1) else: return self._get_defaults(section, opt, default, fail_if_missing) - - + + def print_content(self, substitute_values = True): """ print all the options variables substituted. - + :param a_substitue_vals: bool for substituting values :returns: the string containing all sections and variables """ - + result_str = "" - + for section_name in self._sections: result_str += "[%s]\n" % (section_name) section = self._sections[section_name] @@ -497,20 +499,20 @@ def print_content(self, substitute_values = True): result_str += "%s = %s\n" % (option, self.get(section_name, option)) else: result_str += "%s = %s\n" % (option, self._sections[section_name][option]) - + result_str += "\n" - + return result_str - + def items(self, section): """ return all items from a section. Items is a list of tuples (option,value) - + Args: section. The section where to find the option - + Returns: a list of tuples (option,value) - + Raises: exception NoSectionError if the section cannot be found """ @@ -521,9 +523,9 @@ def items(self, section): # remove __name__ from d if "__name__" in a_copy: del a_copy["__name__"] - - return a_copy.items() - + + return list(a_copy.items()) + except KeyError: raise exceptions.NoSectionError(section) @@ -534,35 +536,35 @@ def has_option(self, section, option): option = self.optionxform(option) has_option = (option in self._sections[section]) return has_option - + def has_section(self, section): """Check for the existence of a given section in the configuration.""" has_section = False if section in self._sections: has_section = True return has_section - + @classmethod def _get_closing_bracket_index(cls, index, the_str, location, lineno): """ private method used by _replace_vars to count the closing brackets. - + Args: index. The index from where to look for a closing bracket s. The string to parse group. group and options that are substituted. Mainly used to create a nice exception message option. option that is substituted. Mainly used to create a nice exception message - + Returns: the index of the found closing bracket - + Raises: exception NoSectionError if the section cannot be found """ - + tolook = the_str[index + 2:] - + opening_brack = 1 closing_brack_index = index + 2 - + i = 0 for _ch in tolook: if _ch == ')': @@ -570,63 +572,63 @@ def _get_closing_bracket_index(cls, index, the_str, location, lineno): return closing_brack_index else: opening_brack -= 1 - + elif _ch == '(': if tolook[i - 1] == '%': opening_brack += 1 - + # inc index closing_brack_index += 1 i += 1 - + raise exceptions.SubstitutionError(lineno, location, "Missing a closing bracket in %s" % (tolook)) # very permissive regex _SUBSGROUPRE = re.compile(r"%\((?P\w*)\[(?P