diff --git a/amalgamate.py b/amalgamate.py index 9e3f08c..550f6a6 100644 --- a/amalgamate.py +++ b/amalgamate.py @@ -56,7 +56,7 @@ def amalgamate_source( source_top_dir=None, target_source_path: output .cpp path header_include_path: generated header path relative to target_source_path. """ - print ("Amalgating header...") + print("Amalgating header...") header = AmalgamationFile( source_top_dir ) header.add_text( "/// Json-cpp amalgated header (http://jsoncpp.sourceforge.net/)." ) header.add_text( "/// It is intented to be used with #include <%s>" % header_include_path ) @@ -77,12 +77,12 @@ def amalgamate_source( source_top_dir=None, header.add_text( "#endif //ifndef JSON_AMALGATED_H_INCLUDED" ) target_header_path = os.path.join( os.path.dirname(target_source_path), header_include_path ) - print ("Writing amalgated header to %r" % target_header_path) + print("Writing amalgated header to %r" % target_header_path) header.write_to( target_header_path ) base, ext = os.path.splitext( header_include_path ) forward_header_include_path = base + "-forwards" + ext - print ("Amalgating forward header...") + print("Amalgating forward header...") header = AmalgamationFile( source_top_dir ) header.add_text( "/// Json-cpp amalgated forward header (http://jsoncpp.sourceforge.net/)." ) header.add_text( "/// It is intented to be used with #include <%s>" % forward_header_include_path ) @@ -99,10 +99,10 @@ def amalgamate_source( source_top_dir=None, target_forward_header_path = os.path.join( os.path.dirname(target_source_path), forward_header_include_path ) - print ("Writing amalgated forward header to %r" % target_forward_header_path) + print("Writing amalgated forward header to %r" % target_forward_header_path) header.write_to( target_forward_header_path ) - print ("Amalgating source...") + print("Amalgating source...") source = AmalgamationFile( source_top_dir ) source.add_text( "/// Json-cpp amalgated source (http://jsoncpp.sourceforge.net/)." ) source.add_text( "/// It is intented to be used with #include <%s>" % header_include_path ) @@ -118,7 +118,7 @@ def amalgamate_source( source_top_dir=None, source.add_file( os.path.join(lib_json, "json_value.cpp") ) source.add_file( os.path.join(lib_json, "json_writer.cpp") ) - print ("Writing amalgated source to %r" % target_source_path) + print("Writing amalgated source to %r" % target_source_path) source.write_to( target_source_path ) def main(): @@ -144,7 +144,7 @@ Generate a single amalgated source and header file from the sources. sys.stderr.write( msg + "\n" ) sys.exit( 1 ) else: - print ("Source succesfully amalagated") + print("Source succesfully amalagated") if __name__ == "__main__": main() diff --git a/devtools/antglob.py b/devtools/antglob.py index 30837b5..8b7b4ca 100644 --- a/devtools/antglob.py +++ b/devtools/antglob.py @@ -2,6 +2,7 @@ # encoding: utf-8 # Baptiste Lepilleur, 2009 +from __future__ import print_function from dircache import listdir import re import fnmatch @@ -190,12 +191,12 @@ if __name__ == "__main__": test_cases.append( (ant_pattern, local_path(accepted_matches), local_path( rejected_matches )) ) for ant_pattern, accepted_matches, rejected_matches in test_cases: rex = ant_pattern_to_re( ant_pattern ) - print 'ant_pattern:', ant_pattern, ' => ', rex.pattern + print('ant_pattern:', ant_pattern, ' => ', rex.pattern) for accepted_match in accepted_matches: - print 'Accepted?:', accepted_match - self.assert_( rex.match( accepted_match ) is not None ) + print('Accepted?:', accepted_match) + self.assertTrue( rex.match( accepted_match ) is not None ) for rejected_match in rejected_matches: - print 'Rejected?:', rejected_match - self.assert_( rex.match( rejected_match ) is None ) + print('Rejected?:', rejected_match) + self.assertTrue( rex.match( rejected_match ) is None ) unittest.main() diff --git a/devtools/batchbuild.py b/devtools/batchbuild.py index 5da74db..6f57945 100644 --- a/devtools/batchbuild.py +++ b/devtools/batchbuild.py @@ -1,280 +1,281 @@ -import collections -import itertools -import json -import os -import os.path -import re -import shutil -import string -import subprocess -import sys -import cgi - -class BuildDesc: - def __init__(self, prepend_envs=None, variables=None, build_type=None, generator=None): - self.prepend_envs = prepend_envs or [] # [ { "var": "value" } ] - self.variables = variables or [] - self.build_type = build_type - self.generator = generator - - def merged_with( self, build_desc ): - """Returns a new BuildDesc by merging field content. - Prefer build_desc fields to self fields for single valued field. - """ - return BuildDesc( self.prepend_envs + build_desc.prepend_envs, - self.variables + build_desc.variables, - build_desc.build_type or self.build_type, - build_desc.generator or self.generator ) - - def env( self ): - environ = os.environ.copy() - for values_by_name in self.prepend_envs: - for var, value in values_by_name.items(): - var = var.upper() - if type(value) is unicode: - value = value.encode( sys.getdefaultencoding() ) - if var in environ: - environ[var] = value + os.pathsep + environ[var] - else: - environ[var] = value - return environ - - def cmake_args( self ): - args = ["-D%s" % var for var in self.variables] - # skip build type for Visual Studio solution as it cause warning - if self.build_type and 'Visual' not in self.generator: - args.append( "-DCMAKE_BUILD_TYPE=%s" % self.build_type ) - if self.generator: - args.extend( ['-G', self.generator] ) - return args - - def __repr__( self ): - return "BuildDesc( %s, build_type=%s )" % (" ".join( self.cmake_args()), self.build_type) - -class BuildData: - def __init__( self, desc, work_dir, source_dir ): - self.desc = desc - self.work_dir = work_dir - self.source_dir = source_dir - self.cmake_log_path = os.path.join( work_dir, 'batchbuild_cmake.log' ) - self.build_log_path = os.path.join( work_dir, 'batchbuild_build.log' ) - self.cmake_succeeded = False - self.build_succeeded = False - - def execute_build(self): - print 'Build %s' % self.desc - self._make_new_work_dir( ) - self.cmake_succeeded = self._generate_makefiles( ) - if self.cmake_succeeded: - self.build_succeeded = self._build_using_makefiles( ) - return self.build_succeeded - - def _generate_makefiles(self): - print ' Generating makefiles: ', - cmd = ['cmake'] + self.desc.cmake_args( ) + [os.path.abspath( self.source_dir )] - succeeded = self._execute_build_subprocess( cmd, self.desc.env(), self.cmake_log_path ) - print 'done' if succeeded else 'FAILED' - return succeeded - - def _build_using_makefiles(self): - print ' Building:', - cmd = ['cmake', '--build', self.work_dir] - if self.desc.build_type: - cmd += ['--config', self.desc.build_type] - succeeded = self._execute_build_subprocess( cmd, self.desc.env(), self.build_log_path ) - print 'done' if succeeded else 'FAILED' - return succeeded - - def _execute_build_subprocess(self, cmd, env, log_path): - process = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=self.work_dir, - env=env ) - stdout, _ = process.communicate( ) - succeeded = (process.returncode == 0) - with open( log_path, 'wb' ) as flog: - log = ' '.join( cmd ) + '\n' + stdout + '\nExit code: %r\n' % process.returncode - flog.write( fix_eol( log ) ) - return succeeded - - def _make_new_work_dir(self): - if os.path.isdir( self.work_dir ): - print ' Removing work directory', self.work_dir - shutil.rmtree( self.work_dir, ignore_errors=True ) - if not os.path.isdir( self.work_dir ): - os.makedirs( self.work_dir ) - -def fix_eol( stdout ): - """Fixes wrong EOL produced by cmake --build on Windows (\r\r\n instead of \r\n). - """ - return re.sub( '\r*\n', os.linesep, stdout ) - -def load_build_variants_from_config( config_path ): - with open( config_path, 'rb' ) as fconfig: - data = json.load( fconfig ) - variants = data[ 'cmake_variants' ] - build_descs_by_axis = collections.defaultdict( list ) - for axis in variants: - axis_name = axis["name"] - build_descs = [] - if "generators" in axis: - for generator_data in axis["generators"]: - for generator in generator_data["generator"]: - build_desc = BuildDesc( generator=generator, - prepend_envs=generator_data.get("env_prepend") ) - build_descs.append( build_desc ) - elif "variables" in axis: - for variables in axis["variables"]: - build_desc = BuildDesc( variables=variables ) - build_descs.append( build_desc ) - elif "build_types" in axis: - for build_type in axis["build_types"]: - build_desc = BuildDesc( build_type=build_type ) - build_descs.append( build_desc ) - build_descs_by_axis[axis_name].extend( build_descs ) - return build_descs_by_axis - -def generate_build_variants( build_descs_by_axis ): - """Returns a list of BuildDesc generated for the partial BuildDesc for each axis.""" - axis_names = build_descs_by_axis.keys() - build_descs = [] - for axis_name, axis_build_descs in build_descs_by_axis.items(): - if len(build_descs): - # for each existing build_desc and each axis build desc, create a new build_desc - new_build_descs = [] - for prototype_build_desc, axis_build_desc in itertools.product( build_descs, axis_build_descs): - new_build_descs.append( prototype_build_desc.merged_with( axis_build_desc ) ) - build_descs = new_build_descs - else: - build_descs = axis_build_descs - return build_descs - -HTML_TEMPLATE = string.Template(''' - - $title - - - - - - - - $th_vars - - - - $th_build_types - - - -$tr_builds - -
Variables
Build type
-''') - -def generate_html_report( html_report_path, builds ): - report_dir = os.path.dirname( html_report_path ) - # Vertical axis: generator - # Horizontal: variables, then build_type - builds_by_generator = collections.defaultdict( list ) - variables = set() - build_types_by_variable = collections.defaultdict( set ) - build_by_pos_key = {} # { (generator, var_key, build_type): build } - for build in builds: - builds_by_generator[build.desc.generator].append( build ) - var_key = tuple(sorted(build.desc.variables)) - variables.add( var_key ) - build_types_by_variable[var_key].add( build.desc.build_type ) - pos_key = (build.desc.generator, var_key, build.desc.build_type) - build_by_pos_key[pos_key] = build - variables = sorted( variables ) - th_vars = [] - th_build_types = [] - for variable in variables: - build_types = sorted( build_types_by_variable[variable] ) - nb_build_type = len(build_types_by_variable[variable]) - th_vars.append( '%s' % (nb_build_type, cgi.escape( ' '.join( variable ) ) ) ) - for build_type in build_types: - th_build_types.append( '%s' % cgi.escape(build_type) ) - tr_builds = [] - for generator in sorted( builds_by_generator ): - tds = [ '%s\n' % cgi.escape( generator ) ] - for variable in variables: - build_types = sorted( build_types_by_variable[variable] ) - for build_type in build_types: - pos_key = (generator, variable, build_type) - build = build_by_pos_key.get(pos_key) - if build: - cmake_status = 'ok' if build.cmake_succeeded else 'FAILED' - build_status = 'ok' if build.build_succeeded else 'FAILED' - cmake_log_url = os.path.relpath( build.cmake_log_path, report_dir ) - build_log_url = os.path.relpath( build.build_log_path, report_dir ) - td = 'CMake: %s' % ( - build_status.lower(), cmake_log_url, cmake_status.lower(), cmake_status) - if build.cmake_succeeded: - td += '
Build: %s' % ( - build_log_url, build_status.lower(), build_status) - td += '' - else: - td = '' - tds.append( td ) - tr_builds.append( '%s' % '\n'.join( tds ) ) - html = HTML_TEMPLATE.substitute( - title='Batch build report', - th_vars=' '.join(th_vars), - th_build_types=' '.join( th_build_types), - tr_builds='\n'.join( tr_builds ) ) - with open( html_report_path, 'wt' ) as fhtml: - fhtml.write( html ) - print 'HTML report generated in:', html_report_path - -def main(): - usage = r"""%prog WORK_DIR SOURCE_DIR CONFIG_JSON_PATH [CONFIG2_JSON_PATH...] -Build a given CMake based project located in SOURCE_DIR with multiple generators/options.dry_run -as described in CONFIG_JSON_PATH building in WORK_DIR. - -Example of call: -python devtools\batchbuild.py e:\buildbots\jsoncpp\build . devtools\agent_vmw7.json -""" - from optparse import OptionParser - parser = OptionParser(usage=usage) - parser.allow_interspersed_args = True -# parser.add_option('-v', '--verbose', dest="verbose", action='store_true', -# help="""Be verbose.""") - parser.enable_interspersed_args() - options, args = parser.parse_args() - if len(args) < 3: - parser.error( "Missing one of WORK_DIR SOURCE_DIR CONFIG_JSON_PATH." ) - work_dir = args[0] - source_dir = args[1].rstrip('/\\') - config_paths = args[2:] - for config_path in config_paths: - if not os.path.isfile( config_path ): - parser.error( "Can not read: %r" % config_path ) - - # generate build variants - build_descs = [] - for config_path in config_paths: - build_descs_by_axis = load_build_variants_from_config( config_path ) - build_descs.extend( generate_build_variants( build_descs_by_axis ) ) - print 'Build variants (%d):' % len(build_descs) - # assign build directory for each variant - if not os.path.isdir( work_dir ): - os.makedirs( work_dir ) - builds = [] - with open( os.path.join( work_dir, 'matrix-dir-map.txt' ), 'wt' ) as fmatrixmap: - for index, build_desc in enumerate( build_descs ): - build_desc_work_dir = os.path.join( work_dir, '%03d' % (index+1) ) - builds.append( BuildData( build_desc, build_desc_work_dir, source_dir ) ) - fmatrixmap.write( '%s: %s\n' % (build_desc_work_dir, build_desc) ) - for build in builds: - build.execute_build() - html_report_path = os.path.join( work_dir, 'batchbuild-report.html' ) - generate_html_report( html_report_path, builds ) - print 'Done' - - -if __name__ == '__main__': - main() - +from __future__ import print_function +import collections +import itertools +import json +import os +import os.path +import re +import shutil +import string +import subprocess +import sys +import cgi + +class BuildDesc: + def __init__(self, prepend_envs=None, variables=None, build_type=None, generator=None): + self.prepend_envs = prepend_envs or [] # [ { "var": "value" } ] + self.variables = variables or [] + self.build_type = build_type + self.generator = generator + + def merged_with( self, build_desc ): + """Returns a new BuildDesc by merging field content. + Prefer build_desc fields to self fields for single valued field. + """ + return BuildDesc( self.prepend_envs + build_desc.prepend_envs, + self.variables + build_desc.variables, + build_desc.build_type or self.build_type, + build_desc.generator or self.generator ) + + def env( self ): + environ = os.environ.copy() + for values_by_name in self.prepend_envs: + for var, value in list(values_by_name.items()): + var = var.upper() + if type(value) is unicode: + value = value.encode( sys.getdefaultencoding() ) + if var in environ: + environ[var] = value + os.pathsep + environ[var] + else: + environ[var] = value + return environ + + def cmake_args( self ): + args = ["-D%s" % var for var in self.variables] + # skip build type for Visual Studio solution as it cause warning + if self.build_type and 'Visual' not in self.generator: + args.append( "-DCMAKE_BUILD_TYPE=%s" % self.build_type ) + if self.generator: + args.extend( ['-G', self.generator] ) + return args + + def __repr__( self ): + return "BuildDesc( %s, build_type=%s )" % (" ".join( self.cmake_args()), self.build_type) + +class BuildData: + def __init__( self, desc, work_dir, source_dir ): + self.desc = desc + self.work_dir = work_dir + self.source_dir = source_dir + self.cmake_log_path = os.path.join( work_dir, 'batchbuild_cmake.log' ) + self.build_log_path = os.path.join( work_dir, 'batchbuild_build.log' ) + self.cmake_succeeded = False + self.build_succeeded = False + + def execute_build(self): + print('Build %s' % self.desc) + self._make_new_work_dir( ) + self.cmake_succeeded = self._generate_makefiles( ) + if self.cmake_succeeded: + self.build_succeeded = self._build_using_makefiles( ) + return self.build_succeeded + + def _generate_makefiles(self): + print(' Generating makefiles: ', end=' ') + cmd = ['cmake'] + self.desc.cmake_args( ) + [os.path.abspath( self.source_dir )] + succeeded = self._execute_build_subprocess( cmd, self.desc.env(), self.cmake_log_path ) + print('done' if succeeded else 'FAILED') + return succeeded + + def _build_using_makefiles(self): + print(' Building:', end=' ') + cmd = ['cmake', '--build', self.work_dir] + if self.desc.build_type: + cmd += ['--config', self.desc.build_type] + succeeded = self._execute_build_subprocess( cmd, self.desc.env(), self.build_log_path ) + print('done' if succeeded else 'FAILED') + return succeeded + + def _execute_build_subprocess(self, cmd, env, log_path): + process = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=self.work_dir, + env=env ) + stdout, _ = process.communicate( ) + succeeded = (process.returncode == 0) + with open( log_path, 'wb' ) as flog: + log = ' '.join( cmd ) + '\n' + stdout + '\nExit code: %r\n' % process.returncode + flog.write( fix_eol( log ) ) + return succeeded + + def _make_new_work_dir(self): + if os.path.isdir( self.work_dir ): + print(' Removing work directory', self.work_dir) + shutil.rmtree( self.work_dir, ignore_errors=True ) + if not os.path.isdir( self.work_dir ): + os.makedirs( self.work_dir ) + +def fix_eol( stdout ): + """Fixes wrong EOL produced by cmake --build on Windows (\r\r\n instead of \r\n). + """ + return re.sub( '\r*\n', os.linesep, stdout ) + +def load_build_variants_from_config( config_path ): + with open( config_path, 'rb' ) as fconfig: + data = json.load( fconfig ) + variants = data[ 'cmake_variants' ] + build_descs_by_axis = collections.defaultdict( list ) + for axis in variants: + axis_name = axis["name"] + build_descs = [] + if "generators" in axis: + for generator_data in axis["generators"]: + for generator in generator_data["generator"]: + build_desc = BuildDesc( generator=generator, + prepend_envs=generator_data.get("env_prepend") ) + build_descs.append( build_desc ) + elif "variables" in axis: + for variables in axis["variables"]: + build_desc = BuildDesc( variables=variables ) + build_descs.append( build_desc ) + elif "build_types" in axis: + for build_type in axis["build_types"]: + build_desc = BuildDesc( build_type=build_type ) + build_descs.append( build_desc ) + build_descs_by_axis[axis_name].extend( build_descs ) + return build_descs_by_axis + +def generate_build_variants( build_descs_by_axis ): + """Returns a list of BuildDesc generated for the partial BuildDesc for each axis.""" + axis_names = list(build_descs_by_axis.keys()) + build_descs = [] + for axis_name, axis_build_descs in list(build_descs_by_axis.items()): + if len(build_descs): + # for each existing build_desc and each axis build desc, create a new build_desc + new_build_descs = [] + for prototype_build_desc, axis_build_desc in itertools.product( build_descs, axis_build_descs): + new_build_descs.append( prototype_build_desc.merged_with( axis_build_desc ) ) + build_descs = new_build_descs + else: + build_descs = axis_build_descs + return build_descs + +HTML_TEMPLATE = string.Template(''' + + $title + + + + + + + + $th_vars + + + + $th_build_types + + + +$tr_builds + +
Variables
Build type
+''') + +def generate_html_report( html_report_path, builds ): + report_dir = os.path.dirname( html_report_path ) + # Vertical axis: generator + # Horizontal: variables, then build_type + builds_by_generator = collections.defaultdict( list ) + variables = set() + build_types_by_variable = collections.defaultdict( set ) + build_by_pos_key = {} # { (generator, var_key, build_type): build } + for build in builds: + builds_by_generator[build.desc.generator].append( build ) + var_key = tuple(sorted(build.desc.variables)) + variables.add( var_key ) + build_types_by_variable[var_key].add( build.desc.build_type ) + pos_key = (build.desc.generator, var_key, build.desc.build_type) + build_by_pos_key[pos_key] = build + variables = sorted( variables ) + th_vars = [] + th_build_types = [] + for variable in variables: + build_types = sorted( build_types_by_variable[variable] ) + nb_build_type = len(build_types_by_variable[variable]) + th_vars.append( '%s' % (nb_build_type, cgi.escape( ' '.join( variable ) ) ) ) + for build_type in build_types: + th_build_types.append( '%s' % cgi.escape(build_type) ) + tr_builds = [] + for generator in sorted( builds_by_generator ): + tds = [ '%s\n' % cgi.escape( generator ) ] + for variable in variables: + build_types = sorted( build_types_by_variable[variable] ) + for build_type in build_types: + pos_key = (generator, variable, build_type) + build = build_by_pos_key.get(pos_key) + if build: + cmake_status = 'ok' if build.cmake_succeeded else 'FAILED' + build_status = 'ok' if build.build_succeeded else 'FAILED' + cmake_log_url = os.path.relpath( build.cmake_log_path, report_dir ) + build_log_url = os.path.relpath( build.build_log_path, report_dir ) + td = 'CMake: %s' % ( + build_status.lower(), cmake_log_url, cmake_status.lower(), cmake_status) + if build.cmake_succeeded: + td += '
Build: %s' % ( + build_log_url, build_status.lower(), build_status) + td += '' + else: + td = '' + tds.append( td ) + tr_builds.append( '%s' % '\n'.join( tds ) ) + html = HTML_TEMPLATE.substitute( + title='Batch build report', + th_vars=' '.join(th_vars), + th_build_types=' '.join( th_build_types), + tr_builds='\n'.join( tr_builds ) ) + with open( html_report_path, 'wt' ) as fhtml: + fhtml.write( html ) + print('HTML report generated in:', html_report_path) + +def main(): + usage = r"""%prog WORK_DIR SOURCE_DIR CONFIG_JSON_PATH [CONFIG2_JSON_PATH...] +Build a given CMake based project located in SOURCE_DIR with multiple generators/options.dry_run +as described in CONFIG_JSON_PATH building in WORK_DIR. + +Example of call: +python devtools\batchbuild.py e:\buildbots\jsoncpp\build . devtools\agent_vmw7.json +""" + from optparse import OptionParser + parser = OptionParser(usage=usage) + parser.allow_interspersed_args = True +# parser.add_option('-v', '--verbose', dest="verbose", action='store_true', +# help="""Be verbose.""") + parser.enable_interspersed_args() + options, args = parser.parse_args() + if len(args) < 3: + parser.error( "Missing one of WORK_DIR SOURCE_DIR CONFIG_JSON_PATH." ) + work_dir = args[0] + source_dir = args[1].rstrip('/\\') + config_paths = args[2:] + for config_path in config_paths: + if not os.path.isfile( config_path ): + parser.error( "Can not read: %r" % config_path ) + + # generate build variants + build_descs = [] + for config_path in config_paths: + build_descs_by_axis = load_build_variants_from_config( config_path ) + build_descs.extend( generate_build_variants( build_descs_by_axis ) ) + print('Build variants (%d):' % len(build_descs)) + # assign build directory for each variant + if not os.path.isdir( work_dir ): + os.makedirs( work_dir ) + builds = [] + with open( os.path.join( work_dir, 'matrix-dir-map.txt' ), 'wt' ) as fmatrixmap: + for index, build_desc in enumerate( build_descs ): + build_desc_work_dir = os.path.join( work_dir, '%03d' % (index+1) ) + builds.append( BuildData( build_desc, build_desc_work_dir, source_dir ) ) + fmatrixmap.write( '%s: %s\n' % (build_desc_work_dir, build_desc) ) + for build in builds: + build.execute_build() + html_report_path = os.path.join( work_dir, 'batchbuild-report.html' ) + generate_html_report( html_report_path, builds ) + print('Done') + + +if __name__ == '__main__': + main() + diff --git a/devtools/fixeol.py b/devtools/fixeol.py index 4fed6ce..53af761 100644 --- a/devtools/fixeol.py +++ b/devtools/fixeol.py @@ -1,3 +1,4 @@ +from __future__ import print_function import os.path def fix_source_eol( path, is_dry_run = True, verbose = True, eol = '\n' ): @@ -6,8 +7,8 @@ def fix_source_eol( path, is_dry_run = True, verbose = True, eol = '\n' ): raise ValueError( 'Path "%s" is not a file' % path ) try: f = open(path, 'rb') - except IOError, msg: - print >> sys.stderr, "%s: I/O Error: %s" % (file, str(msg)) + except IOError as msg: + print("%s: I/O Error: %s" % (file, str(msg)), file=sys.stderr) return False try: raw_lines = f.readlines() @@ -15,7 +16,7 @@ def fix_source_eol( path, is_dry_run = True, verbose = True, eol = '\n' ): f.close() fixed_lines = [line.rstrip('\r\n') + eol for line in raw_lines] if raw_lines != fixed_lines: - print '%s =>' % path, + print('%s =>' % path, end=' ') if not is_dry_run: f = open(path, "wb") try: @@ -23,7 +24,7 @@ def fix_source_eol( path, is_dry_run = True, verbose = True, eol = '\n' ): finally: f.close() if verbose: - print is_dry_run and ' NEED FIX' or ' FIXED' + print(is_dry_run and ' NEED FIX' or ' FIXED') return True ## ## diff --git a/devtools/licenseupdater.py b/devtools/licenseupdater.py index 866eada..8cb71d7 100644 --- a/devtools/licenseupdater.py +++ b/devtools/licenseupdater.py @@ -1,5 +1,6 @@ """Updates the license text in source file. """ +from __future__ import print_function # An existing license is found if the file starts with the string below, # and ends with the first blank line. @@ -34,11 +35,11 @@ def update_license( path, dry_run, show_diff ): if not dry_run: with open( path, 'wb' ) as fout: fout.write( new_text.replace('\n', newline ) ) - print 'Updated', path + print('Updated', path) if show_diff: import difflib - print '\n'.join( difflib.unified_diff( original_text.split('\n'), - new_text.split('\n') ) ) + print('\n'.join( difflib.unified_diff( original_text.split('\n'), + new_text.split('\n') ) )) return True return False @@ -83,7 +84,7 @@ python devtools\licenseupdater.py include src parser.enable_interspersed_args() options, args = parser.parse_args() update_license_in_source_directories( args, options.dry_run, options.show_diff ) - print 'Done' + print('Done') if __name__ == '__main__': import sys diff --git a/doxybuild.py b/doxybuild.py index 588767f..0b61c39 100644 --- a/doxybuild.py +++ b/doxybuild.py @@ -1,12 +1,12 @@ """Script to generate doxygen documentation. """ - +from __future__ import print_function +from devtools import tarball import re import os import os.path import sys import shutil -from devtools import tarball def find_program(*filenames): """find a program in folders path_lst, and sets env[var] @@ -33,9 +33,9 @@ def do_subst_in_file(targetfile, sourcefile, dict): contents = f.read() f.close() except: - print "Can't read source file %s"%sourcefile + print("Can't read source file %s"%sourcefile) raise - for (k,v) in dict.items(): + for (k,v) in list(dict.items()): v = v.replace('\\','\\\\') contents = re.sub(k, v, contents) try: @@ -43,7 +43,7 @@ def do_subst_in_file(targetfile, sourcefile, dict): f.write(contents) f.close() except: - print "Can't write target file %s"%targetfile + print("Can't write target file %s"%targetfile) raise def run_doxygen(doxygen_path, config_file, working_dir, is_silent): @@ -53,12 +53,12 @@ def run_doxygen(doxygen_path, config_file, working_dir, is_silent): try: os.chdir( working_dir ) cmd = [doxygen_path, config_file] - print 'Running:', ' '.join( cmd ) + print('Running:', ' '.join( cmd )) try: import subprocess except: if os.system( ' '.join( cmd ) ) != 0: - print 'Documentation generation failed' + print('Documentation generation failed') return False else: if is_silent: @@ -67,8 +67,8 @@ def run_doxygen(doxygen_path, config_file, working_dir, is_silent): process = subprocess.Popen( cmd ) stdout, _ = process.communicate() if process.returncode: - print 'Documentation generation failed:' - print stdout + print('Documentation generation failed:') + print(stdout) return False return True finally: @@ -107,7 +107,7 @@ def build_doc( options, make_release=False ): } if os.path.isdir( output_dir ): - print 'Deleting directory:', output_dir + print('Deleting directory:', output_dir) shutil.rmtree( output_dir ) if not os.path.isdir( output_dir ): os.makedirs( output_dir ) @@ -115,15 +115,15 @@ def build_doc( options, make_release=False ): do_subst_in_file( 'doc/doxyfile', 'doc/doxyfile.in', subst_keys ) ok = run_doxygen( options.doxygen_path, 'doc/doxyfile', 'doc', is_silent=options.silent ) if not options.silent: - print open(warning_log_path, 'rb').read() + print(open(warning_log_path, 'rb').read()) index_path = os.path.abspath(os.path.join('doc', subst_keys['%HTML_OUTPUT%'], 'index.html')) - print 'Generated documentation can be found in:' - print index_path + print('Generated documentation can be found in:') + print(index_path) if options.open: import webbrowser webbrowser.open( 'file://' + index_path ) if options.make_tarball: - print 'Generating doc tarball to', tarball_path + print('Generating doc tarball to', tarball_path) tarball_sources = [ output_dir, 'README.txt', diff --git a/makerelease.py b/makerelease.py index e44e439..90276d1 100644 --- a/makerelease.py +++ b/makerelease.py @@ -14,6 +14,7 @@ python makerelease.py 0.5.0 0.6.0-dev Note: This was for Subversion. Now that we are in GitHub, we do not need to build versioned tarballs anymore, so makerelease.py is defunct. """ +from __future__ import print_function import os.path import subprocess import sys @@ -46,7 +47,7 @@ class SVNError(Exception): def svn_command( command, *args ): cmd = ['svn', '--non-interactive', command] + list(args) - print 'Running:', ' '.join( cmd ) + print('Running:', ' '.join( cmd )) process = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) @@ -84,7 +85,7 @@ def svn_check_if_tag_exist( tag_url ): """ try: list_stdout = svn_command( 'list', tag_url ) - except SVNError, e: + except SVNError as e: if e.returncode != 1 or not str(e).find('tag_url'): raise e # otherwise ignore error, meaning tag does not exist @@ -117,7 +118,7 @@ def svn_export( tag_url, export_dir ): def fix_sources_eol( dist_dir ): """Set file EOL for tarball distribution. """ - print 'Preparing exported source file EOL for distribution...' + print('Preparing exported source file EOL for distribution...') prune_dirs = antglob.prune_dirs + 'scons-local* ./build* ./libs ./dist' win_sources = antglob.glob( dist_dir, includes = '**/*.sln **/*.vcproj', @@ -148,7 +149,7 @@ def download( url, target_path ): def check_compile( distcheck_top_dir, platform ): cmd = [sys.executable, 'scons.py', 'platform=%s' % platform, 'check'] - print 'Running:', ' '.join( cmd ) + print('Running:', ' '.join( cmd )) log_path = os.path.join( distcheck_top_dir, 'build-%s.log' % platform ) flog = open( log_path, 'wb' ) try: @@ -179,9 +180,9 @@ def run_sftp_batch( userhost, sftp, batch, retry=0 ): # psftp -agent -C blep,jsoncpp@web.sourceforge.net -batch -b batch.sftp -bc cmd = [sftp, '-agent', '-C', '-batch', '-b', path, '-bc', userhost] error = None - for retry_index in xrange(0, max(1,retry)): + for retry_index in range(0, max(1,retry)): heading = retry_index == 0 and 'Running:' or 'Retrying:' - print heading, ' '.join( cmd ) + print(heading, ' '.join( cmd )) process = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) stdout = process.communicate()[0] if process.returncode != 0: @@ -219,21 +220,21 @@ exit upload_paths = set( [os.path.basename(p) for p in antglob.glob( doc_dir )] ) paths_to_remove = existing_paths - upload_paths if paths_to_remove: - print 'Removing the following file from web:' - print '\n'.join( paths_to_remove ) + print('Removing the following file from web:') + print('\n'.join( paths_to_remove )) stdout = run_sftp_batch( userhost, sftp, """cd htdocs rm %s exit""" % ' '.join(paths_to_remove) ) - print 'Uploading %d files:' % len(upload_paths) + print('Uploading %d files:' % len(upload_paths)) batch_size = 10 upload_paths = list(upload_paths) start_time = time.time() - for index in xrange(0,len(upload_paths),batch_size): + for index in range(0,len(upload_paths),batch_size): paths = upload_paths[index:index+batch_size] file_per_sec = (time.time() - start_time) / (index+1) remaining_files = len(upload_paths) - index remaining_sec = file_per_sec * remaining_files - print '%d/%d, ETA=%.1fs' % (index+1, len(upload_paths), remaining_sec) + print('%d/%d, ETA=%.1fs' % (index+1, len(upload_paths), remaining_sec)) run_sftp_batch( userhost, sftp, """cd htdocs lcd %s mput %s @@ -297,7 +298,7 @@ Warning: --force should only be used when developping/testing the release script else: msg = check_no_pending_commit() if not msg: - print 'Setting version to', release_version + print('Setting version to', release_version) set_version( release_version ) svn_commit( 'Release ' + release_version ) tag_url = svn_join_url( SVN_TAG_ROOT, release_version ) @@ -305,11 +306,11 @@ Warning: --force should only be used when developping/testing the release script if options.retag_release: svn_remove_tag( tag_url, 'Overwriting previous tag' ) else: - print 'Aborting, tag %s already exist. Use --retag to overwrite it!' % tag_url + print('Aborting, tag %s already exist. Use --retag to overwrite it!' % tag_url) sys.exit( 1 ) svn_tag_sandbox( tag_url, 'Release ' + release_version ) - print 'Generated doxygen document...' + print('Generated doxygen document...') ## doc_dirname = r'jsoncpp-api-html-0.5.0' ## doc_tarball_path = r'e:\prg\vc\Lib\jsoncpp-trunk\dist\jsoncpp-api-html-0.5.0.tar.gz' doc_tarball_path, doc_dirname = doxybuild.build_doc( options, make_release=True ) @@ -323,11 +324,11 @@ Warning: --force should only be used when developping/testing the release script source_dir = 'jsoncpp-src-' + release_version source_tarball_path = 'dist/%s.tar.gz' % source_dir - print 'Generating source tarball to', source_tarball_path + print('Generating source tarball to', source_tarball_path) tarball.make_tarball( source_tarball_path, [export_dir], export_dir, prefix_dir=source_dir ) amalgamation_tarball_path = 'dist/%s-amalgamation.tar.gz' % source_dir - print 'Generating amalgamation source tarball to', amalgamation_tarball_path + print('Generating amalgamation source tarball to', amalgamation_tarball_path) amalgamation_dir = 'dist/amalgamation' amalgamate.amalgamate_source( export_dir, '%s/jsoncpp.cpp' % amalgamation_dir, 'json/json.h' ) amalgamation_source_dir = 'jsoncpp-src-amalgamation' + release_version @@ -337,41 +338,41 @@ Warning: --force should only be used when developping/testing the release script # Decompress source tarball, download and install scons-local distcheck_dir = 'dist/distcheck' distcheck_top_dir = distcheck_dir + '/' + source_dir - print 'Decompressing source tarball to', distcheck_dir + print('Decompressing source tarball to', distcheck_dir) rmdir_if_exist( distcheck_dir ) tarball.decompress( source_tarball_path, distcheck_dir ) scons_local_path = 'dist/scons-local.tar.gz' - print 'Downloading scons-local to', scons_local_path + print('Downloading scons-local to', scons_local_path) download( SCONS_LOCAL_URL, scons_local_path ) - print 'Decompressing scons-local to', distcheck_top_dir + print('Decompressing scons-local to', distcheck_top_dir) tarball.decompress( scons_local_path, distcheck_top_dir ) # Run compilation - print 'Compiling decompressed tarball' + print('Compiling decompressed tarball') all_build_status = True for platform in options.platforms.split(','): - print 'Testing platform:', platform + print('Testing platform:', platform) build_status, log_path = check_compile( distcheck_top_dir, platform ) - print 'see build log:', log_path - print build_status and '=> ok' or '=> FAILED' + print('see build log:', log_path) + print(build_status and '=> ok' or '=> FAILED') all_build_status = all_build_status and build_status if not build_status: - print 'Testing failed on at least one platform, aborting...' + print('Testing failed on at least one platform, aborting...') svn_remove_tag( tag_url, 'Removing tag due to failed testing' ) sys.exit(1) if options.user: if not options.no_web: - print 'Uploading documentation using user', options.user + print('Uploading documentation using user', options.user) sourceforge_web_synchro( SOURCEFORGE_PROJECT, doc_distcheck_top_dir, user=options.user, sftp=options.sftp ) - print 'Completed documentation upload' - print 'Uploading source and documentation tarballs for release using user', options.user + print('Completed documentation upload') + print('Uploading source and documentation tarballs for release using user', options.user) sourceforge_release_tarball( SOURCEFORGE_PROJECT, [source_tarball_path, doc_tarball_path], user=options.user, sftp=options.sftp ) - print 'Source and doc release tarballs uploaded' + print('Source and doc release tarballs uploaded') else: - print 'No upload user specified. Web site and download tarbal were not uploaded.' - print 'Tarball can be found at:', doc_tarball_path + print('No upload user specified. Web site and download tarbal were not uploaded.') + print('Tarball can be found at:', doc_tarball_path) # Set next version number and commit set_version( next_version ) diff --git a/scons-tools/substinfile.py b/scons-tools/substinfile.py index 4d30585..ef18b4e 100644 --- a/scons-tools/substinfile.py +++ b/scons-tools/substinfile.py @@ -1,5 +1,6 @@ import re from SCons.Script import * # the usual scons stuff you get in a SConscript +import collections def generate(env): """ @@ -25,28 +26,28 @@ def generate(env): contents = f.read() f.close() except: - raise SCons.Errors.UserError, "Can't read source file %s"%sourcefile - for (k,v) in dict.items(): + raise SCons.Errors.UserError("Can't read source file %s"%sourcefile) + for (k,v) in list(dict.items()): contents = re.sub(k, v, contents) try: f = open(targetfile, 'wb') f.write(contents) f.close() except: - raise SCons.Errors.UserError, "Can't write target file %s"%targetfile + raise SCons.Errors.UserError("Can't write target file %s"%targetfile) return 0 # success def subst_in_file(target, source, env): - if not env.has_key('SUBST_DICT'): - raise SCons.Errors.UserError, "SubstInFile requires SUBST_DICT to be set." + if 'SUBST_DICT' not in env: + raise SCons.Errors.UserError("SubstInFile requires SUBST_DICT to be set.") d = dict(env['SUBST_DICT']) # copy it - for (k,v) in d.items(): - if callable(v): + for (k,v) in list(d.items()): + if isinstance(v, collections.Callable): d[k] = env.subst(v()).replace('\\','\\\\') elif SCons.Util.is_String(v): d[k] = env.subst(v).replace('\\','\\\\') else: - raise SCons.Errors.UserError, "SubstInFile: key %s: %s must be a string or callable"%(k, repr(v)) + raise SCons.Errors.UserError("SubstInFile: key %s: %s must be a string or callable"%(k, repr(v))) for (t,s) in zip(target, source): return do_subst_in_file(str(t), str(s), d) @@ -60,8 +61,8 @@ def generate(env): Returns original target, source tuple unchanged. """ d = env['SUBST_DICT'].copy() # copy it - for (k,v) in d.items(): - if callable(v): + for (k,v) in list(d.items()): + if isinstance(v, collections.Callable): d[k] = env.subst(v()) elif SCons.Util.is_String(v): d[k]=env.subst(v) diff --git a/test/generate_expected.py b/test/generate_expected.py index 5b215c4..f668da2 100644 --- a/test/generate_expected.py +++ b/test/generate_expected.py @@ -1,11 +1,12 @@ +from __future__ import print_function import glob import os.path for path in glob.glob( '*.json' ): text = file(path,'rt').read() target = os.path.splitext(path)[0] + '.expected' if os.path.exists( target ): - print 'skipping:', target + print('skipping:', target) else: - print 'creating:', target + print('creating:', target) file(target,'wt').write(text) diff --git a/test/pyjsontestrunner.py b/test/pyjsontestrunner.py index 504f3db..3f08a8a 100644 --- a/test/pyjsontestrunner.py +++ b/test/pyjsontestrunner.py @@ -1,12 +1,12 @@ # Simple implementation of a json test runner to run the test against json-py. - +from __future__ import print_function import sys import os.path import json import types if len(sys.argv) != 2: - print "Usage: %s input-json-file", sys.argv[0] + print("Usage: %s input-json-file", sys.argv[0]) sys.exit(3) input_path = sys.argv[1] diff --git a/test/runjsontests.py b/test/runjsontests.py index ffe8bd5..a1f6082 100644 --- a/test/runjsontests.py +++ b/test/runjsontests.py @@ -1,3 +1,4 @@ +from __future__ import print_function import sys import os import os.path @@ -11,7 +12,7 @@ def compareOutputs( expected, actual, message ): actual = actual.strip().replace('\r','').split('\n') diff_line = 0 max_line_to_compare = min( len(expected), len(actual) ) - for index in xrange(0,max_line_to_compare): + for index in range(0,max_line_to_compare): if expected[index].strip() != actual[index].strip(): diff_line = index + 1 break @@ -34,7 +35,7 @@ def compareOutputs( expected, actual, message ): def safeReadFile( path ): try: return file( path, 'rt' ).read() - except IOError, e: + except IOError as e: return '' % (path,e) def runAllTests( jsontest_executable_path, input_dir = None, @@ -51,7 +52,7 @@ def runAllTests( jsontest_executable_path, input_dir = None, for input_path in tests + test_jsonchecker: expect_failure = os.path.basename( input_path ).startswith( 'fail' ) is_json_checker_test = (input_path in test_jsonchecker) or expect_failure - print 'TESTING:', input_path, + print('TESTING:', input_path, end=' ') options = is_json_checker_test and '--json-checker' or '' pipe = os.popen( "%s%s %s %s" % ( valgrind_path, jsontest_executable_path, options, @@ -61,24 +62,24 @@ def runAllTests( jsontest_executable_path, input_dir = None, if is_json_checker_test: if expect_failure: if status is None: - print 'FAILED' + print('FAILED') failed_tests.append( (input_path, 'Parsing should have failed:\n%s' % safeReadFile(input_path)) ) else: - print 'OK' + print('OK') else: if status is not None: - print 'FAILED' + print('FAILED') failed_tests.append( (input_path, 'Parsing failed:\n' + process_output) ) else: - print 'OK' + print('OK') else: base_path = os.path.splitext(input_path)[0] actual_output = safeReadFile( base_path + '.actual' ) actual_rewrite_output = safeReadFile( base_path + '.actual-rewrite' ) file(base_path + '.process-output','wt').write( process_output ) if status: - print 'parsing failed' + print('parsing failed') failed_tests.append( (input_path, 'Parsing failed:\n' + process_output) ) else: expected_output_path = os.path.splitext(input_path)[0] + '.expected' @@ -86,23 +87,23 @@ def runAllTests( jsontest_executable_path, input_dir = None, detail = ( compareOutputs( expected_output, actual_output, 'input' ) or compareOutputs( expected_output, actual_rewrite_output, 'rewrite' ) ) if detail: - print 'FAILED' + print('FAILED') failed_tests.append( (input_path, detail) ) else: - print 'OK' + print('OK') if failed_tests: - print - print 'Failure details:' + print() + print('Failure details:') for failed_test in failed_tests: - print '* Test', failed_test[0] - print failed_test[1] - print - print 'Test results: %d passed, %d failed.' % (len(tests)-len(failed_tests), - len(failed_tests) ) + print('* Test', failed_test[0]) + print(failed_test[1]) + print() + print('Test results: %d passed, %d failed.' % (len(tests)-len(failed_tests), + len(failed_tests) )) return 1 else: - print 'All %d tests passed.' % len(tests) + print('All %d tests passed.' % len(tests)) return 0 def main(): diff --git a/test/rununittests.py b/test/rununittests.py index 366184c..6279f80 100644 --- a/test/rununittests.py +++ b/test/rununittests.py @@ -1,8 +1,9 @@ +from __future__ import print_function +from glob import glob import sys import os import os.path import subprocess -from glob import glob import optparse VALGRIND_CMD = 'valgrind --tool=memcheck --leak-check=yes --undef-value-errors=yes' @@ -28,29 +29,29 @@ def runAllTests( exe_path, use_valgrind=False ): test_proxy = TestProxy( exe_path, use_valgrind=use_valgrind ) status, test_names = test_proxy.run( ['--list-tests'] ) if not status: - print >> sys.stderr, "Failed to obtain unit tests list:\n" + test_names + print("Failed to obtain unit tests list:\n" + test_names, file=sys.stderr) return 1 test_names = [name.strip() for name in test_names.strip().split('\n')] failures = [] for name in test_names: - print 'TESTING %s:' % name, + print('TESTING %s:' % name, end=' ') succeed, result = test_proxy.run( ['--test', name] ) if succeed: - print 'OK' + print('OK') else: failures.append( (name, result) ) - print 'FAILED' + print('FAILED') failed_count = len(failures) pass_count = len(test_names) - failed_count if failed_count: - print + print() for name, result in failures: - print result - print '%d/%d tests passed (%d failure(s))' % ( - pass_count, len(test_names), failed_count) + print(result) + print('%d/%d tests passed (%d failure(s))' % ( + pass_count, len(test_names), failed_count)) return 1 else: - print 'All %d tests passed' % len(test_names) + print('All %d tests passed' % len(test_names)) return 0 def main():