GME: build.py Source File

00001 #!/usr/bin/python
00002 #
00003 # Copyright (c) 2006-2008 ISIS, Vanderbilt University
00004 #
00005 # Author: Peter Volgyesi (peter.volgyesi@vanderbilt.edu)
00006 #
00007 """GME Build System - central module"""
00008 
00009 import sys
00010 import os, os.path
00011 import getopt
00012 from prefs import prefs
00013 import tools
00014 import win32com
00015 
00016 
00017 #
00018 # Constants
00019 #
00020 GME_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
00021 
00022 paradigms_root = os.path.join(GME_ROOT, "Paradigms")
00023 METAGME_XMP = os.path.join(paradigms_root, "MetaGME", "MetaGME.xmp")
00024 UML_XMP = os.path.join(paradigms_root, "UML", "UML.xmp")
00025 SF_XMP = os.path.join(paradigms_root, "SF", "SF.xmp")
00026 HFSM_XMP = os.path.join(paradigms_root, "HFSM", "HFSM.xmp")
00027 def replace_ext(file, newext):
00028     import os.path
00029     return os.path.splitext(file)[0] + "." + newext
00030 def mta_for_xmp(file):
00031     return replace_ext(file, "mta")
00032 
00033 #
00034 # Build steps
00035 #
00036 
00037 def check_prerequisites():
00038     "Check prerequisites (required tools, etc.)"
00039     
00040     # Test for GME_ROOT environment variable
00041     if not os.environ['GME_ROOT']:
00042         print "GME_ROOT environment variable is not set! (It should point to the GMESRC folder)"
00043         raise
00044         
00045     if not os.environ['JAVA_HOME']:
00046         print "JAVA_HOME environment variable is not set! (It should point to the JDK root folder)"
00047         raise
00048         
00049     if os.path.normpath(os.path.abspath(os.environ['GME_ROOT'])) != GME_ROOT:
00050         print "GME_ROOT environment variable is not set to the current dev. source tree!"
00051         print "GME_ROOT =", os.environ['GME_ROOT']
00052         print "Current dev. source tree:", GME_ROOT
00053         raise
00054     
00055     # Test for Microsoft Visual Studio 2010
00056     try:
00057         tools.test_VS()
00058     except:
00059         print "Microsoft Visual Studio 2010 is not installed!"
00060         raise
00061     
00062     # Test for zip utility
00063     try:
00064         tools.test_zip()
00065     except:
00066         print "ZIP utility cannot be found!"
00067         raise
00068     
00069     # Test for InstallShield
00070     try:
00071         tools.test_WiX()
00072     except:
00073         print "WiX toolset cannot be found in your path!"
00074         raise
00075     
00076     # Test for SVN
00077     try:
00078         tools.test_SVN()
00079     except:
00080         print "Warning: SVN client cannot be found! You won't be able to check in & tag releases"
00081 
00082 
00083 def update_version_str():
00084     "Update version strings in source files"
00085     with open(os.path.join(GME_ROOT, 'GME/Gme/GMEVersion.h.tmpl')) as template:
00086         template_text = "".join(template.readlines())
00087     old_text = None
00088     try:
00089         with open(os.path.join(GME_ROOT, 'GME/Gme/GMEVersion.h')) as header:
00090             old_text = "".join(header.readlines())
00091     except:
00092         pass
00093     text = template_text % (prefs["version_major"], prefs["version_minor"], prefs["version_patch"], prefs["version_build"])
00094     if text != old_text:
00095         with open(os.path.join(GME_ROOT, 'GME/Gme/GMEVersion.h'), 'w') as header:
00096             header.write(text)
00097 
00098 def _remove_dlldata_from_tlog():
00099     ''' Workaround for http://connect.microsoft.com/VisualStudio/feedback/details/763929/incremental-build-of-idl-files-behavior-changes-after-installing-visual-studio-2012
00100     Incremental build of IDL files behavior changes after installing Visual Studio 2012
00101     MSBuild always rebuilds the project because dlldata.c is included in the file tracker log (midl.read.1.tlog) of the project
00102     '''
00103     tlog = os.path.join(GME_ROOT, "GME", "Interfaces", "Release", "midl.read.1.tlog")
00104     if not os.path.isfile(tlog):
00105         return
00106     import codecs
00107     with codecs.open(tlog, encoding='utf-16-le') as f:
00108         lines = f.readlines()
00109         lines = [line for line in lines if line.find(u'DLLDATA.C') == -1]
00110     with codecs.open(tlog, 'w', encoding='utf-16-le') as f:
00111         for line in lines: f.write(line)
00112         
00113 def compile_GME():
00114     "Compile GME core components"
00115     if prefs['arch'] == 'x64':
00116         # To use 32bit Python.exe for the tests, 32bit CoreCollectionHandler must be registered
00117         sln_file = os.path.join(GME_ROOT, "GME", "GME.sln")
00118         _remove_dlldata_from_tlog()
00119         tools.build_VS(sln_file, 'Release', arch='Win32', target='Core')
00120         _remove_dlldata_from_tlog()
00121         tools.system(['regsvr32', '/s', os.path.join(GME_ROOT, "GME", "Release", "Core.dll")])
00122     sln_file = os.path.join(GME_ROOT, "GME", "GME.sln")
00123     _remove_dlldata_from_tlog()
00124     tools.build_VS( sln_file, "Release" )
00125     _remove_dlldata_from_tlog()
00126     sln_file = os.path.join(GME_ROOT, "GME", "GMEDecorators.sln")
00127     tools.build_VS( sln_file, "Release" )
00128     cmd_dir = os.path.join(GME_ROOT, "GME")
00129     tools.system( ['call', 'regrelease.bat'] + (['x64'] if prefs['arch'] == 'x64' else []) + ['<NUL'], cmd_dir)
00130     sln_file = os.path.join(GME_ROOT, "GME", "DotNetPIAs", "DotNetPIAs.vcxproj")
00131     tools.build_VS( sln_file, "Release" )
00132 
00133 def _Release_PGO_dir():
00134     if prefs['arch'] == 'x64':
00135         return os.path.join(GME_ROOT, 'GME', 'x64', 'Release_PGO')
00136     else:
00137         return os.path.join(GME_ROOT, 'GME', 'Release_PGO')
00138 
00139 def newer (source, target):
00140     """Return true if 'source' exists and is more recently modified than
00141        'target', or if 'source' exists and 'target' doesn't.  Return
00142        false if both exist and 'target' is the same age or younger than
00143        'source'. """
00144 
00145     if not os.path.exists (target):
00146         return 1
00147 
00148     from stat import ST_MTIME
00149     mtime1 = os.stat(source)[ST_MTIME]
00150     mtime2 = os.stat(target)[ST_MTIME]
00151 
00152     return mtime1 > mtime2
00153 
00154 def copy_if_newer(source, target):
00155     import shutil
00156     if newer(source, target):
00157         shutil.copyfile(source, target)
00158 
00159 def compile_GME_PGO_Instrument():
00160     "Compile GME core components (PGO Instrument)"
00161     # e.g. Dispatch("MGA.MgaParser") locks Parser.dll. FreeLibary it here
00162     import ctypes
00163     ctypes.windll.ole32.CoFreeUnusedLibraries()
00164     import errno
00165     try:
00166         os.makedirs(_Release_PGO_dir())
00167     except OSError as exc:
00168         if exc.errno != errno.EEXIST:
00169             raise
00170     VC_path = os.path.join(prefs['VS_dir'], r"VC\bin\%s" % (prefs['arch'] == 'x64' and 'amd64\\' or ''))
00171     copy_if_newer(os.path.join(VC_path, r"pgort%s0.dll" % prefs['toolset']), os.path.join(_Release_PGO_dir(), 'pgort%s0.dll' % prefs['toolset']))
00172     sln_file = os.path.join(GME_ROOT, "GME", "GME.sln")
00173     _remove_dlldata_from_tlog()
00174     tools.build_VS(sln_file, "Release_PGO_Instrument")
00175     _remove_dlldata_from_tlog()
00176     cmd_dir = os.path.join(GME_ROOT, "GME")
00177     tools.system( ['call', 'regPGO.bat'] + (['x64'] if prefs['arch'] == 'x64' else []) + ['<NUL'], cmd_dir)
00178 
00179 def compile_GME_PGO_Optimize():
00180     "Compile GME core components (PGO Optimize)"
00181     sln_file = os.path.join(GME_ROOT, "GME", "GME.sln")
00182     _remove_dlldata_from_tlog()
00183     tools.build_VS( sln_file, "Release_PGO_Optimize" )
00184     _remove_dlldata_from_tlog()
00185 
00186 def PGO_train():
00187     "Run tests/Create training data for the PGO binaries"
00188     import glob
00189     for file in glob.glob(GME_ROOT + '\\GME' + ('\\x64' if prefs['arch'] == 'x64' else '') + '\\Release_PGO\\*.pgc'):
00190         os.remove(file)
00191     tools.system([sys.executable, '-m', 'GPyUnit.__main__', '-x'] + (['-a', 'x64'] if prefs['arch'] == 'x64' else []), os.path.join(GME_ROOT, 'Tests'))
00192     if prefs['arch'] == 'x64':
00193         # wait for dllhost.exe to exit
00194         import time
00195         time.sleep(31)
00196 
00197 def compile_meta():
00198     "Compile MetaGME components"
00199     sln_file = os.path.join(GME_ROOT, "Paradigms", "MetaGME", "MetaGME.sln")
00200     tools.build_VS( sln_file, "Release" )
00201     cmd_dir = os.path.join(GME_ROOT, "Paradigms", "MetaGME")
00202     tools.system( ['call', 'regrelease.bat'] + (['x64'] if prefs['arch'] == 'x64' else []) + ['<NUL'], cmd_dir)
00203 
00204         
00205 def compile_JBON():
00206     "Compile Java component support (JBON)"
00207     if prefs['arch'] == 'x64': return
00208     sln_file = os.path.join(GME_ROOT, "SDK", "Java", "native", "JavaSupport.sln")
00209     tools.build_VS( sln_file, "Release" )
00210 
00211         
00212 def compile_tools():
00213     "Compile external tool components"
00214     
00215     # Auto Layout
00216     sln_file = os.path.join(GME_ROOT, "Tools", "AutoLayout", "AutoLayout.sln")
00217     tools.build_VS( sln_file, "Release" )
00218 
00219     sln_file = os.path.join(GME_ROOT, "SDK", "DotNet", "DsmlGenerator", "DsmlGenerator.sln")
00220     tools.build_VS(sln_file, "Release", arch='Any CPU', msbuild=(prefs['arch'] == 'x64' and tools.MSBUILD.replace('Framework', 'Framework64') or tools.MSBUILD))
00221 
00222     sln_file = os.path.join(GME_ROOT, "Tools", "DumpWMF", "DumpWMF.sln")
00223     tools.build_VS(sln_file, "Release", arch='Any CPU', msbuild=(prefs['arch'] == 'x64' and tools.MSBUILD.replace('Framework', 'Framework64') or tools.MSBUILD))
00224 
00225     if prefs['arch'] == 'x64':
00226         tools.system([r'%windir%\Microsoft.NET\Framework64\v4.0.30319\RegAsm.exe', '/codebase',
00227                       os.path.join(GME_ROOT, 'Tools', 'DumpWMF', 'bin', 'Release', 'DumpWMF.dll')])
00228         import _winreg
00229         with _winreg.OpenKey(_winreg.HKEY_CLASSES_ROOT, r"CLSID\{A051FEEA-E310-3F6A-8D71-A55E3F4F2E14}", 0, _winreg.KEY_WRITE | _winreg.KEY_WOW64_64KEY) as key:
00230             _winreg.SetValueEx(key, "AppID", 0, _winreg.REG_SZ, "{461F30AF-3BF0-11D4-B3F0-005004D38590}")
00231 
00232         tools.system([r'%windir%\Microsoft.NET\Framework64\v4.0.30319\RegAsm.exe', '/codebase',
00233                       os.path.join(GME_ROOT, 'SDK', 'DotNet', 'DsmlGenerator', 'CSharpDsmlGenerator', 'bin', 'Release', 'CSharpDSMLGenerator.dll')])
00234         return
00235 
00236     # Table Editor
00237     sln_file = os.path.join(GME_ROOT, "Tools", "TableEditor", "TableEditor.sln")
00238     tools.build_VS( sln_file, "Release" )
00239     
00240     # GMEplink
00241     sln_file = os.path.join(GME_ROOT, "Tools", "GMEplink", "GMEplink.sln")
00242     tools.build_VS( sln_file, "Release" )
00243 
00244     sln_file = os.path.join(GME_ROOT, "SDK", "DotNet", "CSharpComponentWizard", "CSharpComponentWizard.sln")
00245     tools.build_VS( sln_file, "Release" )
00246 
00247 def compile_samples():
00248     "Compile sample components"
00249     
00250     # UML Paradigm
00251     sln_file = os.path.join(GME_ROOT, "Paradigms", "UML", "decorator", "UMLDecorator.sln")
00252     tools.build_VS( sln_file, "Release" )
00253 
00254     if prefs['arch'] == 'x64': return
00255 
00256     sln_file = os.path.join(GME_ROOT, "SDK", "PatternProcessor", "PatternProcessor.sln")
00257     tools.build_VS( sln_file, "Release" )
00258     
00259     # SF Paradigm
00260     sln_file = os.path.join(GME_ROOT, "Paradigms", "SF", "SFInterpreter", "SFInterpreter.sln")
00261     tools.build_VS( sln_file, "Release" )
00262     sln_file = os.path.join(GME_ROOT, "Paradigms", "SF", "BON2SFSample", "BON2SFSample.sln")
00263     tools.build_VS( sln_file, "Release" )
00264     sln_file = os.path.join(GME_ROOT, "Paradigms", "SF", "BON2SFInterpreter", "BON2SFInterpreter.sln")
00265     tools.build_VS( sln_file, "Release" )
00266     
00267     # HFSM Paradigm
00268     sln_file = os.path.join(GME_ROOT, "Paradigms", "HFSM", "HFSMSimulator", "HFSMSimulator.sln")
00269     tools.build_VS( sln_file, "Release" )
00270     
00271 
00272 def zip_decorsamples():
00273     "Create PlainDecoratorSample.zip"
00274     zip_dir = os.path.join(GME_ROOT, "SDK", "Decorator Examples", "PlainSample")
00275     tools.zip(zip_dir, "PlainDecoratorSample.zip", "packagefiles.lst")
00276 
00277     "Create NewDecoratorSample.zip"
00278     zip_dir = os.path.join(GME_ROOT, "SDK", "Decorator Examples", "NewSample")
00279     tools.zip(zip_dir, "NewDecoratorSample.zip", "packagefiles.lst")
00280 
00281 
00282 def zip_scriptSDK():
00283     "Create ScriptSDK.zip"
00284     zip_dir = os.path.join(GME_ROOT, "SDK", "ScriptSDK")
00285     tools.zip(zip_dir, "ScriptSDK.zip", "packagefiles.lst")
00286 
00287 def generate_meta_files():
00288     "Generate meta files (mta/mga)"
00289     meta_root = os.path.join(GME_ROOT, "Paradigms", "MetaGME")
00290     tools.xmp2mta(METAGME_XMP, "MetaGME")
00291     meta_file = os.path.join(meta_root, "MetaGME-model.xme")
00292     tools.xme2mga(meta_file, "MetaGME")
00293 
00294 
00295 def generate_sample_files():
00296     "Generate sample files (mta/mga)"
00297     samples_root = os.path.join(GME_ROOT, "Paradigms")
00298     
00299     # SF Paradigm
00300     sample_file = os.path.join(samples_root, "SF", "SFMeta.xme")
00301     tools.xme2mga(sample_file, "MetaGME")
00302     tools.xmp2mta(SF_XMP, "SF") 
00303     sample_file = os.path.join(samples_root, "SF", "SFDemo.xme")
00304     tools.xme2mga(sample_file, "SF")
00305 
00306     # HFSM Paradigm
00307     sample_file = os.path.join(samples_root, "HFSM", "HFSM-Meta.xme")
00308     tools.xme2mga(sample_file, "MetaGME")
00309     tools.xmp2mta(HFSM_XMP, "HFSM") 
00310     sample_file = os.path.join(samples_root, "HFSM", "HFSM-Demo01.xme")
00311     tools.xme2mga(sample_file, "HFSM")
00312     sample_file = os.path.join(samples_root, "HFSM", "HFSM-Demo02.xme")
00313     tools.xme2mga(sample_file, "HFSM")
00314     
00315     # UML Paradigm
00316     sample_file = os.path.join(samples_root, "UML", "UMLMeta.xme")
00317     tools.xme2mga(sample_file, "MetaGME")
00318     tools.xmp2mta(UML_XMP, "UML")
00319 
00320 
00321 def build_msms():
00322     "Build WiX merge modules (msm files)"
00323     
00324     # Prepare include file with dynamic data
00325     f = open(os.path.join(GME_ROOT, "Install", "GME_dyn.wxi"), 'w')
00326     print >> f, "<!-- DO NOT EDIT THIS FILE. WILL BE REGENERATED BY THE BUILD SCRIPTS -->"
00327     print >> f, "<Include>"
00328     print >> f, "   <?define GUIDSTRMETAGME='%s' ?>" % (tools.query_GUID(mta_for_xmp(METAGME_XMP)))
00329     print >> f, "   <?define GUIDSTRHFSM='%s' ?>" % (tools.query_GUID(mta_for_xmp(HFSM_XMP)))
00330     print >> f, "   <?define GUIDSTRSF='%s' ?>" % (tools.query_GUID(mta_for_xmp(SF_XMP)))
00331     print >> f, "   <?define GUIDSTRUML='%s' ?>" % (tools.query_GUID(mta_for_xmp(UML_XMP)))
00332     print >> f, "</Include>" 
00333     f.close()
00334    
00335     import glob
00336     tools.build_WiX([]
00337         + [file for file in glob.glob(os.path.join(GME_ROOT, "Install", "*.wxs")) if file.find('GME.wxs') == -1 ]
00338         + glob.glob(os.path.join(GME_ROOT, "Install", "PIA*/*.wxi"))
00339         )
00340 
00341 def build_msi():
00342     "Build WiX installer (msi file)"
00343 
00344     # Build the msi file
00345     tools.build_WiX([os.path.join(GME_ROOT, "Install", "GME.wxs")])
00346    
00347 
00348 def zip_pdb():
00349     "Collect and zip all debug information (*.pdb)"
00350     tools.system(r"call install\symbols_source_server.cmd <NUL".split(), GME_ROOT)
00351     zipname = os.path.join(GME_ROOT, "Install", "GME-" + prefs['version_string'] + "-symbols.zip")
00352     tools.collect_and_zip(GME_ROOT, zipname, "*.pdb *.dll *.exe *.ocx")
00353     pass
00354 
00355 def publish():
00356     "Publish and archive the install image and debug info"
00357     pass
00358 
00359 
00360 def tag_repository():
00361     "Check in and Tag SVN repository (only for releases)"
00362     pass
00363 
00364 
00365 def do_step(num, step):
00366     "Executing one building step given in param 'step'"
00367     print str(num) + ".", step.__doc__, "..."
00368     step()
00369 
00370 
00371 #
00372 # Main entry point
00373 #
00374 
00375 build_steps = [
00376     check_prerequisites,
00377     update_version_str,
00378     compile_GME,
00379     compile_meta,
00380     compile_JBON,
00381     compile_tools,
00382     compile_samples, 
00383     zip_decorsamples, 
00384     zip_scriptSDK, 
00385     generate_meta_files,
00386     generate_sample_files, 
00387     compile_GME_PGO_Instrument,
00388     PGO_train,
00389     compile_GME_PGO_Optimize,
00390     build_msms,
00391     build_msi,
00392     zip_pdb,
00393     publish,
00394     tag_repository
00395     ]
00396 
00397 start_step = 0
00398 end_step = len(build_steps)-1
00399 
00400 usage = """
00401 usage: %s [OPTION]...
00402 Build an installation image (msi) for GME.
00403 
00404   -h, --help          display help (this message) and exit
00405   -v, --verbose       verbose output (default: %s)
00406   -c, --clean         clean projects before building them (default: %s)
00407   -s, --start=NUM     start at build step 'NUM' (default: %d)
00408   -e, --end=NUM       stop at build step 'NUM' (default: %d)
00409   -i, --include=NUM   include build step 'NUM' explicitly
00410   -x, --exclude=NUM   exclude build step 'NUM' explicitly
00411 
00412   -V, --version=MAJOR.MINOR.PATCHLEVEL.BUILD
00413                       set version (default: %d.%d.%d.%d)
00414   -b, --build_version=BUILD
00415                       set only the build version  
00416 
00417   -a, --arch=ARCH     set architecture (x64 or x86)
00418 \tBuild steps:
00419 
00420 \t%s
00421 """ % (sys.argv[0],
00422        prefs["verbose"],
00423        prefs["clean"],
00424        start_step,
00425        end_step,
00426        prefs["version_major"],
00427        prefs["version_minor"],
00428        prefs["version_patch"],
00429        prefs["version_build"],
00430        "\n\t".join([str(build_steps.index(s)) + ": " + s.__doc__ + ' (' + s.__name__ + ')' for s in build_steps])
00431        )
00432 
00433 try:
00434     opts, args = getopt.getopt(sys.argv[1:], 'hvcs:e:i:x:V:b:a:',
00435                                ["help", "verbose", "clean",
00436                                 "start=", "end=", "include=", "exclude=",
00437                                 "version=", "build_version=", "arch="])
00438     include_steps = []
00439     exclude_steps = []
00440     if args:
00441         print usage
00442         sys.exit()
00443     def get_step(arg):
00444         try:
00445             return int(arg)
00446         except ValueError, e:
00447             return [step.__name__ for step in build_steps].index(arg)
00448     for opt, val in opts:
00449         if opt in ("-h", "--help"):
00450             print usage
00451             sys.exit()
00452         if opt in ("-v", "--verbose"):
00453             prefs["verbose"] = True
00454         if opt in ("-c", "--clean"):
00455             prefs["clean"] = True
00456         if opt in ("-s", "--start"):
00457             start_step = get_step(val)
00458         if opt in ("-e", "--end"):
00459             end_step = get_step(val)
00460         if opt in ("-i", "--include"):
00461             step = get_step(val)
00462             if val not in include_steps:
00463                 include_steps.append(step)
00464         if opt in ("-x", "--exclude"):
00465             step = get_step(val)
00466             if val not in exclude_steps:
00467                 exclude_steps.append(step)
00468         if opt in ("-b", "--build_version"):
00469             prefs["version_build"] = int(val)
00470         if opt in ("-V", "--version"):
00471             (M, m, p, b) = val.split(".")
00472             prefs["version_major"] = int(M)
00473             prefs["version_minor"] = int(m)
00474             prefs["version_patch"] = int(p)
00475             prefs["version_build"] = int(b)
00476         if opt in ("-a", "--arch"):
00477             prefs["arch"] = val
00478             
00479 except (getopt.GetoptError, ValueError, AttributeError), e:
00480     print e
00481     print usage
00482     sys.exit(2)
00483     
00484 prefs["version_string"] = ".".join([str(prefs["version_major"]),
00485                                    str(prefs["version_minor"]),
00486                                    str(prefs["version_patch"])] +
00487                                    ( [ str(prefs["version_build"]) ] if prefs["version_build"] != 0 else [] ))
00488 
00489 print "Building GME version " + prefs["version_string"] + " " + prefs["arch"]
00490 
00491 _pfx86 = os.environ.get('ProgramFiles(x86)', os.environ['ProgramFiles'])
00492 if prefs['toolset'] == '11':
00493     prefs['VS_dir'] = os.path.join(_pfx86, r"Microsoft Visual Studio 11.0")
00494 else:
00495     prefs['VS_dir'] = os.path.join(_pfx86, r"Microsoft Visual Studio 10.0")
00496 
00497 try:
00498     for i in range(len(build_steps)):
00499         if i in include_steps:
00500             do_step(i, build_steps[i])
00501             continue
00502         if i in exclude_steps:
00503             continue
00504         if start_step <= i <= end_step:
00505             do_step(i, build_steps[i])
00506 
00507     print "Build SUCCEEDED."
00508 except:
00509     print "!!! Build FAILED: step " + build_steps[i].__name__
00510     raise