From 695ffc56db15e9a23a9b7e22f0972ed15cc173b3 Mon Sep 17 00:00:00 2001 From: Xuefeng Date: Sun, 1 Feb 2015 20:56:53 -0500 Subject: [PATCH 01/22] Added an unit test to check copy_to_target The unit test check if copy_to_target(file_expr, target) works fine. --- tests/ut_common_copytotarget.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 tests/ut_common_copytotarget.py diff --git a/tests/ut_common_copytotarget.py b/tests/ut_common_copytotarget.py new file mode 100644 index 0000000..881584e --- /dev/null +++ b/tests/ut_common_copytotarget.py @@ -0,0 +1,27 @@ +""" +The unit test check if copy_to_target(file_expr, target) works fine. + +""" +import sys +import os +import shutil +import tempfile +import build_component + +def main(): + # create a new temporary file and a new temporary directory. + temp = tempfile.NamedTemporaryFile() + tempdir = tempfile.mkdtemp() + + # copy a temporary file to a temporary directory. + build_component.copy_to_target(temp.name, tempdir) + + # check if this funtion works fine. + if not os.path.isfile(tempdir+ os.path.sep + os.path.basename(temp.name)): + print "this function is failed" + + temp.close() + shutil.rmtree(tempdir) + +if __name__ == "__main__": + main() From 19cdf12599d6bec5ed74717d0f1ee51945240b70 Mon Sep 17 00:00:00 2001 From: Xuefeng Date: Sun, 1 Feb 2015 20:58:46 -0500 Subject: [PATCH 02/22] Added an unit test to check copy_tree_to_target The unit test checks if copy_tree_to_target(source, target, ignore=None) works fine. --- tests/ut_common_copytreetotarget.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 tests/ut_common_copytreetotarget.py diff --git a/tests/ut_common_copytreetotarget.py b/tests/ut_common_copytreetotarget.py new file mode 100644 index 0000000..6ed6c9c --- /dev/null +++ b/tests/ut_common_copytreetotarget.py @@ -0,0 +1,27 @@ +""" +The unit test checks if copy_tree_to_target(source, target, ignore=None) +works fine. +""" +import sys +import os +import tempfile +import shutil +import build_component + +def main(): + # create two temporary directories and a temporary file. + tempdir1 = tempfile.mkdtemp() + tempdir2 = tempfile.mkdtemp() + temp = tempfile.NamedTemporaryFile(prefix='unittest_', suffix='.txt', dir=tempdir1, delete=False) + + # copy one directory to another and check if it works fine. + build_component.copy_tree_to_target(tempdir1, tempdir2) + + if not os.path.isfile(tempdir2+ os.path.sep + os.path.basename(temp.name)): + print "this function is failed" + + shutil.rmtree(tempdir1) + shutil.rmtree(tempdir2) + +if __name__ == "__main__": + main() From 60d4ef71e3c3437e430fa8d2b6e27320ecd4860f Mon Sep 17 00:00:00 2001 From: Xuefeng Date: Sun, 1 Feb 2015 21:03:02 -0500 Subject: [PATCH 03/22] Added an unit test to check if result is correct The unit test checks if build_component.py can copy files to subdirectories of the build target dir. --- tests/ut_common_copyfilessubdir.py | 71 ++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 tests/ut_common_copyfilessubdir.py diff --git a/tests/ut_common_copyfilessubdir.py b/tests/ut_common_copyfilessubdir.py new file mode 100644 index 0000000..b76ac83 --- /dev/null +++ b/tests/ut_common_copyfilessubdir.py @@ -0,0 +1,71 @@ +""" +ut_common_copyfilessubdir.py --- test if build_component.py can copy files to subdirectories +of the build target dir. +""" + +import os +import sys +import subprocess + +def main(): + # set component directory + component_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + # change directory to component directory ../../common/ + os.chdir(component_dir) + + #subprocess.call(["python", "initialize.py"]) + #subprocess.call(["python","scripts/build.py","-t"]) + sub = subprocess.Popen([sys.executable, 'scripts/build.py', '-t'], + stderr=subprocess.PIPE, + stdout=subprocess.PIPE) + (out, err) = sub.communicate() + + if err != '': + print "FAIL" + + if not "Done building!" in out: + print "FAIL" + + # open config_build.py and read each lines + config_file = open("scripts/config_build.txt") + for line in config_file.readlines(): + # ignore these lines started with "#", '','test' + if line.startswith('#') or line.strip() == '': + continue + + if line.startswith('test'): + source_spec = line.split()[1] + try: + sub_target_dir = line.split()[2] + except IndexError: + sub_target_dir = '' + else: + source_spec = line.split()[0] + try: + sub_target_dir = line.split()[1] + except IndexError: + sub_target_dir = '' + # delete '*' + if source_spec[-1] == '*': + source_spec = source_spec[ : -1] + resultdir = os.path.realpath(os.path.join(component_dir, source_spec)) + target_dir = os.path.realpath(os.path.join(component_dir,'RUNNABLE'+ os.path.sep + sub_target_dir)) + resultlist = [ f for f in os.listdir(resultdir) if os.path.isfile(os.path.join(resultdir,f)) ] + RUNNABLElist = [ f for f in os.listdir(target_dir) if os.path.isfile(os.path.join(target_dir,f)) ] + + # remove .DS_Store file + if '.DS_Store' in resultlist: + resultlist.remove('.DS_Store') + if '.DS_Store' in RUNNABLElist: + RUNNABLElist.remove('.DS_Store') + + # check if files are copied to target directory + for i in resultlist: + if i not in RUNNABLElist : + print 'build_component cannot copy all files to target directory.' + + + +if __name__ == "__main__": + main() From d04432028169009817b580870026247b68f04c6d Mon Sep 17 00:00:00 2001 From: Xuefeng Date: Sun, 1 Feb 2015 21:04:13 -0500 Subject: [PATCH 04/22] Added an unit test to check help_exit The unit test checks if help_exit(errMsg, parser) works fine. --- tests/ut_common_helpexit.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 tests/ut_common_helpexit.py diff --git a/tests/ut_common_helpexit.py b/tests/ut_common_helpexit.py new file mode 100644 index 0000000..748d1d3 --- /dev/null +++ b/tests/ut_common_helpexit.py @@ -0,0 +1,16 @@ +#pragma out Prints the given error message and the help string, then exits +""" +The unit test checks if help_exit(errMsg, parser) works fine. + +""" +import build_component +import optparse + +def main(): + helpstring = """This script is not meant to be run individually. See https://seattle.poly.edu/wiki/BuildInstructions for details.""" + parser = optparse.OptionParser(usage=helpstring) + build_component.help_exit('Prints the given error message and the help string, then exits',parser) + + +if __name__ == "__main__": + main() From ff487db2efe6150ea016b08af7926464ba22b0a3 Mon Sep 17 00:00:00 2001 From: Xuefeng Date: Sun, 1 Feb 2015 21:05:18 -0500 Subject: [PATCH 05/22] Added an unit test to check replace_string The unit test checks if replace_string(old_string, new_string, file_name_pattern="*") works fine. --- tests/ut_common_replacestring.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 tests/ut_common_replacestring.py diff --git a/tests/ut_common_replacestring.py b/tests/ut_common_replacestring.py new file mode 100644 index 0000000..75666e7 --- /dev/null +++ b/tests/ut_common_replacestring.py @@ -0,0 +1,25 @@ +""" +The unit test checks if replace_string(old_string, new_string, file_name_pattern="*") +works fine. +""" + +import sys +import os +import tempfile +import build_component + +def main(): + temp = tempfile.NamedTemporaryFile(prefix='testfile_', suffix='.txt', dir= os.path.dirname(os.path.realpath(__file__)), delete=False) + temp.write('unittest_old_string') + temp.seek(0) + + build_component.replace_string('old_string', 'new_string') + if 'new_string' in open(temp.name).read(): + pass + else: + print "failed" + + temp.close() + +if __name__ == "__main__": + main() From a51a2e9b3a0b3fde98b6a49b93f798b3f2f92ba6 Mon Sep 17 00:00:00 2001 From: Xuefeng Date: Sun, 1 Feb 2015 21:06:51 -0500 Subject: [PATCH 06/22] Added an unit test to check build_component.py The unit test checks if build_component.py works fine. --- tests/ut_common_test_file.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 tests/ut_common_test_file.py diff --git a/tests/ut_common_test_file.py b/tests/ut_common_test_file.py new file mode 100644 index 0000000..9aac9b0 --- /dev/null +++ b/tests/ut_common_test_file.py @@ -0,0 +1,18 @@ +""" +The unit test checks if build_component.py works fine. +""" + +import subprocess +import sys + +sub = subprocess.Popen([sys.executable, '../DEPENDENCIES/common/build_component.py', '-t'], + stderr=subprocess.PIPE, + stdout=subprocess.PIPE) +(out, err) = sub.communicate() + +#should cause test to fail if there's anything on stderr +if err != '': + print "FAIL" + +if not "Done building!" in out: + print "FAIL" From 2a290b136d76c4cfb02c34c8a8d0749672023740 Mon Sep 17 00:00:00 2001 From: Xuefeng Date: Tue, 3 Feb 2015 14:52:32 -0500 Subject: [PATCH 07/22] Added an unit test to check process_mix() function The unit test checks if process_mix function works fine. --- tests/ut_common_processmix.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 tests/ut_common_processmix.py diff --git a/tests/ut_common_processmix.py b/tests/ut_common_processmix.py new file mode 100644 index 0000000..02f47ee --- /dev/null +++ b/tests/ut_common_processmix.py @@ -0,0 +1,15 @@ +""" +The unit test checks if process_mix function works fine. +""" + +import os +import build_component + +def main(): + try: + build_component.process_mix("repypp.py", True) + except OSError: + print "process_mix function is failed" + +if __name__ == "__main__": + main() From 9b04352322d3bca1b3ce1344ec8ef5721fc8db28 Mon Sep 17 00:00:00 2001 From: Xuefeng Date: Tue, 3 Feb 2015 14:55:13 -0500 Subject: [PATCH 08/22] Added cloning and copying of seattlelib_v2 repo --- scripts/config_initialize.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/config_initialize.txt b/scripts/config_initialize.txt index c712576..72b1fb2 100644 --- a/scripts/config_initialize.txt +++ b/scripts/config_initialize.txt @@ -29,4 +29,4 @@ https://github.com/SeattleTestbed/common ../DEPENDENCIES/common https://github.com/SeattleTestbed/utf ../DEPENDENCIES/utf - +https://github.com/SeattleTestbed/seattlelib_v2 ../DEPENDENCIES/seattlelib_v2 From 111eef6ac9d2c061010fbbb830f8d020fed1af0a Mon Sep 17 00:00:00 2001 From: Xuefeng Date: Tue, 3 Feb 2015 14:56:06 -0500 Subject: [PATCH 09/22] Added cloning and copying of seattlelib_v2 repo --- scripts/config_build.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/config_build.txt b/scripts/config_build.txt index 1e7c380..ef9ff81 100644 --- a/scripts/config_build.txt +++ b/scripts/config_build.txt @@ -5,4 +5,5 @@ # It is unit-testable however, and here is what we need for this: test ./* test DEPENDENCIES/utf/* +test DEPENDENCISE/seattlelib_v2/* test tests/* From 5b20a11d2837a595d15590c9bf9b5a206be1c245 Mon Sep 17 00:00:00 2001 From: Xuefeng Date: Tue, 3 Feb 2015 15:03:37 -0500 Subject: [PATCH 10/22] Fixed directory name error --- scripts/config_build.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/config_build.txt b/scripts/config_build.txt index ef9ff81..d56819c 100644 --- a/scripts/config_build.txt +++ b/scripts/config_build.txt @@ -5,5 +5,5 @@ # It is unit-testable however, and here is what we need for this: test ./* test DEPENDENCIES/utf/* -test DEPENDENCISE/seattlelib_v2/* +test DEPENDENCIES/seattlelib_v2/* test tests/* From 8bb5c7313158225b69930ffd22f7c638142697f4 Mon Sep 17 00:00:00 2001 From: Xuefeng Date: Wed, 4 Feb 2015 14:03:47 -0500 Subject: [PATCH 11/22] Added a pattern of the file name arg. Added a pattern of the file name arg to test if this function can reduce the number of files we look at. --- tests/ut_common_replacestring.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ut_common_replacestring.py b/tests/ut_common_replacestring.py index 75666e7..2975d0b 100644 --- a/tests/ut_common_replacestring.py +++ b/tests/ut_common_replacestring.py @@ -13,7 +13,7 @@ def main(): temp.write('unittest_old_string') temp.seek(0) - build_component.replace_string('old_string', 'new_string') + build_component.replace_string('old_string', 'new_string','*testfile*') if 'new_string' in open(temp.name).read(): pass else: From bea7292de6fb6d3353fee0bcfc8d1c1fb6b4d045 Mon Sep 17 00:00:00 2001 From: Xuefeng Date: Thu, 5 Feb 2015 15:54:02 -0500 Subject: [PATCH 12/22] Deleted extra spaces --- tests/ut_common_copytotarget.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ut_common_copytotarget.py b/tests/ut_common_copytotarget.py index 881584e..6f2971a 100644 --- a/tests/ut_common_copytotarget.py +++ b/tests/ut_common_copytotarget.py @@ -9,7 +9,7 @@ import build_component def main(): - # create a new temporary file and a new temporary directory. + # create a new temporary file and a new temporary directory. temp = tempfile.NamedTemporaryFile() tempdir = tempfile.mkdtemp() From 18d41b76236d1ac9d9ee0b563fe64c15d2b48e50 Mon Sep 17 00:00:00 2001 From: Xuefeng Date: Sun, 8 Feb 2015 22:35:59 -0500 Subject: [PATCH 13/22] Avoid testing a function using the same function I build a dummy component specific to this unit test which has a separate build config, and instead of re-reading the build config, compare the resulting file list after building with a hardcoded file list. --- tests/ut_common_copyfilessubdir.py | 82 +++++++++++++++--------------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/tests/ut_common_copyfilessubdir.py b/tests/ut_common_copyfilessubdir.py index b76ac83..9559e0c 100644 --- a/tests/ut_common_copyfilessubdir.py +++ b/tests/ut_common_copyfilessubdir.py @@ -8,17 +8,22 @@ import subprocess def main(): - # set component directory - component_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + # set runnable directory + runnable_dir = os.path.dirname(os.path.abspath(__file__)) + # set common directory + common_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - # change directory to component directory ../../common/ - os.chdir(component_dir) + # change directory to runnable directory + os.chdir(runnable_dir) + try: + os.mkdir('unittest') + except OSError: + pass - #subprocess.call(["python", "initialize.py"]) - #subprocess.call(["python","scripts/build.py","-t"]) - sub = subprocess.Popen([sys.executable, 'scripts/build.py', '-t'], - stderr=subprocess.PIPE, - stdout=subprocess.PIPE) + sub = subprocess.Popen([sys.executable, + os.path.realpath(os.path.join(common_dir , 'DEPENDENCIES/common/test_component.py')), '-t', 'unittest'], + stderr=subprocess.PIPE, + stdout=subprocess.PIPE) (out, err) = sub.communicate() if err != '': @@ -26,44 +31,39 @@ def main(): if not "Done building!" in out: print "FAIL" + + #hardcode file list + result_dir = [ os.path.realpath(os.path.join(common_dir, 'DEPENDENCIES/utf/')), + os.path.realpath(os.path.join(common_dir, 'DEPENDENCIES/seattlelib_v2/')) + ] + + RUNNABLE_dir = [ os.path.realpath(os.path.join(common_dir, 'RUNNABLE/unittest/')), + os.path.realpath(os.path.join(common_dir, 'RUNNABLE/unittest/repyv2/')) + ] + + resultlist = [] + RUNNABLElist = [] - # open config_build.py and read each lines - config_file = open("scripts/config_build.txt") - for line in config_file.readlines(): - # ignore these lines started with "#", '','test' - if line.startswith('#') or line.strip() == '': - continue + for item in result_dir: + for f in os.listdir(item): + if os.path.isfile(os.path.join(item, f)): + resultlist.append(f) - if line.startswith('test'): - source_spec = line.split()[1] - try: - sub_target_dir = line.split()[2] - except IndexError: - sub_target_dir = '' - else: - source_spec = line.split()[0] - try: - sub_target_dir = line.split()[1] - except IndexError: - sub_target_dir = '' - # delete '*' - if source_spec[-1] == '*': - source_spec = source_spec[ : -1] - resultdir = os.path.realpath(os.path.join(component_dir, source_spec)) - target_dir = os.path.realpath(os.path.join(component_dir,'RUNNABLE'+ os.path.sep + sub_target_dir)) - resultlist = [ f for f in os.listdir(resultdir) if os.path.isfile(os.path.join(resultdir,f)) ] - RUNNABLElist = [ f for f in os.listdir(target_dir) if os.path.isfile(os.path.join(target_dir,f)) ] + for item in RUNNABLE_dir: + for f in os.listdir(item): + if os.path.isfile(os.path.join(item, f)): + RUNNABLElist.append(f) - # remove .DS_Store file - if '.DS_Store' in resultlist: + # Disregard Mac OS X's .DS_Store metafile + if '.DS_Store' in resultlist: resultlist.remove('.DS_Store') - if '.DS_Store' in RUNNABLElist: + if '.DS_Store' in RUNNABLElist: RUNNABLElist.remove('.DS_Store') - # check if files are copied to target directory - for i in resultlist: - if i not in RUNNABLElist : - print 'build_component cannot copy all files to target directory.' + # check if files are copied to target directory + for f in resultlist: + if f not in RUNNABLElist : + print 'build_component cannot copy all files to target directory.' From 6c902d83cb880669b04138cc8cbb4f73f9b77560 Mon Sep 17 00:00:00 2001 From: Xuefeng Date: Sun, 8 Feb 2015 22:54:31 -0500 Subject: [PATCH 14/22] Remove more .DS_Store metafiles in a list. In the past, it only can delete one .DS_Store. If there are more than one .DS_Store, it cannot delete them all. Now I use for loop to fix it. --- tests/ut_common_copyfilessubdir.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/ut_common_copyfilessubdir.py b/tests/ut_common_copyfilessubdir.py index 9559e0c..1ea72b7 100644 --- a/tests/ut_common_copyfilessubdir.py +++ b/tests/ut_common_copyfilessubdir.py @@ -55,9 +55,11 @@ def main(): RUNNABLElist.append(f) # Disregard Mac OS X's .DS_Store metafile - if '.DS_Store' in resultlist: + for f in resultlist: + if '.DS_Store' == f: resultlist.remove('.DS_Store') - if '.DS_Store' in RUNNABLElist: + for f in RUNNABLElist: + if '.DS_Store' == f: RUNNABLElist.remove('.DS_Store') # check if files are copied to target directory From f9f118937af34279a6fd6ab2a98940e183166b3e Mon Sep 17 00:00:00 2001 From: Xuefeng Date: Sun, 8 Feb 2015 22:59:25 -0500 Subject: [PATCH 15/22] Fixed small typo and gave more explanations. --- tests/ut_common_copytotarget.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/tests/ut_common_copytotarget.py b/tests/ut_common_copytotarget.py index 6f2971a..cece7c4 100644 --- a/tests/ut_common_copytotarget.py +++ b/tests/ut_common_copytotarget.py @@ -1,7 +1,9 @@ """ -The unit test check if copy_to_target(file_expr, target) works fine. - +The unit test check if build_component.copy_to_target works fine. +It creates a temporary file and a temporary directory. Then copying the temporary +file to temporary directory.In the end check if build_component.copy_to_target works fine. """ + import sys import os import shutil @@ -9,19 +11,24 @@ import build_component def main(): - # create a new temporary file and a new temporary directory. - temp = tempfile.NamedTemporaryFile() - tempdir = tempfile.mkdtemp() + # create a temporary file and a temporary directory. + temporary_file = tempfile.NamedTemporaryFile() + temporary_dir = tempfile.mkdtemp() # copy a temporary file to a temporary directory. - build_component.copy_to_target(temp.name, tempdir) + try: + build_component.copy_to_target(temporary_file.name, temporary_dir) + except IOError, e: + print e.errno + print e # check if this funtion works fine. - if not os.path.isfile(tempdir+ os.path.sep + os.path.basename(temp.name)): - print "this function is failed" + if not os.path.isfile(temporary_dir + os.path.sep + os.path.basename(temporary_file.name)): + print "build_component.copy_to_target failed to copy file " + temporary_file.name + " to folder " + temporary_dir - temp.close() - shutil.rmtree(tempdir) + temporary_file.close() + shutil.rmtree(temporary_dir) if __name__ == "__main__": main() + From 0073f42894497b3d6e124d72b1b4661240f21709 Mon Sep 17 00:00:00 2001 From: Xuefeng Date: Sun, 8 Feb 2015 23:00:55 -0500 Subject: [PATCH 16/22] Fixed small typo and gave more explanations. --- tests/ut_common_copytreetotarget.py | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/tests/ut_common_copytreetotarget.py b/tests/ut_common_copytreetotarget.py index 6ed6c9c..c8a9f45 100644 --- a/tests/ut_common_copytreetotarget.py +++ b/tests/ut_common_copytreetotarget.py @@ -1,7 +1,9 @@ """ -The unit test checks if copy_tree_to_target(source, target, ignore=None) -works fine. +The unit test checks if build_component.copy_tree_to_target works fine. +It create two temporary directories and a temporary file in one directory. Then copying one +directory to another.In the end check if build_component.copy_tree_to_target works fine. """ + import sys import os import tempfile @@ -9,19 +11,23 @@ import build_component def main(): - # create two temporary directories and a temporary file. - tempdir1 = tempfile.mkdtemp() - tempdir2 = tempfile.mkdtemp() - temp = tempfile.NamedTemporaryFile(prefix='unittest_', suffix='.txt', dir=tempdir1, delete=False) + # create two temporary directories and a temporary file in one directory. + temporary_dir1 = tempfile.mkdtemp() + temporary_dir2 = tempfile.mkdtemp() + temporary_file = tempfile.NamedTemporaryFile(prefix='unittest_', suffix='.txt', dir=temporary_dir1, delete=False) # copy one directory to another and check if it works fine. - build_component.copy_tree_to_target(tempdir1, tempdir2) + try: + build_component.copy_tree_to_target(temporary_dir1, temporary_dir2) + except IOError, e: + print e.errno + print e - if not os.path.isfile(tempdir2+ os.path.sep + os.path.basename(temp.name)): - print "this function is failed" + if not os.path.isfile(temporary_dir2 + os.path.sep + os.path.basename(temporary_file.name)): + print "build_component.copy_tree_to_target failed to copy folder " + temporary_dir1 + " to folder " + temporary_dir2 - shutil.rmtree(tempdir1) - shutil.rmtree(tempdir2) + shutil.rmtree(temporary_dir1) + shutil.rmtree(temporary_dir2) if __name__ == "__main__": main() From 7e69f077dd975964694912702bf10e67f6a5cdf0 Mon Sep 17 00:00:00 2001 From: Xuefeng Date: Sun, 8 Feb 2015 23:02:17 -0500 Subject: [PATCH 17/22] Added to check if repypp.py exists. --- tests/ut_common_processmix.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/ut_common_processmix.py b/tests/ut_common_processmix.py index 02f47ee..85a7456 100644 --- a/tests/ut_common_processmix.py +++ b/tests/ut_common_processmix.py @@ -6,10 +6,14 @@ import build_component def main(): - try: - build_component.process_mix("repypp.py", True) - except OSError: - print "process_mix function is failed" + if os.path.isfile("repypp.py"): + try: + build_component.process_mix("repypp.py", True) + except OSError: + print "process_mix function is failed" + else: + print "Can't find repypp.py." if __name__ == "__main__": main() + From a392bc1f671c174839620139c3b61c7274dabc7f Mon Sep 17 00:00:00 2001 From: Xuefeng Date: Sun, 8 Feb 2015 23:03:56 -0500 Subject: [PATCH 18/22] Fixed code style issue Using two spaces per indent level instead of tab. --- tests/ut_common_processmix.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/ut_common_processmix.py b/tests/ut_common_processmix.py index 85a7456..4025785 100644 --- a/tests/ut_common_processmix.py +++ b/tests/ut_common_processmix.py @@ -6,14 +6,15 @@ import build_component def main(): - if os.path.isfile("repypp.py"): - try: - build_component.process_mix("repypp.py", True) - except OSError: - print "process_mix function is failed" - else: - print "Can't find repypp.py." + if os.path.isfile("repypp.py"): + try: + build_component.process_mix("repypp.py", True) + except OSError: + print "process_mix function is failed" + else: + print "Can't find repypp.py." if __name__ == "__main__": main() + From 80e9f847e7f4035cb659500d446be87e0e4757c1 Mon Sep 17 00:00:00 2001 From: Xuefeng Date: Sun, 8 Feb 2015 23:09:27 -0500 Subject: [PATCH 19/22] Added it for unit tests. --- test_component.py | 381 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 381 insertions(+) create mode 100644 test_component.py diff --git a/test_component.py b/test_component.py new file mode 100644 index 0000000..6a9ca1a --- /dev/null +++ b/test_component.py @@ -0,0 +1,381 @@ +""" +build_component.py --- build a component of the Seattle Testbed. + +NOTE: This script is not meant to be called individually, but through + a wrapper script, build.py. See the Seattle build instructions wiki + for details: https://seattle.poly.edu/wiki/BuildInstructions + +This script first erases all the files in a target directory, and then +copies the necessary files to build the particular component. +Afterwards, .mix files in the target directory are ran through the +preprocessor. + +The target directory that is passed to the script must exist. It is +emptied before files are copied over. + +This script assumes that you (or a component's scripts/initialize.py) have +checked out all the required repos of SeattleTestbed into the parent directory +of this script. + +NOTE WELL: The repositories are used as-is. No attempt is made to switch + to a specific branch, pull from remotes, etc. + (In a future version of this script, the currently active branch + for each repo will be displayed as a visual reminder of this fact.) + + + build_component.py [-t] [-v] [-r] [TARGET_DIRECTORY] + -t or --testfiles copies in all the files required to run the unit tests + -v or --verbose displays significantly more output on failure to process + a mix file + -r or --randomports replaces the default ports of 12345, 12346, and 12347 + with three random ports between 52000 and 53000. + TARGET_DIRECTORY is optional; the default target dir is "RUNNABLE" + + For details on the build process of Seattle components, + see https://seattle.poly.edu/wiki/BuildInstructions +""" + +import os +import sys +import glob +import random +import shutil +import optparse +import subprocess + + +# Temporarily add this script's path to the PYTHONPATH so we can +# import testportfiller.... +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import testportfiller +# Remove testportfiller's path again +sys.path = sys.path[1:] + + + +def copy_to_target(file_expr, target): + """ + This function copies files (in the current directory) that match the + expression file_expr to the target folder. + The source files are from the current directory. + The target directory must exist. + file_expr may contain wildcards (shell globs). + """ + files_to_copy = glob.glob(file_expr) + if files_to_copy == []: + print "WARNING: File expression '" + file_expr + "' does not match any files. Maybe the directory is empty, or the file / directory doesn't exist?" + + for file_path in files_to_copy: + if os.path.isfile(file_path): + shutil.copyfile(file_path, target + os.path.sep +os.path.basename(file_path)) + + + +def copy_tree_to_target(source, target, ignore=None): + """ + Copies a directory to the target destination. + If you pass a string for ignore, then subdirectories that contain the ignore + string will not be copied over (as well as the files they contain). + """ + + full_source_path = os.path.abspath(source) + full_target_path = os.path.abspath(target) + + for root, directories, filenames in os.walk(source): + # Relative path is needed to build the absolute target path. + + # If we leave a leading directory separator in the relative folder + # path, then attempts to join it will cause the relative folder path + # to be treated as an absolute path. + relative_folder_path = os.path.abspath(root)[len(full_source_path):].lstrip(os.sep) + + # If the ignore string is in the relative path, skip this directory. + if ignore and ignore in relative_folder_path: + continue + + # Attempts to copy over a file when the containing directories above it do not + # exist will trigger an exception. + full_target_subdir_path = os.path.join(full_target_path, relative_folder_path) + if not os.path.isdir(full_target_subdir_path): + os.makedirs(full_target_subdir_path) + + for name in filenames: + relative_path = os.path.join(relative_folder_path, name) + shutil.copyfile( + os.path.join(full_source_path, relative_path), + os.path.join(full_target_path, relative_path)) + + + +def process_mix(script_path, verbose): + """ + Run the .mix files in current directory through the preprocessor. + script_path specifies the name of the preprocessor script. + The preprocessor script must be in the working directory. + """ + mix_files = glob.glob("*.mix") + error_list = [] + + for file_path in mix_files: + # Generate a .py file for the .mix file specified by file_path + processed_file_path = (os.path.basename(file_path)).replace(".mix",".py") + (theout, theerr) = exec_command(sys.executable + " " + script_path + " " + file_path + " " + processed_file_path) + + # If there was any problem processing the files, then notify the user. + if theerr: + print "Unable to process the file: " + file_path + error_list.append((file_path, theerr)) + + # If the verbose option is on then print the error. + if verbose and len(error_list) > 0: + print "\n" + '#'*50 + "\nPrinting all the exceptions (verbose option)\n" + '#'*50 + for file_name, error in error_list: + print "\n" + file_name + ":" + print error + print '-'*80 + + + +def exec_command(command): + """ + Execute command on a shell, return a tuple containing the resulting + standard output and standard error (as strings). + """ + # Windows does not like close_fds and we shouldn't need it so... + process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + # get the output and close + theout = process.stdout.read() + process.stdout.close() + + # get the errput and close + theerr = process.stderr.read() + process.stderr.close() + + # FreeBSD prints on stdout, when it gets a signal... + # I want to look at the last line. It ends in \n, so I use index -2 + if len(theout.split('\n')) > 1 and theout.split('\n')[-2].strip() == 'Terminated': + # remove the last line + theout = '\n'.join(theout.split('\n')[:-2]) + + # However we threw away an extra '\n'. If anything remains, let's replace it + if theout != '': + theout = theout + '\n' + + # OS's besides FreeBSD uses stderr + if theerr.strip() == 'Terminated': + theerr = '' + + # Windows isn't fond of this either... + # clean up after the child + #os.waitpid(p.pid,0) + + return (theout, theerr) + + + +def replace_string(old_string, new_string, file_name_pattern="*"): + """ + + Go through all the files in the current folder and replace + every match of the old string in the file with the new + string. + + old_string - The string we want to replace. + + new_string - The new string we want to replace the old string + with. + file_name_pattern - The pattern of the file name if you want + to reduce the number of files we look at. By default the + function looks at all files. + + None. + + Many files may get modified. + + None + """ + + for testfile in glob.glob(file_name_pattern): + # Read in the initial file. + inFile = file(testfile, 'r') + filestring = inFile.read() + inFile.close() + + # Replace any form of the matched old string with + # the new string. + filestring = filestring.replace(old_string, new_string) + + # Write the file back. + outFile = file(testfile, 'w') + outFile.write(filestring) + outFile.close() + + + +def help_exit(errMsg, parser): + """ + Prints the given error message and the help string, then exits + """ + print errMsg + parser.print_help() + sys.exit(1) + + + +def main(): + helpstring = """This script is not meant to be run individually. +See https://seattle.poly.edu/wiki/BuildInstructions for details.""" + + # Parse the options provided. + parser = optparse.OptionParser(usage=helpstring) + + parser.add_option("-t", "--testfiles", action="store_true", + dest="include_tests", default=False, + help="Include files required to run the unit tests ") + parser.add_option("-v", "--verbose", action="store_true", + dest="verbose", default=False, + help="Show more output on failure to process a .mix file") + parser.add_option("-r", "--randomports", action="store_true", + dest="randomports", default=False, + help="Replace the default ports with random ports between 52000 and 53000. ") + + (options, args) = parser.parse_args() + + # Determine the target directory. + # Use path/to/component/DEPENDENCIES/common/../../RUNNABLE + # unless overridden by the user. + if len(args) == 0: + component_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + target_dir = os.path.realpath(os.path.join(component_dir, "RUNNABLE")) + if not os.path.exists(target_dir): + os.makedirs(target_dir) + + else: + # The user supplied a target directory. Make it an absolute path, + # and check if it exists. + target_dir = os.path.realpath(args[0]) + + if not os.path.isdir(target_dir): + help_exit("Supplied target '" + target_dir + + "' doesn't exist or is not a directory", parser) + + # Print let the world know we run + print "Building into", target_dir + + # Set variables according to the provided options. + repytest = options.include_tests + RANDOMPORTS = options.randomports + verbose = options.verbose + + + # This script's parent directory is the root dir of all dependent + # repositories, path/to/component/DEPENDENCIES/ + repos_root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + # Set working directory to the target + os.chdir(target_dir) + files_to_remove = glob.glob("*") + + # Empty the destination + for entry in files_to_remove: + if os.path.isdir(entry): + shutil.rmtree(entry) + else: + os.remove(entry) + + + # Return to the grand-parent directory of the dependent repositories, + # i.e. "/path/to/component/DEPENDENCIES/.." + # We now have + # "." with the sources of the component we want to build, + # "scripts/" with the build config file, and + # "DEPENDENCIES/" with all the dependent repos. + os.chdir(os.path.join(repos_root_dir, "..")) + # Copy the necessary files to the respective target folders, + # following the instructions in scripts/config_build.txt. + config_file = open("DEPENDENCIES/common/config_build_test.txt") + + for line in config_file.readlines(): + # Ignore comments and blank lines + if line.startswith("#") or line.strip() == '': + continue + + # Anything non-comment and non-empty specifies a + # source file or directory for us to use. + if line.startswith("test"): + # Build instructions for unit tests look like this: + # "test ../relative/path/to/required/file_or_fileglob" + if repytest: + source_spec = line.split()[1].strip() + try: + sub_target_dir = line.split()[2].strip() + except IndexError: + sub_target_dir = '' + else: + # Tests weren't requested. Skip. + continue + else: + # This is a non-test instruction. + source_spec = line.split()[0].strip() + try: + sub_target_dir = line.split()[1].strip() + except IndexError: + sub_target_dir = '' + + os.chdir(target_dir) + if not os.path.exists(sub_target_dir) and sub_target_dir: + os.makedirs(sub_target_dir) + + os.chdir(os.path.join(repos_root_dir, "..")) + copy_to_target(source_spec, target_dir + os.path.sep + sub_target_dir) + + + # Set working directory to the target + os.chdir(target_dir) + + # Set up dynamic port information + if RANDOMPORTS: + print "\n[ Randomports option was chosen ]\n"+'-'*50 + ports_as_ints = random.sample(range(52000, 53000), 5) + ports_as_strings = [] + for port in ports_as_ints: + ports_as_strings.append(str(port)) + + print "Randomly chosen ports: ", ports_as_strings + testportfiller.replace_ports(ports_as_strings, ports_as_strings) + + # Replace the string with a random port + random_nodemanager_port = random.randint(53000, 54000) + print "Chosen random nodemanager port: " + str(random_nodemanager_port) + print '-'*50 + "\n" + replace_string("", str(random_nodemanager_port), "*nm*") + replace_string("", str(random_nodemanager_port), "*securitylayers*") + + else: + # Otherwise use the default ports... + testportfiller.replace_ports(['12345','12346','12347', '12348', '12349'], ['12345','12346','12347', '12348', '12349']) + + # Use default port 1224 for the nodemanager port if --random flag is not provided. + replace_string("", '1224', "*nm*") + replace_string("", '1224', "*securitylayers*") + + # If we have a repyV1 dir, we need to preprocess files that use the + # `include` functionality there. + try: + os.chdir("repyV1") + process_mix("repypp.py", verbose) + # Change back to root project directory + os.chdir(repos_root_dir) + except OSError: + # There was no repyV1 dir. Continue. + pass + + print "Done building!" + + + +if __name__ == '__main__': + main() + From 454c0bfd55659aeab1f1585736dcb77e0f45cd3e Mon Sep 17 00:00:00 2001 From: Xuefeng Date: Sun, 8 Feb 2015 23:12:05 -0500 Subject: [PATCH 20/22] Added it for unit tests. --- config_build_test.txt | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 config_build_test.txt diff --git a/config_build_test.txt b/config_build_test.txt new file mode 100644 index 0000000..ff2c403 --- /dev/null +++ b/config_build_test.txt @@ -0,0 +1,2 @@ +test DEPENDENCIES/utf/* +test DEPENDENCIES/seattlelib_v2/* repyv2 From 9f06b2f8411bdbd787c1cf752106384ece745b74 Mon Sep 17 00:00:00 2001 From: Xuefeng Date: Sun, 8 Feb 2015 23:20:47 -0500 Subject: [PATCH 21/22] Use different branch of `common` for testing --- scripts/config_initialize.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/config_initialize.txt b/scripts/config_initialize.txt index 72b1fb2..568f740 100644 --- a/scripts/config_initialize.txt +++ b/scripts/config_initialize.txt @@ -27,6 +27,6 @@ # telling initialize.py what to download where # scripts/initialize.py -https://github.com/SeattleTestbed/common ../DEPENDENCIES/common +https://github.com/XuefengHuang/common -b unittests ../DEPENDENCIES/common https://github.com/SeattleTestbed/utf ../DEPENDENCIES/utf https://github.com/SeattleTestbed/seattlelib_v2 ../DEPENDENCIES/seattlelib_v2 From 4863466eb6e4390ff7fa08144cac5779975f0b22 Mon Sep 17 00:00:00 2001 From: Xuefeng Date: Mon, 9 Feb 2015 08:48:21 -0500 Subject: [PATCH 22/22] add more explanations --- tests/ut_common_replacestring.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/tests/ut_common_replacestring.py b/tests/ut_common_replacestring.py index 2975d0b..df416c9 100644 --- a/tests/ut_common_replacestring.py +++ b/tests/ut_common_replacestring.py @@ -1,6 +1,5 @@ """ -The unit test checks if replace_string(old_string, new_string, file_name_pattern="*") -works fine. +The unit test checks if build_component.replace_string works fine. """ import sys @@ -9,17 +8,17 @@ import build_component def main(): - temp = tempfile.NamedTemporaryFile(prefix='testfile_', suffix='.txt', dir= os.path.dirname(os.path.realpath(__file__)), delete=False) - temp.write('unittest_old_string') - temp.seek(0) + # create a temporary file include a string("unittest_old_string") + temporary_file = tempfile.NamedTemporaryFile(prefix='testfile_', suffix='.txt', dir= os.path.dirname(os.path.realpath(__file__)), delete= True) + temporary_file.write('unittest_old_string') + temporary_file.seek(0) build_component.replace_string('old_string', 'new_string','*testfile*') - if 'new_string' in open(temp.name).read(): + if 'new_string' in open(temporary_file.name).read(): pass else: - print "failed" + print "build_component.replace_string failed to replace old_string with new_string." - temp.close() if __name__ == "__main__": main()