diff --git a/config_build_test.txt b/config_build_test.txt new file mode 100644 index 0000000..ff2c403 --- /dev/null +++ b/config_build_test.txt @@ -0,0 +1,2 @@ +test DEPENDENCIES/utf/* +test DEPENDENCIES/seattlelib_v2/* repyv2 diff --git a/scripts/config_build.txt b/scripts/config_build.txt index 1e7c380..d56819c 100644 --- a/scripts/config_build.txt +++ b/scripts/config_build.txt @@ -5,4 +5,5 @@ # It is unit-testable however, and here is what we need for this: test ./* test DEPENDENCIES/utf/* +test DEPENDENCIES/seattlelib_v2/* test tests/* diff --git a/scripts/config_initialize.txt b/scripts/config_initialize.txt index c712576..568f740 100644 --- a/scripts/config_initialize.txt +++ b/scripts/config_initialize.txt @@ -27,6 +27,6 @@ # telling initialize.py what to download where # scripts/initialize.py -https://github.com/SeattleTestbed/common ../DEPENDENCIES/common +https://github.com/XuefengHuang/common -b unittests ../DEPENDENCIES/common https://github.com/SeattleTestbed/utf ../DEPENDENCIES/utf - +https://github.com/SeattleTestbed/seattlelib_v2 ../DEPENDENCIES/seattlelib_v2 diff --git a/test_component.py b/test_component.py new file mode 100644 index 0000000..6a9ca1a --- /dev/null +++ b/test_component.py @@ -0,0 +1,381 @@ +""" +build_component.py --- build a component of the Seattle Testbed. + +NOTE: This script is not meant to be called individually, but through + a wrapper script, build.py. See the Seattle build instructions wiki + for details: https://seattle.poly.edu/wiki/BuildInstructions + +This script first erases all the files in a target directory, and then +copies the necessary files to build the particular component. +Afterwards, .mix files in the target directory are ran through the +preprocessor. + +The target directory that is passed to the script must exist. It is +emptied before files are copied over. + +This script assumes that you (or a component's scripts/initialize.py) have +checked out all the required repos of SeattleTestbed into the parent directory +of this script. + +NOTE WELL: The repositories are used as-is. No attempt is made to switch + to a specific branch, pull from remotes, etc. + (In a future version of this script, the currently active branch + for each repo will be displayed as a visual reminder of this fact.) + + + build_component.py [-t] [-v] [-r] [TARGET_DIRECTORY] + -t or --testfiles copies in all the files required to run the unit tests + -v or --verbose displays significantly more output on failure to process + a mix file + -r or --randomports replaces the default ports of 12345, 12346, and 12347 + with three random ports between 52000 and 53000. + TARGET_DIRECTORY is optional; the default target dir is "RUNNABLE" + + For details on the build process of Seattle components, + see https://seattle.poly.edu/wiki/BuildInstructions +""" + +import os +import sys +import glob +import random +import shutil +import optparse +import subprocess + + +# Temporarily add this script's path to the PYTHONPATH so we can +# import testportfiller.... +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import testportfiller +# Remove testportfiller's path again +sys.path = sys.path[1:] + + + +def copy_to_target(file_expr, target): + """ + This function copies files (in the current directory) that match the + expression file_expr to the target folder. + The source files are from the current directory. + The target directory must exist. + file_expr may contain wildcards (shell globs). + """ + files_to_copy = glob.glob(file_expr) + if files_to_copy == []: + print "WARNING: File expression '" + file_expr + "' does not match any files. Maybe the directory is empty, or the file / directory doesn't exist?" + + for file_path in files_to_copy: + if os.path.isfile(file_path): + shutil.copyfile(file_path, target + os.path.sep +os.path.basename(file_path)) + + + +def copy_tree_to_target(source, target, ignore=None): + """ + Copies a directory to the target destination. + If you pass a string for ignore, then subdirectories that contain the ignore + string will not be copied over (as well as the files they contain). + """ + + full_source_path = os.path.abspath(source) + full_target_path = os.path.abspath(target) + + for root, directories, filenames in os.walk(source): + # Relative path is needed to build the absolute target path. + + # If we leave a leading directory separator in the relative folder + # path, then attempts to join it will cause the relative folder path + # to be treated as an absolute path. + relative_folder_path = os.path.abspath(root)[len(full_source_path):].lstrip(os.sep) + + # If the ignore string is in the relative path, skip this directory. + if ignore and ignore in relative_folder_path: + continue + + # Attempts to copy over a file when the containing directories above it do not + # exist will trigger an exception. + full_target_subdir_path = os.path.join(full_target_path, relative_folder_path) + if not os.path.isdir(full_target_subdir_path): + os.makedirs(full_target_subdir_path) + + for name in filenames: + relative_path = os.path.join(relative_folder_path, name) + shutil.copyfile( + os.path.join(full_source_path, relative_path), + os.path.join(full_target_path, relative_path)) + + + +def process_mix(script_path, verbose): + """ + Run the .mix files in current directory through the preprocessor. + script_path specifies the name of the preprocessor script. + The preprocessor script must be in the working directory. + """ + mix_files = glob.glob("*.mix") + error_list = [] + + for file_path in mix_files: + # Generate a .py file for the .mix file specified by file_path + processed_file_path = (os.path.basename(file_path)).replace(".mix",".py") + (theout, theerr) = exec_command(sys.executable + " " + script_path + " " + file_path + " " + processed_file_path) + + # If there was any problem processing the files, then notify the user. + if theerr: + print "Unable to process the file: " + file_path + error_list.append((file_path, theerr)) + + # If the verbose option is on then print the error. + if verbose and len(error_list) > 0: + print "\n" + '#'*50 + "\nPrinting all the exceptions (verbose option)\n" + '#'*50 + for file_name, error in error_list: + print "\n" + file_name + ":" + print error + print '-'*80 + + + +def exec_command(command): + """ + Execute command on a shell, return a tuple containing the resulting + standard output and standard error (as strings). + """ + # Windows does not like close_fds and we shouldn't need it so... + process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + # get the output and close + theout = process.stdout.read() + process.stdout.close() + + # get the errput and close + theerr = process.stderr.read() + process.stderr.close() + + # FreeBSD prints on stdout, when it gets a signal... + # I want to look at the last line. It ends in \n, so I use index -2 + if len(theout.split('\n')) > 1 and theout.split('\n')[-2].strip() == 'Terminated': + # remove the last line + theout = '\n'.join(theout.split('\n')[:-2]) + + # However we threw away an extra '\n'. If anything remains, let's replace it + if theout != '': + theout = theout + '\n' + + # OS's besides FreeBSD uses stderr + if theerr.strip() == 'Terminated': + theerr = '' + + # Windows isn't fond of this either... + # clean up after the child + #os.waitpid(p.pid,0) + + return (theout, theerr) + + + +def replace_string(old_string, new_string, file_name_pattern="*"): + """ + + Go through all the files in the current folder and replace + every match of the old string in the file with the new + string. + + old_string - The string we want to replace. + + new_string - The new string we want to replace the old string + with. + file_name_pattern - The pattern of the file name if you want + to reduce the number of files we look at. By default the + function looks at all files. + + None. + + Many files may get modified. + + None + """ + + for testfile in glob.glob(file_name_pattern): + # Read in the initial file. + inFile = file(testfile, 'r') + filestring = inFile.read() + inFile.close() + + # Replace any form of the matched old string with + # the new string. + filestring = filestring.replace(old_string, new_string) + + # Write the file back. + outFile = file(testfile, 'w') + outFile.write(filestring) + outFile.close() + + + +def help_exit(errMsg, parser): + """ + Prints the given error message and the help string, then exits + """ + print errMsg + parser.print_help() + sys.exit(1) + + + +def main(): + helpstring = """This script is not meant to be run individually. +See https://seattle.poly.edu/wiki/BuildInstructions for details.""" + + # Parse the options provided. + parser = optparse.OptionParser(usage=helpstring) + + parser.add_option("-t", "--testfiles", action="store_true", + dest="include_tests", default=False, + help="Include files required to run the unit tests ") + parser.add_option("-v", "--verbose", action="store_true", + dest="verbose", default=False, + help="Show more output on failure to process a .mix file") + parser.add_option("-r", "--randomports", action="store_true", + dest="randomports", default=False, + help="Replace the default ports with random ports between 52000 and 53000. ") + + (options, args) = parser.parse_args() + + # Determine the target directory. + # Use path/to/component/DEPENDENCIES/common/../../RUNNABLE + # unless overridden by the user. + if len(args) == 0: + component_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + target_dir = os.path.realpath(os.path.join(component_dir, "RUNNABLE")) + if not os.path.exists(target_dir): + os.makedirs(target_dir) + + else: + # The user supplied a target directory. Make it an absolute path, + # and check if it exists. + target_dir = os.path.realpath(args[0]) + + if not os.path.isdir(target_dir): + help_exit("Supplied target '" + target_dir + + "' doesn't exist or is not a directory", parser) + + # Print let the world know we run + print "Building into", target_dir + + # Set variables according to the provided options. + repytest = options.include_tests + RANDOMPORTS = options.randomports + verbose = options.verbose + + + # This script's parent directory is the root dir of all dependent + # repositories, path/to/component/DEPENDENCIES/ + repos_root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + # Set working directory to the target + os.chdir(target_dir) + files_to_remove = glob.glob("*") + + # Empty the destination + for entry in files_to_remove: + if os.path.isdir(entry): + shutil.rmtree(entry) + else: + os.remove(entry) + + + # Return to the grand-parent directory of the dependent repositories, + # i.e. "/path/to/component/DEPENDENCIES/.." + # We now have + # "." with the sources of the component we want to build, + # "scripts/" with the build config file, and + # "DEPENDENCIES/" with all the dependent repos. + os.chdir(os.path.join(repos_root_dir, "..")) + # Copy the necessary files to the respective target folders, + # following the instructions in scripts/config_build.txt. + config_file = open("DEPENDENCIES/common/config_build_test.txt") + + for line in config_file.readlines(): + # Ignore comments and blank lines + if line.startswith("#") or line.strip() == '': + continue + + # Anything non-comment and non-empty specifies a + # source file or directory for us to use. + if line.startswith("test"): + # Build instructions for unit tests look like this: + # "test ../relative/path/to/required/file_or_fileglob" + if repytest: + source_spec = line.split()[1].strip() + try: + sub_target_dir = line.split()[2].strip() + except IndexError: + sub_target_dir = '' + else: + # Tests weren't requested. Skip. + continue + else: + # This is a non-test instruction. + source_spec = line.split()[0].strip() + try: + sub_target_dir = line.split()[1].strip() + except IndexError: + sub_target_dir = '' + + os.chdir(target_dir) + if not os.path.exists(sub_target_dir) and sub_target_dir: + os.makedirs(sub_target_dir) + + os.chdir(os.path.join(repos_root_dir, "..")) + copy_to_target(source_spec, target_dir + os.path.sep + sub_target_dir) + + + # Set working directory to the target + os.chdir(target_dir) + + # Set up dynamic port information + if RANDOMPORTS: + print "\n[ Randomports option was chosen ]\n"+'-'*50 + ports_as_ints = random.sample(range(52000, 53000), 5) + ports_as_strings = [] + for port in ports_as_ints: + ports_as_strings.append(str(port)) + + print "Randomly chosen ports: ", ports_as_strings + testportfiller.replace_ports(ports_as_strings, ports_as_strings) + + # Replace the string with a random port + random_nodemanager_port = random.randint(53000, 54000) + print "Chosen random nodemanager port: " + str(random_nodemanager_port) + print '-'*50 + "\n" + replace_string("", str(random_nodemanager_port), "*nm*") + replace_string("", str(random_nodemanager_port), "*securitylayers*") + + else: + # Otherwise use the default ports... + testportfiller.replace_ports(['12345','12346','12347', '12348', '12349'], ['12345','12346','12347', '12348', '12349']) + + # Use default port 1224 for the nodemanager port if --random flag is not provided. + replace_string("", '1224', "*nm*") + replace_string("", '1224', "*securitylayers*") + + # If we have a repyV1 dir, we need to preprocess files that use the + # `include` functionality there. + try: + os.chdir("repyV1") + process_mix("repypp.py", verbose) + # Change back to root project directory + os.chdir(repos_root_dir) + except OSError: + # There was no repyV1 dir. Continue. + pass + + print "Done building!" + + + +if __name__ == '__main__': + main() + diff --git a/tests/ut_common_copyfilessubdir.py b/tests/ut_common_copyfilessubdir.py new file mode 100644 index 0000000..1ea72b7 --- /dev/null +++ b/tests/ut_common_copyfilessubdir.py @@ -0,0 +1,73 @@ +""" +ut_common_copyfilessubdir.py --- test if build_component.py can copy files to subdirectories +of the build target dir. +""" + +import os +import sys +import subprocess + +def main(): + # set runnable directory + runnable_dir = os.path.dirname(os.path.abspath(__file__)) + # set common directory + common_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + # change directory to runnable directory + os.chdir(runnable_dir) + try: + os.mkdir('unittest') + except OSError: + pass + + sub = subprocess.Popen([sys.executable, + os.path.realpath(os.path.join(common_dir , 'DEPENDENCIES/common/test_component.py')), '-t', 'unittest'], + stderr=subprocess.PIPE, + stdout=subprocess.PIPE) + (out, err) = sub.communicate() + + if err != '': + print "FAIL" + + if not "Done building!" in out: + print "FAIL" + + #hardcode file list + result_dir = [ os.path.realpath(os.path.join(common_dir, 'DEPENDENCIES/utf/')), + os.path.realpath(os.path.join(common_dir, 'DEPENDENCIES/seattlelib_v2/')) + ] + + RUNNABLE_dir = [ os.path.realpath(os.path.join(common_dir, 'RUNNABLE/unittest/')), + os.path.realpath(os.path.join(common_dir, 'RUNNABLE/unittest/repyv2/')) + ] + + resultlist = [] + RUNNABLElist = [] + + for item in result_dir: + for f in os.listdir(item): + if os.path.isfile(os.path.join(item, f)): + resultlist.append(f) + + for item in RUNNABLE_dir: + for f in os.listdir(item): + if os.path.isfile(os.path.join(item, f)): + RUNNABLElist.append(f) + + # Disregard Mac OS X's .DS_Store metafile + for f in resultlist: + if '.DS_Store' == f: + resultlist.remove('.DS_Store') + for f in RUNNABLElist: + if '.DS_Store' == f: + RUNNABLElist.remove('.DS_Store') + + # check if files are copied to target directory + for f in resultlist: + if f not in RUNNABLElist : + print 'build_component cannot copy all files to target directory.' + + + +if __name__ == "__main__": + main() diff --git a/tests/ut_common_copytotarget.py b/tests/ut_common_copytotarget.py new file mode 100644 index 0000000..cece7c4 --- /dev/null +++ b/tests/ut_common_copytotarget.py @@ -0,0 +1,34 @@ +""" +The unit test check if build_component.copy_to_target works fine. +It creates a temporary file and a temporary directory. Then copying the temporary +file to temporary directory.In the end check if build_component.copy_to_target works fine. +""" + +import sys +import os +import shutil +import tempfile +import build_component + +def main(): + # create a temporary file and a temporary directory. + temporary_file = tempfile.NamedTemporaryFile() + temporary_dir = tempfile.mkdtemp() + + # copy a temporary file to a temporary directory. + try: + build_component.copy_to_target(temporary_file.name, temporary_dir) + except IOError, e: + print e.errno + print e + + # check if this funtion works fine. + if not os.path.isfile(temporary_dir + os.path.sep + os.path.basename(temporary_file.name)): + print "build_component.copy_to_target failed to copy file " + temporary_file.name + " to folder " + temporary_dir + + temporary_file.close() + shutil.rmtree(temporary_dir) + +if __name__ == "__main__": + main() + diff --git a/tests/ut_common_copytreetotarget.py b/tests/ut_common_copytreetotarget.py new file mode 100644 index 0000000..c8a9f45 --- /dev/null +++ b/tests/ut_common_copytreetotarget.py @@ -0,0 +1,33 @@ +""" +The unit test checks if build_component.copy_tree_to_target works fine. +It create two temporary directories and a temporary file in one directory. Then copying one +directory to another.In the end check if build_component.copy_tree_to_target works fine. +""" + +import sys +import os +import tempfile +import shutil +import build_component + +def main(): + # create two temporary directories and a temporary file in one directory. + temporary_dir1 = tempfile.mkdtemp() + temporary_dir2 = tempfile.mkdtemp() + temporary_file = tempfile.NamedTemporaryFile(prefix='unittest_', suffix='.txt', dir=temporary_dir1, delete=False) + + # copy one directory to another and check if it works fine. + try: + build_component.copy_tree_to_target(temporary_dir1, temporary_dir2) + except IOError, e: + print e.errno + print e + + if not os.path.isfile(temporary_dir2 + os.path.sep + os.path.basename(temporary_file.name)): + print "build_component.copy_tree_to_target failed to copy folder " + temporary_dir1 + " to folder " + temporary_dir2 + + shutil.rmtree(temporary_dir1) + shutil.rmtree(temporary_dir2) + +if __name__ == "__main__": + main() diff --git a/tests/ut_common_helpexit.py b/tests/ut_common_helpexit.py new file mode 100644 index 0000000..748d1d3 --- /dev/null +++ b/tests/ut_common_helpexit.py @@ -0,0 +1,16 @@ +#pragma out Prints the given error message and the help string, then exits +""" +The unit test checks if help_exit(errMsg, parser) works fine. + +""" +import build_component +import optparse + +def main(): + helpstring = """This script is not meant to be run individually. See https://seattle.poly.edu/wiki/BuildInstructions for details.""" + parser = optparse.OptionParser(usage=helpstring) + build_component.help_exit('Prints the given error message and the help string, then exits',parser) + + +if __name__ == "__main__": + main() diff --git a/tests/ut_common_processmix.py b/tests/ut_common_processmix.py new file mode 100644 index 0000000..4025785 --- /dev/null +++ b/tests/ut_common_processmix.py @@ -0,0 +1,20 @@ +""" +The unit test checks if process_mix function works fine. +""" + +import os +import build_component + +def main(): + if os.path.isfile("repypp.py"): + try: + build_component.process_mix("repypp.py", True) + except OSError: + print "process_mix function is failed" + else: + print "Can't find repypp.py." + +if __name__ == "__main__": + main() + + diff --git a/tests/ut_common_replacestring.py b/tests/ut_common_replacestring.py new file mode 100644 index 0000000..df416c9 --- /dev/null +++ b/tests/ut_common_replacestring.py @@ -0,0 +1,24 @@ +""" +The unit test checks if build_component.replace_string works fine. +""" + +import sys +import os +import tempfile +import build_component + +def main(): + # create a temporary file include a string("unittest_old_string") + temporary_file = tempfile.NamedTemporaryFile(prefix='testfile_', suffix='.txt', dir= os.path.dirname(os.path.realpath(__file__)), delete= True) + temporary_file.write('unittest_old_string') + temporary_file.seek(0) + + build_component.replace_string('old_string', 'new_string','*testfile*') + if 'new_string' in open(temporary_file.name).read(): + pass + else: + print "build_component.replace_string failed to replace old_string with new_string." + + +if __name__ == "__main__": + main() diff --git a/tests/ut_common_test_file.py b/tests/ut_common_test_file.py new file mode 100644 index 0000000..9aac9b0 --- /dev/null +++ b/tests/ut_common_test_file.py @@ -0,0 +1,18 @@ +""" +The unit test checks if build_component.py works fine. +""" + +import subprocess +import sys + +sub = subprocess.Popen([sys.executable, '../DEPENDENCIES/common/build_component.py', '-t'], + stderr=subprocess.PIPE, + stdout=subprocess.PIPE) +(out, err) = sub.communicate() + +#should cause test to fail if there's anything on stderr +if err != '': + print "FAIL" + +if not "Done building!" in out: + print "FAIL"