diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 6512931..cf74c91 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -29,10 +29,11 @@ jobs: pip install tox - name: Run Tox run: | + tox -e release_flake8 tox -e release reuse: runs-on: ubuntu-latest - steps: + steps: - uses: actions/checkout@v3 - name: REUSE Compliance Check uses: fsfe/reuse-action@v1 diff --git a/script/copy_notice_files.py b/script/copy_notice_files.py index f7f1902..493bc28 100755 --- a/script/copy_notice_files.py +++ b/script/copy_notice_files.py @@ -29,5 +29,5 @@ def copy_and_create_dir(input_dir): if not os.path.exists(os.path.join(dist_path, file)): shutil.copy2(src_file, dist_path) - except Exception as e: + except Exception: pass diff --git a/script/generate-notice-files.py b/script/generate-notice-files.py index 44bb2fd..c05eec8 100644 --- a/script/generate-notice-files.py +++ b/script/generate-notice-files.py @@ -39,11 +39,12 @@ '"': """, "'": "'", ">": ">", - "<": "<", - } + "<": "<"} + def hexify(s): - return ("%02x"*len(s)) % tuple(map(ord, s)) + return ("%02x" * len(s)) % tuple(map(ord, s)) + def md5sum(filename): """Calculate an MD5 of the file given by FILENAME, @@ -63,9 +64,10 @@ def md5sum(filename): def html_escape(text): """Produce entities within text.""" - return "".join(HTML_ESCAPE_TABLE.get(c,c) for c in text) + return "".join(HTML_ESCAPE_TABLE.get(c, c) for c in text) + -HTML_OUTPUT_CSS=""" +HTML_OUTPUT_CSS = """ """ + def combine_notice_files_html(file_hash, input_dirs, output_filename): """Combine notice files in FILE_HASH and output a HTML version to OUTPUT_FILENAME.""" @@ -85,19 +88,19 @@ def combine_notice_files_html(file_hash, input_dirs, output_filename): id_count = 0 for value in file_hash: for filename in value: - id_table[filename] = id_count + id_table[filename] = id_count id_count += 1 # Open the output file, and output the header pieces output_file = open(output_filename, "wb") - print >> output_file, "" - print >> output_file, HTML_OUTPUT_CSS - print >> output_file, '' + print("", file=output_file) + print(HTML_OUTPUT_CSS, file=output_file) + print('', file=output_file) # Output our table of contents - print >> output_file, '
' - print >> output_file, "
", file=output_file) # Output the individual notice file lists - print >>output_file, '' + print('
', file=output_file) for value in file_hash: - print >> output_file, '" - print >> output_file - print >> output_file - print >> output_file + print("%s
" % (SRC_DIR_STRIP_RE.sub(r"\1", filename)), file=output_file) + print("", file=output_file) + print(file=output_file) + print('
', file=output_file)
+        print(html_escape(open(value[0]).read()), file=output_file)
+        print("
", file=output_file) + print("", file=output_file) + print(file=output_file) + print(file=output_file) + print(file=output_file) # Finish off the file output - print >> output_file, "
' % id_table.get(value[0]) - print >> output_file, '
Notices for file(s):
' - print >> output_file, '
' + print('
' % id_table.get(value[0]), file=output_file) + print('
Notices for file(s):
', file=output_file) for filename in value: - print >> output_file, "%s
" % (SRC_DIR_STRIP_RE.sub(r"\1", filename)) - print >> output_file, "" - print >> output_file - print >> output_file, '
'
-        print >> output_file, html_escape(open(value[0]).read())
-        print >> output_file, "
" - print >> output_file, "
" - print >> output_file, "" + print("", file=output_file) + print("", file=output_file) output_file.close() + def combine_notice_files_text(file_hash, input_dirs, output_filename, file_title): """Combine notice files in FILE_HASH and output a text version to OUTPUT_FILENAME.""" SRC_DIR_STRIP_RE = re.compile("(?:" + "|".join(re.escape(input_dirs)) + ")(/.*).txt") output_file = open(output_filename, "wb") - print >> output_file, file_title + print(file_title, file=output_file) for value in file_hash: - print >> output_file, "============================================================" - print >> output_file, "Notices for file(s):" - for filename in value: - print >> output_file, SRC_DIR_STRIP_RE.sub(r"\1", filename) - print >> output_file, "------------------------------------------------------------" - print >> output_file, open(value[0]).read() + print("============================================================", file=output_file) + print("Notices for file(s):", file=output_file) + + for filename in value: + print(SRC_DIR_STRIP_RE.sub(r"\1", filename), file=output_file) + print("------------------------------------------------------------", file=output_file) + print(open(value[0]).read(), file=output_file) output_file.close() + def combine_notice_files_xml(files_with_same_hash, input_dirs, output_filename): """Combine notice files in FILE_HASH and output a XML version to OUTPUT_FILENAME.""" @@ -157,13 +162,13 @@ def combine_notice_files_xml(files_with_same_hash, input_dirs, output_filename): id_table = {} for file_key in files_with_same_hash.keys(): for filename in files_with_same_hash[file_key]: - id_table[filename] = file_key + id_table[filename] = file_key # Open the output file, and output the header pieces output_file = open(output_filename, "wb") - print >> output_file, '' - print >> output_file, "" + print('', file=output_file) + print("", file=output_file) # Flatten the list of lists into a single list of filenames sorted_filenames = sorted(id_table.keys()) @@ -171,10 +176,10 @@ def combine_notice_files_xml(files_with_same_hash, input_dirs, output_filename): # Print out a nice table of contents for filename in sorted_filenames: stripped_filename = SRC_DIR_STRIP_RE.sub(r"\1", filename) - print >> output_file, '%s' % (id_table.get(filename), stripped_filename) + print('%s' % (id_table.get(filename), stripped_filename), file=output_file) - print >> output_file - print >> output_file + print(file=output_file) + print(file=output_file) processed_file_keys = [] # Output the individual notice file lists @@ -184,13 +189,14 @@ def combine_notice_files_xml(files_with_same_hash, input_dirs, output_filename): continue processed_file_keys.append(file_key) - print >> output_file, '' % (file_key, html_escape(open(filename).read())) - print >> output_file + print('' % (file_key, html_escape(open(filename).read())), file=output_file) + print(file=output_file) # Finish off the file output - print >> output_file, "" + print("", file=output_file) output_file.close() + def get_args(): parser = argparse.ArgumentParser() parser.add_argument( @@ -216,6 +222,7 @@ def get_args(): help='The sub directories which should be excluded.') return parser.parse_args() + def main(argv): args = get_args() @@ -229,7 +236,7 @@ def main(argv): included_subdirs = args.included_subdirs if args.excluded_subdirs is not None: excluded_subdirs = args.excluded_subdirs - + notice_files_to_append = os.path.join(args.source_dir[0], "notice") copy_notice_files.copy_and_create_dir(notice_files_to_append) input_dirs = [os.path.normpath(source_dir) for source_dir in args.source_dir] @@ -242,14 +249,16 @@ def main(argv): if len(included_subdirs) > 0: matched = False for subdir in included_subdirs: - if (root == (input_dir + '/' + subdir) or - root.startswith(input_dir + '/' + subdir + '/')): + matches_subdir = root == (input_dir + '/' + subdir) + starts_with_subdir = root.startswith(input_dir + '/' + subdir + '/') + if matches_subdir or starts_with_subdir: matched = True break elif len(excluded_subdirs) > 0: for subdir in excluded_subdirs: - if (root == (input_dir + '/' + subdir) or - root.startswith(input_dir + '/' + subdir + '/')): + matches_subdir = root == (input_dir + '/' + subdir) + starts_with_subdir = root.startswith(input_dir + '/' + subdir + '/') + if matches_subdir or starts_with_subdir: matched = False break if root.startswith(notice_files_to_append): @@ -268,5 +277,6 @@ def main(argv): if xml_output_file is not None: combine_notice_files_xml(files_with_same_hash, input_dirs, xml_output_file) + if __name__ == "__main__": main(sys.argv) diff --git a/src/fosslight_android/_help.py b/src/fosslight_android/_help.py index 30778ef..1c1a589 100644 --- a/src/fosslight_android/_help.py +++ b/src/fosslight_android/_help.py @@ -21,7 +21,7 @@ -m\t\t\t\t Analyze the source code for the path where the license could not be found. -e \t Path to exclude from source analysis. -p\t\t\t\t Check files that should not be included in the Packaging file. - -f\t\t\t\t Print result of Find Command for binary that can not find Source Code Path. + -f\t\t\t\t Print result of Find Command for binary that can not find Source Code Path. -i\t\t\t\t Disable the function to automatically convert OSS names based on AOSP. -r \t\t result.txt file with a list of binaries to remove.""" diff --git a/src/fosslight_android/android_binary_analysis.py b/src/fosslight_android/android_binary_analysis.py index 6ae76c0..57ec0d4 100755 --- a/src/fosslight_android/android_binary_analysis.py +++ b/src/fosslight_android/android_binary_analysis.py @@ -31,12 +31,9 @@ get_path_by_using_find ) from .check_package_file import check_packaging_files -from .check_notice_file import ( - run_notice_html_checklist, +from .check_notice_file import ( find_bin_in_notice, - read_notice_file, - create_additional_notice, - divide_notice_files_by_binary + read_notice_file ) from ._binary_db_controller import get_oss_info_from_db from ._common import ( @@ -157,7 +154,7 @@ def get_module_json_obj_by_installed_path(module_name, binary_name_with_path, bi else: # Find binary by installed path for key in module_info_json_obj: js_value = module_info_json_obj[key] - output_files = js_value.get("installed",None) + output_files = js_value.get("installed", None) if output_files is not None: for output_file in output_files: if output_file == binary_name_with_path: @@ -168,7 +165,6 @@ def get_module_json_obj_by_installed_path(module_name, binary_name_with_path, bi if path_with_out_dir == output_file: js_value[MODULE_TYPE_NAME] = key return js_value - return "" @@ -204,7 +200,7 @@ def set_env_variables_from_result_log(): # Check the platform version for line in android_log_lines: try: - line = line.strip() + line = line.strip() pattern = re.compile(r'.*PLATFORM_VERSION\s*=\s*(\d+)\.?\d*\S*\s*') matched = pattern.match(line) if matched is not None: @@ -212,7 +208,7 @@ def set_env_variables_from_result_log(): break except Exception: - pass + pass # FIND a NOTICE file and build out path for line in reversed(android_log_lines): @@ -277,7 +273,6 @@ def find_binaries_from_out_dir(): tmp_files = [] for file_rel_path in return_list: if any(re.search(re_except_path, file_rel_path) for re_except_path in EXCEPTIONAL_PATH): - #logger.debug(f"REMOVE (Exceptional Path):{file_rel_path}") continue else: bin_item = AndroidBinary(os.path.abspath(file_rel_path)) @@ -720,7 +715,6 @@ def return_shorter_installed_path_data(origin, new): else: origin_path = origin.bin_name new_path = new.bin_name - if len(origin_path) > len(new_path): return new elif len(origin_path) == 0: @@ -772,31 +766,30 @@ def find_meta_lic_files(): if lic_list: lic = ','.join(lic_list) meta_lic_files[key] = lic - - + + def create_and_copy_notice_zip(notice_files_list, zip_file_path): - final_destination_file_name ="" + final_destination_file_name = "" if len(notice_files_list) == 1: single_file_path = notice_files_list[0] destination_path = os.path.join(os.path.dirname(zip_file_path), os.path.basename(single_file_path)) shutil.copy(single_file_path, destination_path) - final_destination_file_name = destination_path + final_destination_file_name = destination_path logger.debug(f"Notice file is copied to '{destination_path}'.") - else: + else: with zipfile.ZipFile(zip_file_path, 'w') as zipf: - for single_file_path in notice_files_list: + for single_file_path in notice_files_list: zipf.write(single_file_path, arcname=os.path.basename(single_file_path)) - final_destination_file_name = zip_file_path - + final_destination_file_name = zip_file_path + return final_destination_file_name -def main(): +def main(): global android_log_lines, ANDROID_LOG_FILE_NAME, python_script_dir, num_cores, now, logger, final_bin_info - find_empty_path = False - notice_check_ok = False - auto_fill_oss_name = True + find_empty_path = False + auto_fill_oss_name = True analyze_source = False path_to_exclude = [] RESULT_FILE_EXTENSION = ".xlsx" @@ -816,12 +809,12 @@ def main(): parser = argparse.ArgumentParser(description='FOSSLight Android', prog='fosslight_android', add_help=False) parser.add_argument('-h', '--help', action='store_true', required=False) parser.add_argument('-v', '--version', action='store_true', required=False) - parser.add_argument('-s', '--source', type=str, required=False) - parser.add_argument('-m', '--more', action='store_true', required=False) + parser.add_argument('-s', '--source', type=str, required=False) + parser.add_argument('-m', '--more', action='store_true', required=False) parser.add_argument('-a', '--android', type=str, required=False) parser.add_argument('-f', '--find', action='store_true', required=False) parser.add_argument('-i', '--ignore', action='store_true', required=False) - parser.add_argument('-p', '--packaging', type=str, required=False) + parser.add_argument('-p', '--packaging', type=str, required=False) parser.add_argument('-r', '--remove', type=str, required=False) parser.add_argument('-e', '--exclude', nargs="*", required=False, default=[]) @@ -830,11 +823,11 @@ def main(): print_help_msg() if args.version: print_version(PKG_NAME) - if args.source: # android source path + if args.source: os.chdir(args.source) - android_src_path = args.source + android_src_path = args.source if args.more: # Analyze source mode. - analyze_source = True + analyze_source = True if args.android: ANDROID_LOG_FILE_NAME = args.android if args.find: # Execute "find" command when source path is not found. @@ -848,10 +841,10 @@ def main(): if args.packaging: check_packaging_files(args.packaging) - return - + return + if args.remove: # Remove the inputted list from the binary list. - remove_list_file = args.remove + remove_list_file = args.remove read_success, android_log_lines = read_file(ANDROID_LOG_FILE_NAME) if not read_success: logger.error("(-a option) Fail to read a file:" + ANDROID_LOG_FILE_NAME) @@ -891,8 +884,7 @@ def main(): # Print the result result_log["Output Directory"] = python_script_dir try: - #str_final_result_log = yaml.safe_dump(result_log, allow_unicode=True, sort_keys=True) - str_final_result_log = yaml.safe_dump(result_log, allow_unicode=True) + str_final_result_log = yaml.safe_dump(result_log, allow_unicode=True, sort_keys=True) logger.info(str_final_result_log) except Exception as ex: logger.warning(f"Failed to print result log. : {ex}") diff --git a/src/fosslight_android/check_notice_file.py b/src/fosslight_android/check_notice_file.py index 2a8a26f..91f0ea5 100644 --- a/src/fosslight_android/check_notice_file.py +++ b/src/fosslight_android/check_notice_file.py @@ -96,29 +96,12 @@ def find_files_by_extension(path): return files -#def read_notice_file(notice_file_path, notice_html_file): def read_notice_file(notice_file_path): final_notice_file = {} # NOTICE.html need to be skipped the errors related to decode encodings = ["latin-1", "utf-8", "utf-16"] notice_files = [] - - ''' - if notice_html_file != "": - notice_files = notice_html_file.split(",") - else: - if os.path.isfile(notice_file_path): - notice_files.append(notice_file_path) - if notice_file_path.endswith(".html") or notice_file_path.endswith(".xml") or notice_file_path.endswith(".txt"): - notice_file_path = os.path.dirname(notice_file_path) - - if os.path.isdir(notice_file_path): - additional_notice_files = find_files_by_extension(notice_file_path) - if len(additional_notice_files) > 0: - notice_files.extend(additional_notice_files) - notice_files = list(set(notice_files)) - ''' - + if os.path.isfile(notice_file_path): notice_files.append(notice_file_path) if notice_file_path.endswith(".html") or notice_file_path.endswith(".xml") or notice_file_path.endswith(".txt"): @@ -128,9 +111,8 @@ def read_notice_file(notice_file_path): additional_notice_files = find_files_by_extension(notice_file_path) if len(additional_notice_files) > 0: notice_files.extend(additional_notice_files) - notice_files = list(set(notice_files)) + notice_files = list(set(notice_files)) - for file_name in notice_files: file_list = {} file_content = "" diff --git a/test/test_tox.py b/test/test_tox.py index 9328169..a5d6d96 100644 --- a/test/test_tox.py +++ b/test/test_tox.py @@ -33,12 +33,8 @@ def test_release_environment(run_command): # when help_result, _, _ = run_command("fosslight_android -h") - ok_result, _, _ = run_command("fosslight_android -s test/android_12_sample -a android.log") - + success, _, _ = run_command("fosslight_android -s test/android_12_sample -a android.log") # then assert help_result is True, "Help command failed" - assert ok_result is True, "OK command failed" - - - + assert success is True, "Test was failed" diff --git a/tox.ini b/tox.ini index 7b02dc1..16ddbd1 100644 --- a/tox.ini +++ b/tox.ini @@ -3,7 +3,7 @@ envlist = test_run skipdist = true [main] -android_src_path = "android/src_12" +android_src_path = "test/android_12_sample" android_build_log = "android.log" [flake8] @@ -24,6 +24,7 @@ filterwarnings = ignore::DeprecationWarning markers = run: Test for local environment release: Test for CI environment + release_flake8: Test with flake8 for CI environment [testenv] install_command = pip install {opts} {packages} @@ -46,4 +47,12 @@ deps = -r{toxinidir}/requirements-dev.txt commands = - pytest -v --flake8 --ignore=script -m release + pytest -m release + + +[testenv:release_flake8] +deps = + -r{toxinidir}/requirements-dev.txt + +commands = + pytest -v --flake8 \ No newline at end of file