diff --git a/src/fosslight_android/android_binary_analysis.py b/src/fosslight_android/android_binary_analysis.py index 5e07088..3dfe2f1 100755 --- a/src/fosslight_android/android_binary_analysis.py +++ b/src/fosslight_android/android_binary_analysis.py @@ -33,7 +33,8 @@ from .check_package_file import check_packaging_files from .check_notice_file import ( find_bin_in_notice, - read_notice_file + read_notice_file, + read_single_notice_file ) from ._binary_db_controller import get_oss_info_from_db from ._common import ( @@ -65,7 +66,8 @@ final_bin_info = [] module_info_json_obj = {} build_out_path = "" # ex -out/target/product/generic/ -build_out_notice_file_path = "" # build out/target/product/generic/obj/NOTICE.html +# obj/NOTICE.* (str), /obj (str), or Install paths (str or list) +build_out_notice_file_path = "" notice_file_list = {} # Save file list in NOTICE.html platform_version = "" # Android Version. ex- 7.0.0.r1 -> 7.0 @@ -256,6 +258,7 @@ def try_extract_platform_version_from_repo_manifest(android_root): def set_env_variables_from_result_log(android_src_path): global build_out_path, build_out_notice_file_path, platform_version + build_out_notice_file_path = "" pv_manifest = try_extract_platform_version_from_repo_manifest(android_src_path) platform_version_source = "" @@ -288,14 +291,26 @@ def set_env_variables_from_result_log(android_src_path): "(PLATFORM_VERSION=)." ) - # FIND a NOTICE file and build out path - pattern_notice = r'\[.*?\]\s*build\s+(([^\s]*?)/obj/NOTICE\.(?:xml(?:\.gz)?|html|txt))' + # FIND a NOTICE file and build out path. + # Filename is not fixed: NOTICE.xml, NOTICE.html, NOTICE.txt, each optional .gz. + notice_suffix = r'(?:xml|html|txt)(?:\.gz)?' + # <= Android 16: "[...] build /obj/NOTICE." + pattern_notice = rf'\[.*?\]\s*build\s+(([^\s]*?)/obj/NOTICE\.{notice_suffix})(?!\S)' + # Android 17+: "[...] Install: //etc/NOTICE." + pattern_notice_install = rf'\[.*?\]\s*Install:\s+(\S+/NOTICE\.{notice_suffix})(?!\S)' + installed_from_log = [] + seen_install = set() for line in reversed(android_log_lines): match = re.search(pattern_notice, line) - if match: + if match and not build_out_notice_file_path: build_out_notice_file_path = match.group(1) build_out_path = match.group(2) - break + inst = re.search(pattern_notice_install, line) + if inst: + path = inst.group(1) + if path not in seen_install: + seen_install.add(path) + installed_from_log.append(path) if not build_out_path: pattern = re.compile(r'.*Installed file list:\s*([^\s]+)') @@ -311,7 +326,17 @@ def set_env_variables_from_result_log(android_src_path): logger.error("Can't find a build output path.") sys.exit(1) - if not build_out_notice_file_path: + # Legacy obj/NOTICE log line is absent on Android 17. Keep the installed + # Notice paths in the same variable instead of falling back to obj/. + if not build_out_notice_file_path and installed_from_log: + build_out_notice_file_path = ( + installed_from_log if len(installed_from_log) > 1 else installed_from_log[0] + ) + logger.info( + "Notice file from build log (Install): %s", + ", ".join(installed_from_log), + ) + elif not build_out_notice_file_path: build_out_notice_file_path = os.path.join(build_out_path, "obj") read_module_info_from_build_output_file() @@ -409,12 +434,37 @@ def get_result_of_notice_html(found_on_html, notice_file_found): return "nok" +def _notice_paths(): + if isinstance(build_out_notice_file_path, (list, tuple)): + return list(build_out_notice_file_path) + if build_out_notice_file_path: + return [build_out_notice_file_path] + return [] + + +def _read_notice_at(path): + # obj/ holds several NOTICE files. system/etc does not: sibling xml is unrelated. + normalized = path.rstrip("/") + parent_name = os.path.basename(os.path.dirname(normalized)) + if os.path.basename(normalized) == "obj" or parent_name == "obj": + return read_notice_file(os.path.abspath(path)) + return read_single_notice_file(os.path.abspath(path)) + + def find_notice_value(notice_zip_dest_file=""): global notice_file_list, final_bin_info notice_file_comment = "Notice file not found." try: - notice_file_list, notice_files = read_notice_file(os.path.abspath(build_out_notice_file_path)) + notice_file_list = {} + notice_files = [] + for notice_path in _notice_paths(): + found_list, found_files = _read_notice_at(notice_path) + if found_list: + notice_file_list.update(found_list) + for found_file in found_files: + if found_file not in notice_files: + notice_files.append(found_file) if notice_file_list: if notice_files: for notice_file in notice_files: diff --git a/src/fosslight_android/check_notice_file.py b/src/fosslight_android/check_notice_file.py index 1e21878..498caf3 100644 --- a/src/fosslight_android/check_notice_file.py +++ b/src/fosslight_android/check_notice_file.py @@ -15,6 +15,9 @@ logger = logging.getLogger(LOGGER_NAME) CANNOT_FIND_MSG = "CANNOT_FIND_NOTICE" +# On-target and obj/ NOTICE filenames. Android does not fix one extension: +# NOTICE.xml.gz (Android 17 partition install), NOTICE.html(.gz), NOTICE.txt(.gz). +_NOTICE_FILE_SUFFIXES = (".xml.gz", ".html.gz", ".txt.gz", ".xml", ".html", ".txt") def run_notice_html_checklist(binary_file, check_type, notice_file): @@ -73,8 +76,39 @@ def find_bin_in_notice(binary_file_name, notice_file_list): return notice_found +def _is_xml_notice(file_name): + base = os.path.basename(file_name) + if base.endswith(".gz"): + base = base[:-3] + return base.endswith(".xml") + + +def _read_notice_text(file_name): + encodings = ["utf-8", "utf-16", "latin-1"] + is_gz = file_name.endswith(".gz") + for encoding_option in encodings: + try: + if is_gz: + file = gzip.open(file_name, "rt", encoding=encoding_option) + else: + file = open(file_name, encoding=encoding_option) + file_content = file.read() + file.close() + if file_content != "": + return file_content + except Exception: + pass + return "" + + +def _parse_notice_content(file_name, file_content): + if _is_xml_notice(file_name): + return parsing_notice_xml_format(file_content) + return parsing_notice_html_format(file_content) + + def find_files_by_extension(path): - extensions = ['.html', '.xml'] + extensions = ['.html', '.xml', '.txt'] GZ_EXTENSION = '.gz' files = [] try: @@ -89,7 +123,14 @@ def find_files_by_extension(path): logger.info(f"Fail unzip gz file:{error}") for extension in extensions: - files = [os.path.join(path, f) for f in os.listdir(path) if f.endswith(extension)] + files = [] + for file_name in os.listdir(path): + if not file_name.endswith(extension): + continue + # .txt is common for unrelated files. Only NOTICE*.txt is a notice index. + if extension == '.txt' and not file_name.startswith('NOTICE'): + continue + files.append(os.path.join(path, file_name)) if len(files) > 0: break @@ -98,13 +139,11 @@ def find_files_by_extension(path): def read_notice_file(notice_file_path): final_notice_file = {} - # NOTICE.html need to be skipped the errors related to decode - encodings = ["latin-1", "utf-8", "utf-16"] notice_files = [] if os.path.isfile(notice_file_path): notice_files.append(notice_file_path) - if notice_file_path.endswith((".xml", ".html", ".txt", "xml.gz")): + if notice_file_path.endswith(_NOTICE_FILE_SUFFIXES): notice_file_path = os.path.dirname(notice_file_path) if os.path.isdir(notice_file_path): @@ -114,34 +153,39 @@ def read_notice_file(notice_file_path): notice_files = list(set(notice_files)) for file_name in notice_files: - file_list = {} - file_content = "" - if os.path.isfile(file_name): - for encoding_option in encodings: - try: - file = open(file_name, encoding=encoding_option) - file_content = file.read() - file.close() - if file_content != "": - break - except Exception: - pass - if file_content != "": - try: - if file_name.endswith("xml"): - file_list = parsing_notice_xml_format(file_content) - else: - file_list = parsing_notice_html_format(file_content) - except Exception as error: - logger.info("Can't read a notice. :" + file_name) - logger.info(f"{error}") - final_notice_file.update(file_list) - else: - logger.info(f"Notice file is empty. :{file_name}") + found_list = _load_one_notice_file(file_name) + final_notice_file.update(found_list) return final_notice_file, notice_files +def read_single_notice_file(notice_file_path): + """Read one installed NOTICE file. Do not scan the parent directory. + + Partition install paths such as system/etc/NOTICE.xml.gz sit next to + unrelated xml/txt files. A directory scan would treat those as notices. + """ + if not os.path.isfile(notice_file_path): + return {}, [] + return _load_one_notice_file(notice_file_path), [notice_file_path] + + +def _load_one_notice_file(file_name): + file_list = {} + if not os.path.isfile(file_name): + return file_list + file_content = _read_notice_text(file_name) + if file_content != "": + try: + file_list = _parse_notice_content(file_name, file_content) + except Exception as error: + logger.info("Can't read a notice. :" + file_name) + logger.info(f"{error}") + else: + logger.info(f"Notice file is empty. :{file_name}") + return file_list + + def parsing_notice_xml_format(notice_file_content): file_list = {} # NOTICE.xml is XML — use lxml XML parser (not HTML "lxml")