fix(notice): read installed notice paths from log

src/fosslight_android/android_binary_analysis.py

@@ -33,7 +33,8 @@
 from .check_package_file import check_packaging_files
 from .check_notice_file import (
     find_bin_in_notice,
-    read_notice_file
+    read_notice_file,
+    read_single_notice_file
 )
 from ._binary_db_controller import get_oss_info_from_db
 from ._common import (
@@ -65,7 +66,8 @@
 final_bin_info = []
 module_info_json_obj = {}
 build_out_path = ""  # ex -out/target/product/generic/
-build_out_notice_file_path = ""  # build out/target/product/generic/obj/NOTICE.html
+# obj/NOTICE.* (str), <product>/obj (str), or Install paths (str or list)
+build_out_notice_file_path = ""
 notice_file_list = {}  # Save file list in NOTICE.html
 platform_version = ""  # Android Version. ex- 7.0.0.r1 -> 7.0
 
@@ -256,6 +258,7 @@ def try_extract_platform_version_from_repo_manifest(android_root):
 
 def set_env_variables_from_result_log(android_src_path):
     global build_out_path, build_out_notice_file_path, platform_version
+    build_out_notice_file_path = ""
 
     pv_manifest = try_extract_platform_version_from_repo_manifest(android_src_path)
     platform_version_source = ""
@@ -288,14 +291,26 @@ def set_env_variables_from_result_log(android_src_path):
             "(PLATFORM_VERSION=)."
         )
 
-    # FIND a NOTICE file and build out path
-    pattern_notice = r'\[.*?\]\s*build\s+(([^\s]*?)/obj/NOTICE\.(?:xml(?:\.gz)?|html|txt))'
+    # FIND a NOTICE file and build out path.
+    # Filename is not fixed: NOTICE.xml, NOTICE.html, NOTICE.txt, each optional .gz.
+    notice_suffix = r'(?:xml|html|txt)(?:\.gz)?'
+    # <= Android 16: "[...] build <product>/obj/NOTICE.<ext>"
+    pattern_notice = rf'\[.*?\]\s*build\s+(([^\s]*?)/obj/NOTICE\.{notice_suffix})(?!\S)'
+    # Android 17+: "[...] Install: <product>/<partition>/etc/NOTICE.<ext>"
+    pattern_notice_install = rf'\[.*?\]\s*Install:\s+(\S+/NOTICE\.{notice_suffix})(?!\S)'
+    installed_from_log = []
+    seen_install = set()
     for line in reversed(android_log_lines):
         match = re.search(pattern_notice, line)
-        if match:
+        if match and not build_out_notice_file_path:
             build_out_notice_file_path = match.group(1)
             build_out_path = match.group(2)
-            break
+        inst = re.search(pattern_notice_install, line)
+        if inst:
+            path = inst.group(1)
+            if path not in seen_install:
+                seen_install.add(path)
+                installed_from_log.append(path)
 
     if not build_out_path:
         pattern = re.compile(r'.*Installed file list:\s*([^\s]+)')
@@ -311,7 +326,17 @@ def set_env_variables_from_result_log(android_src_path):
             logger.error("Can't find a build output path.")
             sys.exit(1)
 
-    if not build_out_notice_file_path:
+    # Legacy obj/NOTICE log line is absent on Android 17. Keep the installed
+    # Notice paths in the same variable instead of falling back to obj/.
+    if not build_out_notice_file_path and installed_from_log:
+        build_out_notice_file_path = (
+            installed_from_log if len(installed_from_log) > 1 else installed_from_log[0]
+        )
+        logger.info(
+            "Notice file from build log (Install): %s",
+            ", ".join(installed_from_log),
+        )
+    elif not build_out_notice_file_path:
         build_out_notice_file_path = os.path.join(build_out_path, "obj")
 
     read_module_info_from_build_output_file()
@@ -409,12 +434,37 @@ def get_result_of_notice_html(found_on_html, notice_file_found):
         return "nok"
 
 
+def _notice_paths():
+    if isinstance(build_out_notice_file_path, (list, tuple)):
+        return list(build_out_notice_file_path)
+    if build_out_notice_file_path:
+        return [build_out_notice_file_path]
+    return []
+
+
+def _read_notice_at(path):
+    # obj/ holds several NOTICE files. system/etc does not: sibling xml is unrelated.
+    normalized = path.rstrip("/")
+    parent_name = os.path.basename(os.path.dirname(normalized))
+    if os.path.basename(normalized) == "obj" or parent_name == "obj":
+        return read_notice_file(os.path.abspath(path))
+    return read_single_notice_file(os.path.abspath(path))
+
+
 def find_notice_value(notice_zip_dest_file=""):
     global notice_file_list, final_bin_info
     notice_file_comment = "Notice file not found."
 
     try:
-        notice_file_list, notice_files = read_notice_file(os.path.abspath(build_out_notice_file_path))
+        notice_file_list = {}
+        notice_files = []
+        for notice_path in _notice_paths():
+            found_list, found_files = _read_notice_at(notice_path)
+            if found_list:
+                notice_file_list.update(found_list)
+            for found_file in found_files:
+                if found_file not in notice_files:
+                    notice_files.append(found_file)
         if notice_file_list:
             if notice_files:
                 for notice_file in notice_files:

src/fosslight_android/check_notice_file.py

@@ -15,6 +15,9 @@
 
 logger = logging.getLogger(LOGGER_NAME)
 CANNOT_FIND_MSG = "CANNOT_FIND_NOTICE"
+# On-target and obj/ NOTICE filenames. Android does not fix one extension:
+# NOTICE.xml.gz (Android 17 partition install), NOTICE.html(.gz), NOTICE.txt(.gz).
+_NOTICE_FILE_SUFFIXES = (".xml.gz", ".html.gz", ".txt.gz", ".xml", ".html", ".txt")
 
 
 def run_notice_html_checklist(binary_file, check_type, notice_file):
@@ -73,8 +76,39 @@ def find_bin_in_notice(binary_file_name, notice_file_list):
     return notice_found
 
 
+def _is_xml_notice(file_name):
+    base = os.path.basename(file_name)
+    if base.endswith(".gz"):
+        base = base[:-3]
+    return base.endswith(".xml")
+
+
+def _read_notice_text(file_name):
+    encodings = ["utf-8", "utf-16", "latin-1"]
+    is_gz = file_name.endswith(".gz")
+    for encoding_option in encodings:
+        try:
+            if is_gz:
+                file = gzip.open(file_name, "rt", encoding=encoding_option)
+            else:
+                file = open(file_name, encoding=encoding_option)
+            file_content = file.read()
+            file.close()
+            if file_content != "":
+                return file_content
+        except Exception:
+            pass
+    return ""
+
+
+def _parse_notice_content(file_name, file_content):
+    if _is_xml_notice(file_name):
+        return parsing_notice_xml_format(file_content)
+    return parsing_notice_html_format(file_content)
+
+
 def find_files_by_extension(path):
-    extensions = ['.html', '.xml']
+    extensions = ['.html', '.xml', '.txt']
     GZ_EXTENSION = '.gz'
     files = []
     try:
@@ -89,7 +123,14 @@ def find_files_by_extension(path):
         logger.info(f"Fail unzip gz file:{error}")
 
     for extension in extensions:
-        files = [os.path.join(path, f) for f in os.listdir(path) if f.endswith(extension)]
+        files = []
+        for file_name in os.listdir(path):
+            if not file_name.endswith(extension):
+                continue
+            # .txt is common for unrelated files. Only NOTICE*.txt is a notice index.
+            if extension == '.txt' and not file_name.startswith('NOTICE'):
+                continue
+            files.append(os.path.join(path, file_name))
         if len(files) > 0:
             break
 
@@ -98,13 +139,11 @@ def find_files_by_extension(path):
 
 def read_notice_file(notice_file_path):
     final_notice_file = {}
-    # NOTICE.html need to be skipped the errors related to decode
-    encodings = ["latin-1", "utf-8", "utf-16"]
     notice_files = []
 
     if os.path.isfile(notice_file_path):
         notice_files.append(notice_file_path)
-        if notice_file_path.endswith((".xml", ".html", ".txt", "xml.gz")):
+        if notice_file_path.endswith(_NOTICE_FILE_SUFFIXES):
             notice_file_path = os.path.dirname(notice_file_path)
 
     if os.path.isdir(notice_file_path):
@@ -114,34 +153,39 @@ def read_notice_file(notice_file_path):
             notice_files = list(set(notice_files))
 
     for file_name in notice_files:
-        file_list = {}
-        file_content = ""
-        if os.path.isfile(file_name):
-            for encoding_option in encodings:
-                try:
-                    file = open(file_name, encoding=encoding_option)
-                    file_content = file.read()
-                    file.close()
-                    if file_content != "":
-                        break
-                except Exception:
-                    pass
-            if file_content != "":
-                try:
-                    if file_name.endswith("xml"):
-                        file_list = parsing_notice_xml_format(file_content)
-                    else:
-                        file_list = parsing_notice_html_format(file_content)
-                except Exception as error:
-                    logger.info("Can't read a notice. :" + file_name)
-                    logger.info(f"{error}")
-                final_notice_file.update(file_list)
-            else:
-                logger.info(f"Notice file is empty. :{file_name}")
+        found_list = _load_one_notice_file(file_name)
+        final_notice_file.update(found_list)
 
     return final_notice_file, notice_files
 
 
+def read_single_notice_file(notice_file_path):
+    """Read one installed NOTICE file. Do not scan the parent directory.
+
+    Partition install paths such as system/etc/NOTICE.xml.gz sit next to
+    unrelated xml/txt files. A directory scan would treat those as notices.
+    """
+    if not os.path.isfile(notice_file_path):
+        return {}, []
+    return _load_one_notice_file(notice_file_path), [notice_file_path]
+
+
+def _load_one_notice_file(file_name):
+    file_list = {}
+    if not os.path.isfile(file_name):
+        return file_list
+    file_content = _read_notice_text(file_name)
+    if file_content != "":
+        try:
+            file_list = _parse_notice_content(file_name, file_content)
+        except Exception as error:
+            logger.info("Can't read a notice. :" + file_name)
+            logger.info(f"{error}")
+    else:
+        logger.info(f"Notice file is empty. :{file_name}")
+    return file_list
+
+
 def parsing_notice_xml_format(notice_file_content):
     file_list = {}
     # NOTICE.xml is XML — use lxml XML parser (not HTML "lxml")