Add AIX support

CMakeLists.txt

@@ -629,7 +629,7 @@ if(OCOS_ENABLE_VISION)
     if (NOT OCOS_ENABLE_DLIB)
       message(FATAL_ERROR "Vision operators require DLIB to be enabled.") # for now, we need dlib for image processing
     endif()
-    include(ext_imgcodecs)  
+    include(ext_imgcodecs)
     target_include_directories(ocos_operators PUBLIC ${libPNG_SOURCE_DIR} ${libJPEG_SOURCE_DIR})
     target_link_libraries(ocos_operators PUBLIC ${PNG_LIBRARY} ${JPEG_LIBRARY})
   endif()
@@ -763,9 +763,14 @@ if(OCOS_BUILD_SHARED_LIB)
   source_group(TREE ${PROJECT_SOURCE_DIR} FILES ${shared_TARGET_SRC})
   standardize_output_folder(extensions_shared)
 
-  if(LINUX OR ANDROID)
-    set_property(TARGET extensions_shared APPEND_STRING PROPERTY LINK_FLAGS
-      " -Wl,--version-script -Wl,${PROJECT_SOURCE_DIR}/shared/ortcustomops.ver")
+ if(LINUX OR ANDROID OR CMAKE_SYSTEM_NAME STREQUAL "AIX")
+    if(CMAKE_SYSTEM_NAME STREQUAL "AIX")
+      set_property(TARGET extensions_shared APPEND_STRING PROPERTY LINK_FLAGS
+        " -Wl,-bE:${PROJECT_SOURCE_DIR}/shared/ortcustomops.exp")
+    else()
+      set_property(TARGET extensions_shared APPEND_STRING PROPERTY LINK_FLAGS
+        " -Wl,--version-script -Wl,${PROJECT_SOURCE_DIR}/shared/ortcustomops.ver")
+    endif()
     # strip if not a debug build
     if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
       set_property(TARGET extensions_shared APPEND_STRING PROPERTY LINK_FLAGS " -Wl,-s")

cmake/externals/blingfire-aix.patch

@@ -0,0 +1,42 @@
+diff --git a/blingfireclient.library/inc/FAConfig.h b/blingfireclient.library/inc/FAConfig.h
+index 8dfa36e..b6c7cb1 100644
+--- a/blingfireclient.library/inc/FAConfig.h
++++ b/blingfireclient.library/inc/FAConfig.h
+@@ -23,7 +23,7 @@
+ #include <ctype.h>
+ #include <fcntl.h>
+
+-#ifdef BLING_FIRE_MAC
++#if defined(BLING_FIRE_MAC) || defined(_AIX)
+   #include <string>
+ #else
+   #include <malloc.h>
+
+
+
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index 5d47b3a..428c888 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -26,7 +26,11 @@ IF (WIN32 AND MSVC)
+   add_link_options("$<$<NOT:$<CONFIG:Debug>>:/LTCG>")
+   set (default_build_type "RelWithDebInfo")
+ ELSE()
+-  set (CMAKE_CXX_FLAGS " -std=c++11 -fpic ${OPTIMIZE} -DNDEBUG")
++  IF(AIX)
++  set (CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -std=c++11 -fpic ${OPTIMIZE} -DNDEBUG")
++  ELSE()
++  set (CMAKE_CXX_FLAGS " -std=c++11 -fpic ${OPTIMIZE} -DNDEBUG")
++  ENDIF()
+   IF(APPLE)
+     set (CMAKE_CXX_FLAGS_RELEASE " -std=c++11 -fpic ${OPTIMIZE} -DNDEBUG -ffunction-sections -fdata-sections -Wl,-dead_strip")
+   ELSE()
+@@ -107,7 +111,7 @@ FOREACH(dir ${SUBDIRS})
+       IF(${dirname} STREQUAL "any_test")
+         target_link_libraries(${dirname} ${CMAKE_DL_LIBS})
+       ENDIF()
+-      IF (APPLE)
++      IF (APPLE OR CMAKE_SYSTEM_NAME MATCHES "AIX")
+         target_link_libraries(${dirname} iconv)
+       ENDIF()
+     ENDIF()

cmake/externals/blingfire.cmake

@@ -3,7 +3,13 @@ FetchContent_Declare(
   GIT_REPOSITORY https://github.com/microsoft/BlingFire.git
   GIT_TAG 0831265c1aca95ca02eca5bf1155e4251e545328
   EXCLUDE_FROM_ALL
-  PATCH_COMMAND git checkout . && git apply --ignore-space-change --ignore-whitespace ${PROJECT_SOURCE_DIR}/cmake/externals/blingfire_cmake.patch)
+  PATCH_COMMAND git checkout . && git apply --ignore-space-change --ignore-whitespace ${PROJECT_SOURCE_DIR}/cmake/externals/blingfire_cmake.patch  && git apply --ignore-space-change --ignore-whitespace  ${PROJECT_SOURCE_DIR}/cmake/externals/blingfire-aix.patch
+  )
+
+if(CMAKE_SYSTEM_NAME STREQUAL "AIX")
+  find_package(Iconv REQUIRED)
+  link_libraries(Iconv::Iconv)
+endif()
 
 FetchContent_MakeAvailable(Blingfire)
 set_target_properties(bingfirtinydll_static PROPERTIES FOLDER

cmake/externals/opencv.cmake

@@ -149,8 +149,8 @@ FetchContent_Declare(
     -DBUILD_TESTS:BOOL=FALSE
     -DBUILD_SHARED_LIBS:BOOL=FALSE
     -DCMAKE_INSTALL_PREFIX:PATH=${CMAKE_CURRENT_BINARY_DIR}/opencv
-    PATCH_COMMAND git checkout . && git apply --whitespace=fix --ignore-space-change --ignore-whitespace ${CMAKE_CURRENT_SOURCE_DIR}/cmake/externals/opencv-no-rtti.patch
     EXCLUDE_FROM_ALL
+    PATCH_COMMAND git checkout . && git apply --whitespace=fix --ignore-space-change --ignore-whitespace ${CMAKE_CURRENT_SOURCE_DIR}/cmake/externals/opencv-no-rtti.patch
 )
 
 FetchContent_MakeAvailable(opencv)

cmake/externals/sentencepieceproject.cmake

@@ -39,7 +39,10 @@ endif()
 # PB files was seperated as another patch file to avoid the patch file too large to be reviewed.
 set(spm_patches
   "${PROJECT_SOURCE_DIR}/cmake/externals/sentencepieceproject_cmake.patch"
-  "${PROJECT_SOURCE_DIR}/cmake/externals/sentencepieceproject_pb.patch")
+  "${PROJECT_SOURCE_DIR}/cmake/externals/sentencepieceproject_pb.patch"
+ "${PROJECT_SOURCE_DIR}/cmake/externals/sentencepieceproject_aix.patch"
+ "${PROJECT_SOURCE_DIR}/cmake/externals/sentencepieceproject_enum.patch"
+)
 
 set(spm_patch_command git checkout . && git apply --ignore-space-change --ignore-whitespace ${spm_patches})
 

cmake/externals/sentencepieceproject_aix.patch

@@ -0,0 +1,15 @@
+diff --git a/src/util.h b/src/util.h
+index 0d15863..b2d490f 100644
+--- a/src/util.h
++++ b/src/util.h
+@@ -36,7 +36,9 @@
+ #include <pthread.h>
+ #endif
+
+-#if !defined(__APPLE__) && !defined(_WIN32)
++#if defined(_AIX)
++#define IS_BIG_ENDIAN
++#elif !defined(__APPLE__) && !defined(_WIN32)
+ #include <endian.h>
+ #if BYTE_ORDER == __BIG_ENDIAN
+ #define IS_BIG_ENDIAN

cmake/externals/sentencepieceproject_enum.patch

@@ -0,0 +1,13 @@
+diff --git a/src/unicode_script.h b/src/unicode_script.h
+index 917c49d..d249539 100644
+--- a/src/unicode_script.h
++++ b/src/unicode_script.h
+@@ -19,7 +19,7 @@
+
+ namespace sentencepiece {
+ namespace unicode_script {
+-enum ScriptType {
++enum ScriptType:int32_t {
+   U_Adlam,
+   U_Ahom,
+   U_Anatolian_Hieroglyphs,

operators/text/vector_to_string.hpp

@@ -13,7 +13,7 @@ namespace std {
 
 template <class T>
 struct hash<std::vector<T>> {
-  size_t operator()(const vector<T>& __vector) const noexcept;
+  size_t operator()(const vector<T>& __vec) const noexcept;
 };
 }  // namespace std
 

operators/tokenizer/ugm_kernels.hpp

@@ -99,6 +99,12 @@ struct SpmUgmTokenizer {
       }
       uint32_t xcda_blob_size;
       memcpy(&xcda_blob_size, charsmap_data_.data(), sizeof(uint32_t));
+
+      // Convert from little-endian to host byte order
+      #if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+      xcda_blob_size = __builtin_bswap32(xcda_blob_size);
+      #endif
+
       charsmap_offset += sizeof(xcda_blob_size);
       if (xcda_blob_size > charsmap_data_.size() - charsmap_offset) {
         return OrtxStatus(extError_t::kOrtxErrorCorruptData, "Index out of array bounds in precompiled charsmap!");
@@ -301,7 +307,7 @@ struct SpmUgmTokenizer {
                         "Invalid UTF-8 encoding detected in input string at position " +
                         std::to_string(-validation_result));
     }
-    
+
     std::string normalized;
     if (case_encoder_) {
       normalized = NmtNormalize(input);
@@ -642,7 +648,12 @@ struct SpmUgmTokenizer {
       if (index >= xcda_array_size_) {  // Fix #2: off-by-one correction
         ORTX_CXX_API_THROW("[UgmTok]Index out of array bounds in XCDA array!", ORT_RUNTIME_EXCEPTION);
       }
-      return xcda_array_[index];
+      uint32_t value = xcda_array_[index];
+      // Convert from little-endian to host byte order
+      #if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+      value = __builtin_bswap32(value);
+      #endif
+      return value;
     }
     const uint32_t* xcda_array_;
     size_t xcda_array_size_;
@@ -1067,4 +1078,4 @@ class SpmUgmDecoder {
   std::set<extTokenId_t> special_token_ids_;
 };
 
-}  // namespace ort_extensions
+}  // namespace ort_extensions

shared/api/nemo_mel_spectrogram.cc

@@ -18,6 +18,10 @@
 #define M_PI 3.14159265358979323846
 #endif
 
+#if defined(_AIX) && defined(hz)
+#undef hz
+#endif
+
 namespace nemo_mel {
 
 // Apply pre-emphasis filter: y[n] = x[n] - preemph * x[n-1]

shared/api/nemo_mel_spectrogram.h

@@ -9,6 +9,10 @@
 #include <utility>
 #include <vector>
 
+#if defined _AIX && defined (hz)
+#undef hz
+#endif
+
 namespace nemo_mel {
 
 struct NemoMelConfig {

shared/ortcustomops.exp

@@ -0,0 +1,3 @@
+RegisterCustomOps
+AddExternalCustomOp
+GetActiveOrtAPIVersion