cmake - Cython、CMake 和源外构建导致双重编译

Question

我有一个大项目，其中包含多个创建库/可执行文件的项目，甚至还有一个使用库的 python 模块。

为了简单起见，我包含了一个名为的 MWE myprogram，它创建了一个可执行文件和一个共享库，供 Cython 代码使用，从而为相同的代码创建一个 python 包。我使用 CMake 编译代码，但由于我的设置，如此处详述， python 例程的构建过程执行两次（在build和期间install）。

在构建期间，输出为：

Scanning dependencies of target pymyprogram
[ 80%] Generating build/timestamp
Compiling /builds/myprogram/src/myprogram/pymyprogram.pyx because it changed.
[1/1] Cythonizing /builds/myprogram/src/myprogram/pymyprogram.pyx
running build_ext
building 'pymyprogram' extension
creating build
creating build/temp.linux-x86_64-3.8
creating build/temp.linux-x86_64-3.8/builds
creating build/temp.linux-x86_64-3.8/builds/myprogram
creating build/temp.linux-x86_64-3.8/builds/myprogram/src
creating build/temp.linux-x86_64-3.8/builds/myprogram/src/myprogram
gcc -Wno-unused-result -Wsign-compare -DNDEBUG -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector-strong --param=ssp-buffer-size=4 -grecord-gcc-switches -m64 -mtune=generic -D_GNU_SOURCE -fPIC -fwrapv -I/opt/rh/rh-python38/root/usr/include -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector-strong --param=ssp-buffer-size=4 -grecord-gcc-switches -m64 -mtune=generic -D_GNU_SOURCE -fPIC -fwrapv -I/opt/rh/rh-python38/root/usr/include -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector-strong --param=ssp-buffer-size=4 -grecord-gcc-switches -m64 -mtune=generic -D_GNU_SOURCE -fPIC -fwrapv -fPIC -I/builds/myprogram/src/myprogram -I/builds/myprogram/include -I/builds/myprogram/buildRelease -I/opt/rh/rh-python38/root/usr/include/python3.8 -c /builds/myprogram/src/myprogram/pymyprogram.c -o build/temp.linux-x86_64-3.8/builds/myprogram/src/myprogram/pymyprogram.o
creating build/lib.linux-x86_64-3.8
gcc -pthread -shared -L/opt/rh/rh-python38/root/usr/lib64-Wl,-z,relro -Wl,-rpath,/opt/rh/rh-python38/root/usr/lib64 -Wl,--enable-new-dtags -g -L/opt/rh/rh-python38/root/usr/lib64-Wl,-z,relro -Wl,-rpath,/opt/rh/rh-python38/root/usr/lib64 -Wl,--enable-new-dtags -g build/temp.linux-x86_64-3.8/builds/myprogram/src/myprogram/pymyprogram.o -L/builds/myprogram/buildRelease/src/myprogram -L/opt/rh/rh-python38/root/usr/lib64 -lmyprogram -o build/lib.linux-x86_64-3.8/pymyprogram.cpython-38-x86_64-linux-gnu.so -Wl,-rpath=/builds/myprogram/lib/
[ 80%] Built target pymyprogram

在安装过程中：

-- Installing: /builds/myprogram/bin/myprogram.x
running install
running build
running build_ext
building 'pymyprogram' extension
creating build
creating build/temp.linux-x86_64-3.8
creating build/temp.linux-x86_64-3.8/builds
creating build/temp.linux-x86_64-3.8/builds/myprogram
creating build/temp.linux-x86_64-3.8/builds/myprogram/src
creating build/temp.linux-x86_64-3.8/builds/myprogram/src/myprogram
gcc -Wno-unused-result -Wsign-compare -DNDEBUG -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector-strong --param=ssp-buffer-size=4 -grecord-gcc-switches -m64 -mtune=generic -D_GNU_SOURCE -fPIC -fwrapv -I/opt/rh/rh-python38/root/usr/include -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector-strong --param=ssp-buffer-size=4 -grecord-gcc-switches -m64 -mtune=generic -D_GNU_SOURCE -fPIC -fwrapv -I/opt/rh/rh-python38/root/usr/include -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector-strong --param=ssp-buffer-size=4 -grecord-gcc-switches -m64 -mtune=generic -D_GNU_SOURCE -fPIC -fwrapv -fPIC -I/builds/myprogram/src/myprogram -I/builds/myprogram/include -I/builds/myprogram/buildRelease -I/opt/rh/rh-python38/root/usr/include/python3.8 -c /builds/myprogram/src/myprogram/pymyprogram.c -o build/temp.linux-x86_64-3.8/builds/myprogram/src/myprogram/pymyprogram.o
creating build/lib.linux-x86_64-3.8
gcc -pthread -shared -L/opt/rh/rh-python38/root/usr/lib64-Wl,-z,relro -Wl,-rpath,/opt/rh/rh-python38/root/usr/lib64 -Wl,--enable-new-dtags -g -L/opt/rh/rh-python38/root/usr/lib64-Wl,-z,relro -Wl,-rpath,/opt/rh/rh-python38/root/usr/lib64 -Wl,--enable-new-dtags -g build/temp.linux-x86_64-3.8/builds/myprogram/src/myprogram/pymyprogram.o -L/builds/myprogram/buildRelease/src/myprogram -L/opt/rh/rh-python38/root/usr/lib64 -lmyprogram -o build/lib.linux-x86_64-3.8/pymyprogram.cpython-38-x86_64-linux-gnu.so -Wl,-rpath=/builds/myprogram/lib/
running install_lib
creating /builds/myprogram/lib64
creating /builds/myprogram/lib64/python3.8
creating /builds/myprogram/lib64/python3.8/site-packages
copying build/lib.linux-x86_64-3.8/pymyprogram.cpython-38-x86_64-linux-gnu.so -> /builds/program/lib/python3.8/site-packages
running install_egg_info
Writing /builds/myprogram/lib/python3.8/site-packages/pymyprogram-1.1A-py3.8.egg-info

此外，当我对任何一个 pyx 代码库进行更改otherprogram.c但没有更改时myprogram，在安装阶段，无论如何都会构建该库，这会增加编译时间。

因此，我搜索了一个解决方案并从PJ_Finnegan找到了该方法，但有一些问题，所以想知道其他人是否已经解决了这个问题或可以提供帮助。

我的 MWE 工作树是：

.
├── README.TXT
├── README.md
├── bin
├── CMakeLists.txt
├── buildDebug
│   ├── CMakeCache.txt
│   ├── CMakeFiles
│   ├── Makefile
│   ├── cmake_install.cmake
│   ├── install_manifest.txt
│   ├── src
│   └── utils
├── include
│   ├── version.h
│   └── version.h.txt
├── lib
│   └── libmyprogram.dylib
├─── src
│   ├── some other myprogram
│   │   ├── CMakeLists.txt
│   │   ├── othermyprogram.h
│   │   ├── othermyprogram.c
│   ├── myprogram
│   │   ├── CMakeLists.txt
│   │   ├── myprogram.h
│   │   ├── myprogram.c
│   │   ├── myprogram.pyx
│   │   ├── setup.py.in
│   └── versioning.cmake

顶层 CMakeLists.txt 包含add_subdirectory(src/myprogram)src/myprograms 中的 CMakeLists.txt 的内容是：

cmake_minimum_required(VERSION 2.8.8...3.20.5 FATAL_ERROR) 
project (myprogram)

message ("-- Configuring: *** myprogram **")

if(APPLE)
    set(CMAKE_MACOSX_RPATH 1)
endif()

set(CMAKE_C_FLAGS           "${CFLAGS} -O0 -ggdb -fPIC")
set(CMAKE_C_FLAGS_DEBUG     "${CFLAGS} -O0 -ggdb -fPIC")
set(CMAKE_C_FLAGS_RELEASE   "${CFLAGS} -O3       -fPIC")
set(CMAKE_CXX_FLAGS         "${CFLAGS} -O0 -ggdb")
set(CMAKE_CXX_FLAGS_DEBUG   "${CFLAGS} -O0 -ggdb")
set(CMAKE_CXX_FLAGS_RELEASE "${CFLAGS} -O3")

# *** myprogram.so   ***
SET(libmyprogram_SRCS
  myprogram.c
)

ADD_LIBRARY(myprogram SHARED
    ${libmyprogram_SRCS}
)

ADD_DEPENDENCIES(myprogram versioning)

TARGET_LINK_LIBRARIES(myprogram
    m
)

install(TARGETS myprogram 
         DESTINATION "lib")

if (APPLE)
    install(CODE "execute_process(COMMAND ln -sf ${PROJECT_HOME}/lib/libmyprogram.dylib /usr/local/lib/libmyprogram.dylib)")
endif()

# *** myprogram.x   ***
SET(myprogram_SRCS
  myprogram.c
)

ADD_EXECUTABLE(myprogram.x
    ${myprogram_SRCS}
)

ADD_DEPENDENCIES(myprogram.x myprogram versioning)

TARGET_LINK_LIBRARIES(myprogram.x
    m
)

set (EXECUTABLES "${EXECUTABLES}" myprogram.x)

# install executables and scripts
install (TARGETS ${EXECUTABLES} 
         RUNTIME DESTINATION "bin")

# *** pymyprogram.pyx   ***
find_package(PythonInterp 3 REQUIRED) 
if (CMAKE_BUILD_TYPE MATCHES "Debug")
    message(STATUS "Python: version=${PYTHON_VERSION_STRING} interpreter=${PYTHON_EXECUTABLE}")
    set(PYTHON_BUILD_FLAGS  "--debug")
endif()

# Get location of user site-packages to install libs to
execute_process(COMMAND python3 -m site --user-site OUTPUT_VARIABLE PYTHON_INSTALL_DIR OUTPUT_STRIP_TRAILING_WHITESPACE)

if (PYTHONINTERP_FOUND)
    set(SETUP_PY_IN "${CMAKE_CURRENT_SOURCE_DIR}/setup.py.in")
    set(SETUP_PY    "${CMAKE_CURRENT_BINARY_DIR}/setup.py")
    set(DEPS        "${CMAKE_CURRENT_SOURCE_DIR}/pymyprogram.pyx")
    set(OUTPUT      "${CMAKE_CURRENT_BINARY_DIR}/build/timestamp")

    message ("-- Building Python Extension using: " ${PYTHON_EXECUTABLE})

    configure_file(${SETUP_PY_IN} ${SETUP_PY})

    add_custom_command(OUTPUT ${OUTPUT}
                       COMMAND ${PYTHON_EXECUTABLE} ${SETUP_PY} build ${PYTHON_BUILD_FLAGS} --build-lib ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_PROJECT_NAME}
                       COMMAND ${CMAKE_COMMAND} -E touch ${OUTPUT}
                       DEPENDS ${DEPS})

    add_custom_target(pymyprogram ALL DEPENDS ${OUTPUT})

    add_dependencies(pymyprogram myprogram versioning)
    install(CODE "execute_process(COMMAND ${PYTHON_EXECUTABLE} ${SETUP_PY} install_lib --skip-build
                --build-dir ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_PROJECT_NAME}
                --install-dir=${PYTHON_INSTALL_DIR} --force 
                COMMAND ${PYTHON_EXECUTABLE} ${SETUP_PY} install_egg_info
                --install-dir=${PYTHON_INSTALL_DIR})" DEPENDS ${DEPS})
else()
    message (WARNING "-- Could not build *** pymyprogram **, could not locate PYTHON: ${PYTHON_EXECUTABLE} or improper version")
endif()

以及使用的 setup.py.in：

import os, sys
from distutils.core import setup, Extension
from Cython.Build import cythonize


link_arguments = []
if (sys.platform == 'darwin'):
    link_arguments.append("-Wl,-rpath")
    link_arguments.append("-Wl,@loader_path/")
    os.environ["CC"] = "gcc"
else:
    link_arguments.append("-Wl,-rpath=${CMAKE_SOURCE_DIR}/lib/")

myprogram_extension = Extension(
    name="pymyprogram",
    sources=["${CMAKE_CURRENT_SOURCE_DIR}/pymyprogram.pyx"],
    libraries=["myprogram"],
    extra_link_args = link_arguments,
    library_dirs=["${PROJECT_BINARY_DIR}"],
    include_dirs=["${CMAKE_SOURCE_DIR}/include", "${CMAKE_SOURCE_DIR}/build${CMAKE_BUILD_TYPE}"],
)

setup(name="pymyprogram",
      author="My Name",
      author_email="my-email",
      version="1.1A",
      description="Python wrapper for myprogram",
      package_dir={ "": "${CMAKE_SOURCE_DIR}/lib/" },
      license="MIT License",
      ext_modules=cythonize([myprogram_extension], compiler_directives={'language_level' : "3"})
      )

代码在不同的机器（Linux 或 macOS）上运行，我的模块之前是使用此方法构建和安装的，安装命令的--user或取决于代码是在 Linux 还是 macOS 上运行。--prefix

为了避免 python 模块的双重构建步骤，我从 PJ_Finnegan 实现了一个新方法，如上所述，并在我的 CMakeLists.txt 的代码示例中显示。此方法使用install_lib来避免第二次构建，它允许使用--build-dir它很好但不再适用--user或--prefix不需要，因为它需要我手动定义路径并且路径不再自动依赖于 python 版本。

我也想知道使用install-lib与install之间的区别是什么。我注意到没有egg-info创建文件，这是一个问题吗？因此，我也添加了install_egg_info步骤。

此外，mypogram 代码构建了两次（可执行文件和库），第三次构建 python 包。是否有一些优化可以减少编译时间？

任何其他方式来构建和安装 python/Cython 扩展和单独的独立可执行文件并帮助安装步骤找到构建代码？也许使用sdist和pip install?

编辑

我已将 CMAKE 文件中的 INSTALL 命令更改为：

install(CODE "execute_process(COMMAND ${PYTHON_EXECUTABLE} ${SETUP_PY} install ${PYTHON_INSTALL_PREFIX} --skip-build --force WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})")

这将安装步骤减少为仅安装并且不再构建，因为现在可以找到上一个构建步骤的构建目录。

现在问题仍然存在，如何使 CMAKE 安装命令不是每次都运行，并且仅在 *.pyx 文件发生更改时运行。我知道 add_custom_target 总是被认为是过时的，所以可能有解决方法吗？