CMake - linking to library downloaded from ExternalProject_add()

Question

I am trying to use ExternalProject_add() to download/install dependencies. It installs fine, but I can't figure out how to actually link the libraries after they are downloaded.

I want to call target_link_libraries() on the library that was just downloaded, but the path to the library will vary by system.

If this were a system dependency, I could just call find_package() - but the packages weren't installed on the default search path. I don't think you can specify a search path for find_package in module mode.

Here's a snippet of my CMakeLists.txt that doesn't work:

ExternalProject_Add(
protobuf
URL http://protobuf.googlecode.com/files/protobuf-2.4.1.tar.gz
CONFIGURE_COMMAND <SOURCE_DIR>/configure --prefix=<INSTALL_DIR>
PREFIX ${MYPROJ_SOURCE_DIR}/dependencies
)
find_package(protobuf REQUIRED)
set(LIBS ${LIBS} ${PROTOBUF_LIBRARIES})
target_link_libraries (mybinary ${LIBS})

Knowles

See also #37553780 – Kaufmann

Answer 1

When you're using ExternalProject_Add, you can't use find_package, since there's nothing to find when CMake runs to configure the outer project.

So, if library locations vary by platform you will need conditional logic based on your platform. (I don't know protobuf's libraries or structure here, so this is just an example, but it should get you headed in the right direction...) Something like this:

if(WIN32)
  set(PROTOBUF_LIB_DIR "${MYPROJ_SOURCE_DIR}/dependencies/win"
  set(prefix "")
  set(suffix ".lib")
elseif(APPLE)
  set(PROTOBUF_LIB_DIR "${MYPROJ_SOURCE_DIR}/dependencies/mac"
  set(prefix "lib")
  set(suffix ".a")
else()
  set(PROTOBUF_LIB_DIR "${MYPROJ_SOURCE_DIR}/dependencies/linux"
  set(prefix "lib")
  set(suffix ".a")
endif()

set(PROTOBUF_LIBRARIES
  "${PROTOBUF_LIB_DIR}/${prefix}protobufLib1${suffix}"
  "${PROTOBUF_LIB_DIR}/${prefix}protobufLib2${suffix}")

Granted, this is less convenient than using find_package. If you can use a pre-built/pre-installed package, you should, so that you can use find_package. If you must build the other package from source code as part of your project, though, ExternalProject_Add is useful, even though it is unable to abstract away all the details for you.

Knobkerrie

Thanks, that's really helpful. Also just so I don't cause confusion, protobuf doesn't have variable library locations, but R does. – Knowles

cmake provides prefix and suffix variables for your platform: "${install_dir}/lib/${CMAKE_SHARED_MODULE_PREFIX}protobufLib1${CMAKE_SHARED_LIBRARY_SUFFIX}" – Allocate

While this works, I think it's horrible. What's the point of using cmake if I have to write platform conditional code? Moreover, I would expect ExternalProject to provide a simple way to import the built targets, why would I want to build an external project if I can not import them easily afterwards? What's the point? Am I missing something? – Opinionated

You're not missing anything @Jens, but the scenario is not the normal use of ExternalProject with CMake. In fact, I would recommend against it. The typical scenario is all ExternalProject, or no ExternalProject, not this mixed-case of some external, some not. When all components are built/installed by ExternalProject in a so called "super build" scenario, later built components can use find_package to find earlier built/installed components. That's the recommended way to use ExternalProject. – Knobkerrie

Answer 2

Because you're downloading the external project, you already know where everything is because you just downloaded it, so it doesn't need 'finding'.

I got it working with add_library. This is my actual code that works:

ExternalProject_Add(ForexConnectDownload
    PREFIX 3rd_party
    #--Download step--------------
    URL http://fxcodebase.com/bin/forexconnect/1.3.1/ForexConnectAPI-1.3.1-Linux-x86_64.tar.gz
    URL_HASH SHA1=7fdb90a2d45085feb8b76167cae419ad4c211d6b
    #--Configure step-------------
    CONFIGURE_COMMAND ""
    #--Build step-----------------
    BUILD_COMMAND ""
    #--Install step---------------
    UPDATE_COMMAND "" # Skip annoying updates for every build
    INSTALL_COMMAND ""
)

SET(FXCM_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/3rd_party/src/ForexConnectDownload/include)
SET(FXCM_LIB_DIR ${CMAKE_CURRENT_BINARY_DIR}/3rd_party/src/ForexConnectDownload/lib)

add_library(ForexConnect SHARED IMPORTED)
set_target_properties(ForexConnect PROPERTIES IMPORTED_LOCATION ${FXCM_LIB_DIR}/libForexConnect.so)

From there, each program that depends on it needs a add_dependencies and of course target_link_libraries. Eg:

include_directories(${FXCM_INCLUDE_DIR})
add_executable(syncDatabase syncDatabase.cpp trader/database.cpp trader/fxcm.cpp)
target_link_libraries(syncDatabase ForexConnect)
add_dependencies(syncDatabase ForexConnectDownload)

• include_directories - tells it to search for directories there
• target_link_libraries - just add your library, as you named it (not a variable)

The add_dependencies makes it wait before trying to include the dirs needed.

That does the trick for me. Works with make -j4. Get's all the dependencies right.

Firewarden

Since cmake 3.2 you can use UPDATE_DISCONNECTED to avoid automatic updates. You can still update adding a custom target if necessary. – Opinionated

how does it know ForexConnect ? example here add_library(ForexConnect SHARED IMPORTED) – Pinnatipartite

Really great solution, that even goes beyond the simple differentiation and manages dependencies and linking. – Lovash

Look like an elegant solution. It works well with make but not with ninja. For the latter, you have to set the BUILD_BYPRODUCTS propertie of your external project with the path of the generated file. If not, Ninja will scream that it does not find your library. – Stemson

@Pinnatipartite Just to not let that go unanswered: it doesn't know ForexConnect, the add_library() creates it as a new target. ( add_library always creates new targets, IMPORTED just specifies what type of target to create.) the IMPORTED_LOCATION property gets set on it after, that's how it knows where the ForexConnect library file can be found. – Dynamoelectric

This looks like it only works on Linux (your usage of lib***.so). How would you get this to cross-compile? – Fallow

The IMPORTED target ForexConnect could incorporate include directories with target_include_directories(ForexConnect INTERFACE ${FXCM_INCLUDE_DIR}) and could incorporate dependency from the external project with add_dependencies(ForexConnect ForexConnectDownload). So using the library for the executable will require only a single line: target_link_libraries(syncDatabase ForexConnect). – Adjust

@JoshuaHyatt answer could be edited to use CMAKE_SHARED_LIBRARY_SUFFIX. – Pothole

@Tsyvarev, is your comment an alternate/improved solution to the problem presented by @gagou7? – Pothole

My comment suggested the calls to target_include_directories and add_dependencies instead the ones provided in the answer. BUILD_BYPRODUCTS is still needed for work with Ninja. – Adjust

Answer 3

To expand on DLRdave answer above you don't need set manually prefixes and suffixes for static libraries because CMAKE provides variables with the right ones for each platform.

See CMake Useful Variables for more information.

For example:

• CMAKE_SHARED_LIBRARY_PREFIX
• CMAKE_SHARED_LIBRARY_SUFFIX
• CMAKE_STATIC_LIBRARY_PREFIX
• CMAKE_STATIC_LIBRARY_SUFFIX

Refresher

Answer 4

You can use the link_directories command to link libraries within a specific directory. In your case the directory were your external project is build.

ExternalProject_Add(MyExternalLibrary ...)

Add the output directory to the search path:

link_directories(${CMAKE_BINARY_DIR}/lib/MyExternalLibrary-prefix/lib)

Make sure to add the executable after specifying the link directory:

add_executable(MyProgram main.c)

Specify the libraries your project should be linked to:

target_link_libraries(MyProgram ExternalLibraryName)

Don't forget to depend on the external project:

add_dependencies(MyProgram MyExternalLibrary)

Goddamned

I feel like actual link_directories string should be longer as I see something like MyExternalLibrary-build/lib under lib in my case. – Tempa

I tried this technique and it doesn't work on a new build. You get the error (your library couldn't be found). – Firewarden

Answer 5

Another idiom you can use to solve this problem:

1. Make your find_package commands optional (remove 'REQUIRED')
2. Add some condition code to only build your targets if find_package succeeds.
3. find_package will fail and your targets won't be built the first time around, but the External Projects will be built.
4. Run cmake/make again, this time find_package will succeed and your targets will be built.

You can see this idiom in action in https://github.com/biometrics/likely.

Blister

Answer 6

You can use find_package with ExternalProject_add like the following code snippet:

# --------------------------------------------------

function (build_external_project target file_name)

    set(CMAKELIST_CONTENT "
        cmake_minimum_required(VERSION ${CMAKE_MINIMUM_REQUIRED_VERSION})

        project(build_external_project)

        file(MD5 \"${file_name}\" FILE_HASH)

        include(ExternalProject)
        ExternalProject_add(${target}
            URL \"${file_name}\"
            URL_MD5 \${FILE_HASH}
            CMAKE_GENERATOR \"${CMAKE_GENERATOR}\"
            CMAKE_GENERATOR_PLATFORM \"${CMAKE_GENERATOR_PLATFORM}\"
            CMAKE_GENERATOR_TOOLSET \"${CMAKE_GENERATOR_TOOLSET}\"
            CMAKE_GENERATOR_INSTANCE \"${CMAKE_GENERATOR_INSTANCE}\"
            CMAKE_ARGS ${ARGN})

        add_custom_target(build_external_project)
        add_dependencies(build_external_project ${target})
    ")

    set(TARGET_DIR "${CMAKE_CURRENT_BINARY_DIR}/ExternalProjects/${target}")

    file(WRITE "${TARGET_DIR}/CMakeLists.txt" "${CMAKELIST_CONTENT}")

    file(MAKE_DIRECTORY "${TARGET_DIR}" "${TARGET_DIR}/build")

    execute_process(COMMAND ${CMAKE_COMMAND}
        -G "${CMAKE_GENERATOR}"
        -A "${CMAKE_GENERATOR_PLATFORM}"
        -T "${CMAKE_GENERATOR_TOOLSET}"
        ..
        WORKING_DIRECTORY "${TARGET_DIR}/build")

    execute_process(COMMAND ${CMAKE_COMMAND}
        --build .
        --config ${CMAKE_BUILD_TYPE}
        WORKING_DIRECTORY "${TARGET_DIR}/build")

endfunction()

#----------------------------------------------------------------------------------------------------

set(THIDR_PARTY_DIR "${CMAKE_CURRENT_LIST_DIR}")
set(THIDR_PARTY_INSTALL_DIR "${CMAKE_CURRENT_BINARY_DIR}/ThirdParty")

#----------------------------------------------------------------------------------------------------

# OpenCV.

set(OPENCV_OPTIONS "")

list(APPEND OPENCV_OPTIONS -D WITH_1394=ON)
list(APPEND OPENCV_OPTIONS -D WITH_ADE=ON)
list(APPEND OPENCV_OPTIONS -D WITH_CUDA=OFF)
list(APPEND OPENCV_OPTIONS -D WITH_EIGEN=ON)
list(APPEND OPENCV_OPTIONS -D WITH_FFMPEG=ON)
list(APPEND OPENCV_OPTIONS -D WITH_IPP=ON)
list(APPEND OPENCV_OPTIONS -D WITH_ITT=ON)
list(APPEND OPENCV_OPTIONS -D WITH_JASPER=ON)
list(APPEND OPENCV_OPTIONS -D WITH_JPEG=ON)
list(APPEND OPENCV_OPTIONS -D WITH_OPENCL=ON)
list(APPEND OPENCV_OPTIONS -D WITH_OPENEXR=ON)
list(APPEND OPENCV_OPTIONS -D WITH_OPENJPEG=ON)
list(APPEND OPENCV_OPTIONS -D WITH_PNG=ON)
list(APPEND OPENCV_OPTIONS -D WITH_PROTOBUF=ON)
list(APPEND OPENCV_OPTIONS -D WITH_QT=OFF)
list(APPEND OPENCV_OPTIONS -D WITH_TBB=ON)
list(APPEND OPENCV_OPTIONS -D WITH_TIFF=ON)
list(APPEND OPENCV_OPTIONS -D WITH_WEBP=ON)

list(APPEND OPENCV_OPTIONS -D BUILD_CUDA_STUBS=OFF)
list(APPEND OPENCV_OPTIONS -D BUILD_DOCS=OFF)
list(APPEND OPENCV_OPTIONS -D BUILD_EXAMPLES=OFF)
list(APPEND OPENCV_OPTIONS -D BUILD_IPP_IW=ON)
list(APPEND OPENCV_OPTIONS -D BUILD_ITT=ON)
list(APPEND OPENCV_OPTIONS -D BUILD_JASPER=ON)
list(APPEND OPENCV_OPTIONS -D BUILD_JAVA=OFF)
list(APPEND OPENCV_OPTIONS -D BUILD_JPEG=ON)
list(APPEND OPENCV_OPTIONS -D BUILD_OPENEXR=ON)
list(APPEND OPENCV_OPTIONS -D BUILD_PACKAGE=OFF)
list(APPEND OPENCV_OPTIONS -D BUILD_PERF_TESTS=OFF)
list(APPEND OPENCV_OPTIONS -D BUILD_PNG=ON)
list(APPEND OPENCV_OPTIONS -D BUILD_PROTOBUF=ON)
list(APPEND OPENCV_OPTIONS -D BUILD_TBB=ON)
list(APPEND OPENCV_OPTIONS -D BUILD_TESTS=OFF)
list(APPEND OPENCV_OPTIONS -D BUILD_TIFF=ON)
list(APPEND OPENCV_OPTIONS -D BUILD_WEBP=ON)
list(APPEND OPENCV_OPTIONS -D BUILD_ZLIB=ON)

list(APPEND OPENCV_OPTIONS -D BUILD_opencv_apps=OFF)
list(APPEND OPENCV_OPTIONS -D BUILD_opencv_java_bindings_generator=OFF)
list(APPEND OPENCV_OPTIONS -D BUILD_opencv_js=OFF)
list(APPEND OPENCV_OPTIONS -D BUILD_opencv_python_bindings_generator=OFF)
list(APPEND OPENCV_OPTIONS -D BUILD_opencv_python_tests=OFF)
list(APPEND OPENCV_OPTIONS -D BUILD_opencv_world=OFF)

list(APPEND OPENCV_OPTIONS -D OPENCV_ENABLE_NONFREE=OFF)
list(APPEND OPENCV_OPTIONS -D OPENCV_FORCE_3RDPARTY_BUILD=ON)
list(APPEND OPENCV_OPTIONS -D BUILD_WITH_STATIC_CRT=OFF)

list(APPEND OPENCV_OPTIONS -D CMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE})
list(APPEND OPENCV_OPTIONS -D CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE})
list(APPEND OPENCV_OPTIONS -D CMAKE_POSITION_INDEPENDENT_CODE=ON)
list(APPEND OPENCV_OPTIONS -D BUILD_SHARED_LIBS=OFF)
list(APPEND OPENCV_OPTIONS -D CMAKE_INSTALL_PREFIX=${THIDR_PARTY_INSTALL_DIR}/OpenCV)

build_external_project(OpenCV "${THIDR_PARTY_DIR}/OpenCV/opencv-4.3.0+cache.zip" ${OPENCV_OPTIONS})

set(OpenCV_STATIC ON)

#----------------------------------------------------------------------------------------------------

# Dlib.

set(DLIB_OPTIONS "")

list(APPEND DLIB_OPTIONS -D USE_SSE2_INSTRUCTIONS=ON)
list(APPEND DLIB_OPTIONS -D USE_SSE4_INSTRUCTIONS=ON)
list(APPEND DLIB_OPTIONS -D USE_AVX_INSTRUCTIONS=ON)

list(APPEND DLIB_OPTIONS -D DLIB_GIF_SUPPORT=ON)
list(APPEND DLIB_OPTIONS -D DLIB_JPEG_SUPPORT=ON)
list(APPEND DLIB_OPTIONS -D DLIB_PNG_SUPPORT=ON)
list(APPEND DLIB_OPTIONS -D DLIB_USE_BLAS=ON)
list(APPEND DLIB_OPTIONS -D DLIB_USE_CUDA=OFF)
list(APPEND DLIB_OPTIONS -D DLIB_USE_LAPACK=ON)
list(APPEND DLIB_OPTIONS -D DLIB_USE_MKL_FFT=OFF)
list(APPEND DLIB_OPTIONS -D DLIB_LINK_WITH_SQLITE3=OFF)
list(APPEND DLIB_OPTIONS -D DLIB_NO_GUI_SUPPORT=OFF)
list(APPEND DLIB_OPTIONS -D DLIB_ENABLE_ASSERTS=OFF)
list(APPEND DLIB_OPTIONS -D DLIB_ENABLE_STACK_TRACE=OFF)

list(APPEND DLIB_OPTIONS -D CMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE})
list(APPEND DLIB_OPTIONS -D CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE})
list(APPEND DLIB_OPTIONS -D CMAKE_POSITION_INDEPENDENT_CODE=ON)
list(APPEND DLIB_OPTIONS -D BUILD_SHARED_LIBS=OFF)
list(APPEND DLIB_OPTIONS -D CMAKE_INSTALL_PREFIX=${THIDR_PARTY_INSTALL_DIR}/dlib)

build_external_project(dlib ${THIDR_PARTY_DIR}/Dlib/dlib-19.20.tar.bz2 ${DLIB_OPTIONS})

#----------------------------------------------------------------------------------------------------

####################################################################################################
# Now, you can use find_package:
find_package(OpenCV REQUIRED PATHS "${THIDR_PARTY_INSTALL_DIR}/OpenCV" NO_DEFAULT_PATH)
find_package(dlib REQUIRED PATHS "${THIDR_PARTY_INSTALL_DIR}/dlib" NO_DEFAULT_PATH)
####################################################################################################

https://gist.github.com/amir-saniyan/4339e6f3ef109c75eda8018f7d5192a7

Keats

Interesting workaround. It is really bad practice for projects to not be flexible enough to be used with add_subdirectory instead of find_package depending on context. – Hensley

One may also read: #37553780 – Kaufmann