sci-libs/tensorflow/tensorflow-2.0.0.ebuild

   1 # Copyright 1999-2020 Gentoo Authors
   2 # Distributed under the terms of the GNU General Public License v2
   3
   4 EAPI=7
   5
   6 DISTUTILS_OPTIONAL=1
   7 PYTHON_COMPAT=( python{3_6,3_7} )
   8 MY_PV=${PV/_rc/-rc}
   9 MY_P=${PN}-${MY_PV}
  10
  11 inherit bazel check-reqs cuda distutils-r1 flag-o-matic toolchain-funcs
  12
  13 DESCRIPTION="Computation framework using data flow graphs for scalable machine learning"
  14 HOMEPAGE="https://www.tensorflow.org/"
  15
  16 LICENSE="Apache-2.0"
  17 SLOT="0"
  18 KEYWORDS="~amd64"
  19 IUSE="cuda mpi +python xla"
  20 CPU_USE_FLAGS_X86="sse sse2 sse3 sse4_1 sse4_2 avx avx2 fma3 fma4"
  21 for i in $CPU_USE_FLAGS_X86; do
  22         IUSE+=" cpu_flags_x86_$i"
  23 done
  24
  25 # distfiles that bazel uses for the workspace, will be copied to basel-distdir
  26 bazel_external_uris="
  27         https://storage.googleapis.com/mirror.tensorflow.org/www.kurims.kyoto-u.ac.jp/~ooura/fft2d.tgz -> oourafft2d-20061228.tgz
  28         https://bitbucket.org/eigen/eigen/get/049af2f56331.tar.gz -> eigen-049af2f56331.tar.gz
  29         https://github.com/abseil/abseil-cpp/archive/43ef2148c0936ebf7cb4be6b19927a9d9d145b8f.tar.gz -> abseil-cpp-43ef2148c0936ebf7cb4be6b19927a9d9d145b8f.tar.gz
  30         https://github.com/bazelbuild/bazel-skylib/releases/download/0.8.0/bazel-skylib.0.8.0.tar.gz
  31         https://github.com/bazelbuild/bazel-toolchains/archive/92dd8a7a518a2fb7ba992d47c8b38299fe0be825.tar.gz -> bazel-toolchains-92dd8a7a518a2fb7ba992d47c8b38299fe0be825.tar.gz
  32         https://github.com/bazelbuild/rules_closure/archive/308b05b2419edb5c8ee0471b67a40403df940149.tar.gz -> bazelbuild-rules_closure-308b05b2419edb5c8ee0471b67a40403df940149.tar.gz
  33         https://github.com/bazelbuild/rules_docker/releases/download/v0.10.0/rules_docker-v0.10.0.tar.gz -> bazelbuild-rules_docker-v0.10.0.tar.gz
  34         https://github.com/bazelbuild/rules_swift/releases/download/0.11.1/rules_swift.0.11.1.tar.gz -> bazelbuild-rules_swift.0.11.1.tar.gz
  35         https://github.com/google/farmhash/archive/816a4ae622e964763ca0862d9dbd19324a1eaf45.tar.gz -> farmhash-816a4ae622e964763ca0862d9dbd19324a1eaf45.tar.gz
  36         https://github.com/google/gemmlowp/archive/12fed0cd7cfcd9e169bf1925bc3a7a58725fdcc3.zip -> gemmlowp-12fed0cd7cfcd9e169bf1925bc3a7a58725fdcc3.zip
  37         https://github.com/google/highwayhash/archive/fd3d9af80465e4383162e4a7c5e2f406e82dd968.tar.gz -> highwayhash-fd3d9af80465e4383162e4a7c5e2f406e82dd968.tar.gz
  38         https://github.com/mborgerding/kissfft/archive/36dbc057604f00aacfc0288ddad57e3b21cfc1b8.tar.gz -> kissfft-36dbc057604f00aacfc0288ddad57e3b21cfc1b8.tar.gz
  39         https://github.com/nlopezgi/bazel-toolchains/archive/94d31935a2c94fe7e7c7379a0f3393e181928ff7.tar.gz -> bazel-toolchains-94d31935a2c94fe7e7c7379a0f3393e181928ff7.tar.gz
  40         https://github.com/pybind/pybind11/archive/v2.3.0.tar.gz -> pybind11-v2.3.0.tar.gz
  41         https://github.com/llvm-mirror/llvm/archive/b7d166cebcf619a3691eed3f994384aab3d80fa6.tar.gz -> llvm-b7d166cebcf619a3691eed3f994384aab3d80fa6.tar.gz
  42         cuda? (
  43                 https://github.com/nvidia/nccl/archive/0ceaec9cee96ae7658aa45686853286651f36384.tar.gz -> nvidia-nccl-0ceaec9cee96ae7658aa45686853286651f36384.tar.gz
  44                 https://github.com/NVlabs/cub/archive/1.8.0.zip -> cub-1.8.0.zip
  45         )
  46         python? (
  47                 https://github.com/intel/ARM_NEON_2_x86_SSE/archive/1200fe90bb174a6224a525ee60148671a786a71f.tar.gz -> ARM_NEON_2_x86_SSE-1200fe90bb174a6224a525ee60148671a786a71f.tar.gz
  48                 https://storage.googleapis.com/mirror.tensorflow.org/docs.python.org/2.7/_sources/license.rst.txt -> tensorflow-1.15.0-python-license.rst.txt
  49                 https://pypi.python.org/packages/bc/cc/3cdb0a02e7e96f6c70bd971bc8a90b8463fda83e264fa9c5c1c98ceabd81/backports.weakref-1.0rc1.tar.gz
  50         )"
  51
  52 SRC_URI="https://github.com/${PN}/${PN}/archive/v${MY_PV}.tar.gz -> ${P}.tar.gz
  53                 ${bazel_external_uris}"
  54
  55 RDEPEND="
  56         app-arch/snappy
  57         dev-db/lmdb
  58         dev-db/sqlite
  59         dev-libs/double-conversion
  60         dev-libs/icu
  61         ~dev-libs/jsoncpp-1.9.1
  62         dev-libs/libpcre
  63         dev-libs/nsync
  64         dev-libs/openssl:0=
  65         >=dev-libs/protobuf-3.6.1:=
  66         >=dev-libs/re2-0.2018.04.01
  67         media-libs/giflib
  68         media-libs/libjpeg-turbo
  69         media-libs/libpng:0
  70         >=net-libs/grpc-1.22.0
  71         net-misc/curl
  72         sys-libs/zlib
  73         >=sys-apps/hwloc-2
  74         cuda? (
  75                 >=dev-util/nvidia-cuda-toolkit-9.1[profiler]
  76                 dev-libs/cudnn
  77         )
  78         mpi? ( virtual/mpi )
  79         python? (
  80                 ${PYTHON_DEPS}
  81                 >=dev-libs/flatbuffers-1.8.0
  82                 dev-python/absl-py[${PYTHON_USEDEP}]
  83                 >=dev-python/astor-0.7.1[${PYTHON_USEDEP}]
  84                 dev-python/gast[${PYTHON_USEDEP}]
  85                 >=dev-python/numpy-1.16[${PYTHON_USEDEP}]
  86                 dev-python/google-pasta[${PYTHON_USEDEP}]
  87                 dev-python/opt-einsum[${PYTHON_USEDEP}]
  88                 >=dev-python/protobuf-python-3.6.1[${PYTHON_USEDEP}]
  89                 dev-python/six[${PYTHON_USEDEP}]
  90                 dev-python/termcolor[${PYTHON_USEDEP}]
  91                 >=dev-python/grpcio-1.22.0[${PYTHON_USEDEP}]
  92                 >=dev-python/wrapt-1.11.1[${PYTHON_USEDEP}]
  93                 >=net-libs/google-cloud-cpp-0.10.0
  94                 >=sci-libs/keras-applications-1.0.8[${PYTHON_USEDEP}]
  95                 >=sci-libs/keras-preprocessing-1.0.5[${PYTHON_USEDEP}]
  96                 >=sci-visualization/tensorboard-2.0.0[${PYTHON_USEDEP}]
  97         )"
  98 DEPEND="${RDEPEND}
  99         dev-python/mock"
 100 PDEPEND="python? (
 101                 >=sci-libs/tensorflow-estimator-2.0.0[${PYTHON_USEDEP}]
 102         )"
 103 BDEPEND="
 104         app-arch/unzip
 105         >=dev-libs/protobuf-3.6.0
 106         dev-java/java-config
 107         dev-python/mock
 108         dev-lang/swig
 109         dev-python/cython
 110         || (
 111                 =dev-util/bazel-0.24*
 112                 =dev-util/bazel-0.27*
 113         )
 114         cuda? (
 115                 >=dev-util/nvidia-cuda-toolkit-9.1[profiler]
 116         )
 117         !python? ( dev-lang/python )
 118         python? (
 119                 >=dev-python/grpcio-tools-1.22.0
 120         )"
 121 REQUIRED_USE="python? ( ${PYTHON_REQUIRED_USE} )"
 122
 123 S="${WORKDIR}/${MY_P}"
 124
 125 PATCHES=(
 126         "${FILESDIR}/tensorflow-1.15.0_rc0-0001-WORKSPACE-add-rules-docker-http_archive-bazel-toolch.patch"
 127 )
 128 DOCS=( AUTHORS CONTRIBUTING.md ISSUE_TEMPLATE.md README.md RELEASE.md )
 129 CHECKREQS_MEMORY="5G"
 130 CHECKREQS_DISK_BUILD="5G"
 131
 132 get-cpu-flags() {
 133         local i f=()
 134         # Keep this list in sync with tensorflow/core/platform/cpu_feature_guard.cc.
 135         for i in sse sse2 sse3 sse4_1 sse4_2 avx avx2 fma4; do
 136                 use cpu_flags_x86_${i} && f+=( -m${i/_/.} )
 137         done
 138         use cpu_flags_x86_fma3 && f+=( -mfma )
 139         echo "${f[*]}"
 140 }
 141
 142 pkg_setup() {
 143         ewarn "TensorFlow 2.0 is a major release that contains some incompatibilities"
 144         ewarn "with TensorFlow 1.x. For more information about migrating to TF2.0 see:"
 145         ewarn "https://www.tensorflow.org/guide/migrate"
 146
 147         local num_pythons_enabled
 148         num_pythons_enabled=0
 149         count_impls(){
 150                 num_pythons_enabled=$((${num_pythons_enabled} + 1))
 151         }
 152         use python && python_foreach_impl count_impls
 153
 154         # 5G to build C/C++ libs, 5G per python impl
 155         CHECKREQS_DISK_BUILD="$((5 + 5 * ${num_pythons_enabled}))G"
 156         check-reqs_pkg_setup
 157 }
 158
 159 src_unpack() {
 160         # Only unpack the main distfile
 161         unpack "${P}.tar.gz"
 162         bazel_load_distfiles "${bazel_external_uris}"
 163 }
 164
 165 src_prepare() {
 166         export JAVA_HOME=$(java-config --jre-home) # so keepwork works
 167
 168         append-flags $(get-cpu-flags)
 169         bazel_setup_bazelrc
 170
 171         default
 172         use python && python_copy_sources
 173
 174         use cuda && cuda_add_sandbox
 175 }
 176
 177 src_configure() {
 178         export JAVA_HOME=$(java-config --jre-home) # so keepwork works
 179
 180         do_configure() {
 181                 export CC_OPT_FLAGS=" "
 182                 export TF_ENABLE_XLA=$(usex xla 1 0)
 183                 export TF_NEED_OPENCL_SYCL=0
 184                 export TF_NEED_OPENCL=0
 185                 export TF_NEED_COMPUTECPP=0
 186                 export TF_NEED_ROCM=0
 187                 export TF_NEED_MPI=$(usex mpi 1 0)
 188                 export TF_SET_ANDROID_WORKSPACE=0
 189
 190                 if use python; then
 191                         export PYTHON_BIN_PATH="${PYTHON}"
 192                         export PYTHON_LIB_PATH="$(python_get_sitedir)"
 193                 else
 194                         export PYTHON_BIN_PATH="$(which python)"
 195                         export PYTHON_LIB_PATH="$(python -c 'from distutils.sysconfig import *; print(get_python_lib())')"
 196                 fi
 197
 198                 export TF_NEED_CUDA=$(usex cuda 1 0)
 199                 export TF_DOWNLOAD_CLANG=0
 200                 export TF_CUDA_CLANG=0
 201                 export TF_NEED_TENSORRT=0
 202                 if use cuda; then
 203                         export TF_CUDA_PATHS="${EPREFIX}/opt/cuda"
 204                         export GCC_HOST_COMPILER_PATH="$(cuda_gccdir)/$(tc-getCC)"
 205                         export TF_CUDA_VERSION="$(cuda_toolkit_version)"
 206                         export TF_CUDNN_VERSION="$(cuda_cudnn_version)"
 207                         einfo "Setting CUDA version: $TF_CUDA_VERSION"
 208                         einfo "Setting CUDNN version: $TF_CUDNN_VERSION"
 209
 210                         if [[ -z "$TF_CUDA_COMPUTE_CAPABILITIES" ]]; then
 211                                 ewarn "WARNING: Tensorflow is being built with its default CUDA compute capabilities: 3.5 and 7.0."
 212                                 ewarn "These may not be optimal for your GPU."
 213                                 ewarn ""
 214                                 ewarn "To configure Tensorflow with the CUDA compute capability that is optimal for your GPU,"
 215                                 ewarn "set TF_CUDA_COMPUTE_CAPABILITIES in your make.conf, and re-emerge tensorflow."
 216                                 ewarn "For example, to use CUDA capability 7.5 & 3.5, add: TF_CUDA_COMPUTE_CAPABILITIES=7.5,3.5"
 217                                 ewarn ""
 218                                 ewarn "You can look up your GPU's CUDA compute capability at https://developer.nvidia.com/cuda-gpus"
 219                                 ewarn "or by running /opt/cuda/extras/demo_suite/deviceQuery | grep 'CUDA Capability'"
 220                         fi
 221                 fi
 222
 223                 local SYSLIBS=(
 224                         absl_py
 225                         astor_archive
 226                         boringssl
 227                         com_github_googleapis_googleapis
 228                         com_github_googlecloudplatform_google_cloud_cpp
 229                         com_google_protobuf
 230                         com_googlesource_code_re2
 231                         curl
 232                         cython
 233                         double_conversion
 234                         enum34_archive
 235                         flatbuffers
 236                         functools32_archive
 237                         gast_archive
 238                         gif_archive
 239                         grpc
 240                         hwloc
 241                         icu
 242                         jpeg
 243                         jsoncpp_git
 244                         keras_applications_archive
 245                         lmdb
 246                         nasm
 247                         nsync
 248                         opt_einsum_archive
 249                         org_sqlite
 250                         pasta
 251                         pcre
 252                         png_archive
 253                         six_archive
 254                         snappy
 255                         swig
 256                         termcolor_archive
 257                         wrapt
 258                         zlib_archive
 259                 )
 260
 261                 export TF_SYSTEM_LIBS="${SYSLIBS[@]}"
 262                 export TF_IGNORE_MAX_BAZEL_VERSION=1
 263
 264                 # This is not autoconf
 265                 ./configure || die
 266
 267                 echo 'build --config=noaws --config=nohdfs --config=noignite --config=nokafka' >> .bazelrc || die
 268                 echo 'build --define tensorflow_mkldnn_contraction_kernel=0' >> .bazelrc || die
 269                 echo 'build --incompatible_no_support_tools_in_action_inputs=false' >> .bazelrc || die
 270         }
 271         if use python; then
 272                 python_foreach_impl run_in_build_dir do_configure
 273         else
 274                 do_configure
 275         fi
 276 }
 277
 278 src_compile() {
 279         export JAVA_HOME=$(java-config --jre-home) # so keepwork works
 280
 281         if use python; then
 282                 python_setup
 283                 BUILD_DIR="${S}-${EPYTHON/./_}"
 284                 cd "${BUILD_DIR}"
 285         fi
 286
 287         # fail early if any deps are missing
 288         ebazel build --nobuild \
 289                 //tensorflow:libtensorflow_framework.so \
 290                 //tensorflow:libtensorflow.so \
 291                 //tensorflow:libtensorflow_cc.so \
 292                 $(usex python '//tensorflow/tools/pip_package:build_pip_package' '')
 293
 294         ebazel build \
 295                 //tensorflow:libtensorflow_framework.so \
 296                 //tensorflow:libtensorflow.so
 297         ebazel build //tensorflow:libtensorflow_cc.so
 298
 299         do_compile() {
 300                 ebazel build //tensorflow/tools/pip_package:build_pip_package
 301         }
 302         BUILD_DIR="${S}"
 303         cd "${BUILD_DIR}"
 304         use python && python_foreach_impl run_in_build_dir do_compile
 305         ebazel shutdown
 306 }
 307
 308 src_install() {
 309         local i j
 310         export JAVA_HOME=$(java-config --jre-home) # so keepwork works
 311
 312         do_install() {
 313                 einfo "Installing ${EPYTHON} files"
 314                 local srcdir="${T}/src-${MULTIBUILD_VARIANT}"
 315                 mkdir -p "${srcdir}" || die
 316                 bazel-bin/tensorflow/tools/pip_package/build_pip_package --src "${srcdir}" || die
 317                 cd "${srcdir}" || die
 318                 esetup.py install
 319
 320                 # libtensorflow_framework.so is in /usr/lib already
 321                 rm -f "${D}/$(python_get_sitedir)"/${PN}/lib${PN}_framework.so* || die
 322                 rm -f "${D}/$(python_get_sitedir)"/${PN}_core/lib${PN}_framework.so* || die
 323                 python_optimize
 324         }
 325
 326         if use python; then
 327                 python_foreach_impl run_in_build_dir do_install
 328
 329                 # Symlink to python-exec scripts
 330                 for i in "${ED}"/usr/lib/python-exec/*/*; do
 331                         n="${i##*/}"
 332                         [[ -e "${ED}/usr/bin/${n}" ]] || dosym ../lib/python-exec/python-exec2 "/usr/bin/${n}"
 333                 done
 334
 335                 python_setup
 336                 local BUILD_DIR="${S}-${EPYTHON/./_}"
 337                 cd "${BUILD_DIR}" || die
 338         fi
 339
 340         einfo "Installing headers"
 341         ebazel build //tensorflow:install_headers
 342         ebazel shutdown
 343         insinto /usr/include/${PN}/
 344         doins -r bazel-genfiles/tensorflow/include/*
 345
 346         einfo "Installing libs"
 347         # Generate pkg-config file
 348         ${PN}/c/generate-pc.sh --prefix="${EPREFIX}"/usr --libdir=$(get_libdir) --version=${MY_PV} || die
 349         insinto /usr/$(get_libdir)/pkgconfig
 350         doins ${PN}.pc ${PN}_cc.pc
 351
 352         for l in libtensorflow{,_framework,_cc}.so; do
 353                 dolib.so bazel-bin/tensorflow/${l}
 354                 dolib.so bazel-bin/tensorflow/${l}.$(ver_cut 1)
 355                 dolib.so bazel-bin/tensorflow/${l}.$(ver_cut 1-3)
 356         done
 357
 358         einstalldocs
 359 }