Add named-lto option to meson build to named with LTO

Enabling LTO yields substantial performance gains on both authoritative
and resolver benchmarks.
But since LTO defers many optimization passes to link time, enabling LTO
across the board would cause an increase in compilation time, as passes
that would be run only once would need to be run for each executable.

As a compromise, this commit adds a named-lto build option, that
compiles the individual object files with the -ffat-lto-object option
and then enables LTO only for the named executable. Object files are
reused between lib*.so and the named executable.

.gitlab-ci.yml

@@ -1214,7 +1214,7 @@ scan-build:
   variables:
     CC: "${CLANG}"
     CFLAGS: "${CFLAGS_COMMON}"
-    EXTRA_CONFIGURE: "-Didn=enabled"
+    EXTRA_CONFIGURE: "-Didn=enabled --native-file ci/clang-trixie.ini"
   before_script:
     - *list_installed_package_versions
   script:
@@ -1427,7 +1427,7 @@ clang:asan:
   variables:
     CC: ${CLANG}
     CFLAGS: "${CFLAGS_COMMON}"
-    EXTRA_CONFIGURE: "-Db_sanitize=address,undefined -Db_lundef=false -Didn=enabled -Djemalloc=disabled -Dtracing=disabled"
+    EXTRA_CONFIGURE: "-Db_sanitize=address,undefined -Db_lundef=false -Didn=enabled -Djemalloc=disabled -Dtracing=disabled --native-file ci/clang-trixie.ini"
   <<: *base_image
   <<: *build_job
 
@@ -1487,7 +1487,8 @@ clang:tsan:
     CC: "${CLANG}"
     CFLAGS: "${CFLAGS_COMMON}"
     LDFLAGS: "-Wl,--disable-new-dtags"
-    EXTRA_CONFIGURE: "${TSAN_CONFIGURE_FLAGS_COMMON} -Db_lundef=false"
+    EXTRA_CONFIGURE: "${TSAN_CONFIGURE_FLAGS_COMMON} -Db_lundef=false -Dnamed-lto=off --native-file ci/clang-trixie.ini"
+  <<: *build_job
 
 system:clang:tsan:
   variables:
@@ -1542,6 +1543,7 @@ clang:trixie:amd64:
   variables:
     CC: ${CLANG}
     CFLAGS: "${CFLAGS_COMMON}"
+    EXTRA_CONFIGURE: "--native-file ci/clang-trixie.ini"
     RUN_MESON_INSTALL: 1
   <<: *debian_trixie_amd64_image
   <<: *build_job
@@ -1801,7 +1803,7 @@ respdiff:tsan:
     CC: "${CLANG}"
     CFLAGS: "${CFLAGS_COMMON}"
     LDFLAGS: "-Wl,--disable-new-dtags"
-    EXTRA_CONFIGURE: "${TSAN_CONFIGURE_FLAGS_COMMON} -Db_lundef=false"
+    EXTRA_CONFIGURE: "${TSAN_CONFIGURE_FLAGS_COMMON} -Dnamed-lto=off -Db_lundef=false"
     MAX_DISAGREEMENTS_PERCENTAGE: "0.3"
     TSAN_OPTIONS: "${TSAN_OPTIONS_DEBIAN}"
   script:
@@ -1928,9 +1930,13 @@ reproducible-build:
   before_script:
     - *list_installed_package_versions
   script:
+    # dnstap produces an intermediate .a file, and meson considers all .a
+    # files to be final results independently of whether they are installed or
+    # not. But the content of the .a file might be unstable under LTO due to
+    # -ffat-lto-objects. Hence we disable dnstap for reproducibility tests.
     - meson reprotest
-      --intermediaries
       --
+      -Ddnstap=disabled
       -Ddoc=disabled
       -Doptimization=1
   artifacts:

ci/clang-trixie.ini

@@ -0,0 +1,20 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+# LTO builds with clang
+
+[binaries]
+ar = 'llvm-ar-20'
+c = 'clang-20'
+c_ld = 'lld'
+
+[project options]
+named-lto = 'thin'

doc/arm/build.inc.rst

@@ -87,6 +87,22 @@ To improve performance, use of the ``jemalloc`` library
 (https://jemalloc.net/) is strongly recommended. Version 4.0.0 or newer is
 required when in use.
 
+To further improve performance, compilation with link-time optimization is
+recommended. This is enabled by default via the ``-Dnamed-lto`` option
+(default: ``thin``). Link-time optimization can be disabled if needed by
+using ``-Dnamed-lto=off``. The optimization level can be controlled with
+``-Dnamed-lto=thin`` or ``-Dnamed-lto=full``.
+
+Link-time optimization requires close coordination between the compiler and
+the linker. Due to ``clang`` limitations, compiling ``named`` with ``clang``
+and link-time optimization is only supported with the ``lld`` linker.
+
+Meson provides an alternative way to enable link-time optimization through
+the ``-Db_lto=true`` flag. However, this option is incompatible with
+BIND's ``-Dnamed-lto`` option. Meson's ``-Db_lto`` may also be incompatible
+with certain BIND build options and can result in lower performance and
+higher compile times compared to ``-Dnamed-lto``.
+
 To support :rfc:`DNS over HTTPS (DoH) <8484>`, the server must be linked
 with ``libnghttp2`` (https://nghttp2.org/). If the library is
 unavailable, ``-Ddoh=disabled`` can be used to disable DoH support.

meson.build

@@ -45,6 +45,7 @@ developer_mode = get_option('developer').enabled()
 c_std = get_option('c_std')
 optimization = get_option('optimization')
 sanitizer = get_option('b_sanitize')
+meson_lto = get_option('b_lto')
 
 trace_logging = get_option('trace-logging')
 rcu_flavor = get_option('rcu-flavor')
@@ -64,6 +65,7 @@ leak_opt = get_option('leak-detection')
 line_opt = get_option('line')
 lmdb_opt = get_option('lmdb')
 locktype_opt = get_option('locktype')
+named_lto_opt = get_option('named-lto')
 stats_json_opt = get_option('stats-json')
 stats_xml_opt = get_option('stats-xml')
 tracing_opt = get_option('tracing')
@@ -898,6 +900,61 @@ assert(
     'tracing is requested but dtrace is not found',
 )
 
+# LTO
+
+static_lto_c_args = []
+static_lto_link_args = []
+
+if named_lto_opt == 'full'
+    static_lto_c_args = ['-ffat-lto-objects', '-flto']
+    static_lto_link_args = ['-flto']
+elif named_lto_opt == 'thin' and cc.get_id() == 'clang' and cc.get_linker_id() == 'ld.lld'
+    # Per LLVM docs [1], -ffat-lto-objects is supported only with lld and gold,
+    # and gold is deprecated/unmantained.
+    # [1]: https://llvm.org/docs/FatLTO.html
+
+    static_lto_c_args = ['-ffat-lto-objects', '-flto=thin']
+    static_lto_link_args = ['-flto=thin']
+elif named_lto_opt == 'thin' and cc.get_id() == 'gcc'
+    static_lto_c_args = ['-ffat-lto-objects', '-flto=auto']
+    static_lto_link_args = ['-flto=auto']
+elif named_lto_opt == 'thin'
+    error('LTO requires clang with ld.lld, or gcc with any linker')
+endif
+
+add_project_arguments(static_lto_c_args, language: 'c')
+if named_lto_opt != 'off' and cc.get_id() == 'clang' and sanitizer.contains('address')
+    # Needed to suppress the
+    #    warning: Redundant instrumentation detected, with module flag:
+    #    nosanitize_address [-Werror,-Wbackend-plugin]
+    # warning in address sanitizer. This warning indicates that the object file
+    # has been processed already by address sanitizer instrumentation pass.
+    # From looking at the pass code, when address sanitizer detects that
+    # an object file has already been instrumented, it just skips it.
+    # Therefore it should be safe to suppress the warning.
+
+    add_project_arguments('-Wno-backend-plugin', language: 'c')
+endif
+
+if meson_lto and named_lto_opt != 'off'
+    # Meson's builtin LTO settings do not set -ffat-lto-objects, which can cause
+    # build issues.
+    # Since we don't want two, possibly conflicting, sets of LTO flags, we
+    # error out if both are set.
+
+    error(
+        '''
+            Meson builtin -Db_lto and BIND's -Dnamed-lto options are incompatible.
+            Either disable named-lto with -Dnamed-lto=off, or avoid setting
+            -Db_lto.
+
+            Note that using -Db_lto is not a recommended configuration, might
+            yield reduced performance compared to -Dnamed-lto and conflict
+            with other build flags.
+        ''',
+    )
+endif
+
 ### Finalize configuration
 configure_file(output: 'config.h', configuration: config)
 add_project_arguments('-include', meson.project_build_root() / 'config.h', language: 'c')
@@ -1158,8 +1215,6 @@ libisccfg_dep = declare_dependency(
     include_directories: isccfg_inc,
 )
 
-named_srcconf = named_srcset.apply(config, strict: false)
-
 executable(
     'arpaname',
     arpaname_src,
@@ -1459,22 +1514,58 @@ executable(
 
 )
 
-executable(
-    'named',
-    named_srcconf.sources(),
-    export_dynamic: true,
-    implicit_include_directories: true,
-    include_directories: named_inc_p,
-    install: true,
-    install_dir: sbindir,
-    sources: bind_keys,
-    dependencies: [
+named_c_args = []
+named_link_args = []
+named_deps = []
+
+if named_lto_opt == 'off'
+    named_deps = [
         libdns_dep,
         libisc_dep,
         libisccc_dep,
         libisccfg_dep,
         libns_dep,
+    ]
+    named_inc = named_inc_p
 
+    named_objects = []
+else
+    named_deps = [
+        dns_srcconf.dependencies(),
+        isc_srcconf.dependencies(),
+        isccc_srcconf.dependencies(),
+        isccfg_srcconf.dependencies(),
+        ns_srcconf.dependencies(),
+    ]
+    named_inc = [isc_inc, dns_inc, isccc_inc, isccfg_inc, ns_inc, named_inc_p]
+
+    named_srcset.add(dns_gen_headers)
+
+    named_objects = [
+        libisc.extract_all_objects(recursive: true),
+        libdns.extract_all_objects(recursive: true),
+        libns.extract_all_objects(recursive: true),
+        libisccc.extract_all_objects(recursive: true),
+        libisccfg.extract_all_objects(recursive: true),
+    ]
+endif
+
+named_srcconf = named_srcset.apply(config, strict: false)
+
+executable(
+    'named',
+    named_srcconf.sources(),
+    objects: named_objects,
+    c_args: static_lto_c_args,
+    link_args: static_lto_link_args,
+    export_dynamic: true,
+    implicit_include_directories: true,
+    include_directories: named_inc,
+    install: true,
+    install_dir: sbindir,
+    sources: bind_keys,
+    dependencies: named_deps
+    + [
         openssl_dep,
 
         cap_dep,

meson_options.txt

@@ -188,3 +188,11 @@ option(
     value: 'disabled',
     description: 'enable the memory leak detection in external libraries (libxml2, libuv, OpenSSL)',
 )
+
+option(
+    'named-lto',
+    type: 'combo',
+    choices: ['off', 'thin', 'full'],
+    value: 'thin',
+    description: 'Enable Link Time Optimization for named.',
+)