Add LTO build option (#505)

* Add LTO build option

An old PR which I used as a base:
#150

Co-Authored-by: Dan Gohman <dev@sunfishcode.online>

* Exclude atexit.c from LTO

This fixes a failure in wasi-sdk "make check".
("undefined symbol: __cxa_atexit" for ctors_dtors.c test)

* avoid specifying multiple lto flags for LIBC_NONLTO_OBJS

---------

Co-authored-by: Dan Gohman <dev@sunfishcode.online>

Author: yamt

Makefile

@@ -25,6 +25,16 @@ BUILD_LIBC_TOP_HALF ?= yes
 BUILD_LIBSETJMP ?= yes
 # The directory where we will store intermediate artifacts.
 OBJDIR ?= build/$(TARGET_TRIPLE)
+
+# LTO; no, full, or thin
+# Note: thin LTO here is just for experimentation. It has known issues:
+# - https://github.com/llvm/llvm-project/issues/91700
+# - https://github.com/llvm/llvm-project/issues/91711
+LTO ?= no
+ifneq ($(LTO),no)
+CLANG_VERSION ?= $(shell ${CC} -dumpversion)
+override OBJDIR := $(OBJDIR)/llvm-lto/$(CLANG_VERSION)
+endif
 # The directory where we store files and tools for generating WASIp2 bindings
 BINDING_WORK_DIR ?= build/bindings
 # URL from which to retrieve the WIT files used to generate the WASIp2 bindings
@@ -251,6 +261,11 @@ LIBC_TOP_HALF_MUSL_SOURCES = \
                  $(wildcard $(LIBC_TOP_HALF_MUSL_SRC_DIR)/complex/*.c)) \
     $(wildcard $(LIBC_TOP_HALF_MUSL_SRC_DIR)/crypt/*.c)
 
+LIBC_NONLTO_SOURCES = \
+    $(addprefix $(LIBC_TOP_HALF_MUSL_SRC_DIR)/, \
+        exit/atexit.c \
+    )
+
 ifeq ($(WASI_SNAPSHOT), p2)
 LIBC_TOP_HALF_MUSL_SOURCES += \
     $(addprefix $(LIBC_TOP_HALF_MUSL_SRC_DIR)/, \
@@ -396,6 +411,18 @@ ASMFLAGS += -matomics
 CFLAGS += -I$(LIBC_BOTTOM_HALF_CLOUDLIBC_SRC)
 endif
 
+ifneq ($(LTO),no)
+ifeq ($(LTO),full)
+CFLAGS += -flto=full
+else
+ifeq ($(LTO),thin)
+CFLAGS += -flto=thin
+else
+$(error unknown LTO value: $(LTO))
+endif
+endif
+endif
+
 ifeq ($(WASI_SNAPSHOT), p2)
 CFLAGS += -D__wasilibc_use_wasip2
 endif
@@ -452,10 +479,14 @@ LIBWASI_EMULATED_SIGNAL_MUSL_OBJS = $(call objs,$(LIBWASI_EMULATED_SIGNAL_MUSL_S
 LIBDL_OBJS = $(call objs,$(LIBDL_SOURCES))
 LIBSETJMP_OBJS = $(call objs,$(LIBSETJMP_SOURCES))
 LIBC_BOTTOM_HALF_CRT_OBJS = $(call objs,$(LIBC_BOTTOM_HALF_CRT_SOURCES))
+LIBC_NONLTO_OBJS = $(call objs,$(LIBC_NONLTO_SOURCES))
 
 # These variables describe the locations of various files and
 # directories in the generated sysroot tree.
 SYSROOT_LIB := $(SYSROOT)/lib/$(TARGET_TRIPLE)
+ifneq ($(LTO),no)
+override SYSROOT_LIB := $(SYSROOT_LIB)/llvm-lto/$(CLANG_VERSION)
+endif
 SYSROOT_INC = $(SYSROOT)/include/$(TARGET_TRIPLE)
 SYSROOT_SHARE = $(SYSROOT)/share/$(TARGET_TRIPLE)
 
@@ -627,6 +658,8 @@ $(SYSROOT_LIB)/libsetjmp.a: $(LIBSETJMP_OBJS)
 
 $(PIC_OBJS): CFLAGS += -fPIC -fvisibility=default
 
+$(LIBC_NONLTO_OBJS): CFLAGS := $(filter-out -flto% -fno-lto, $(CFLAGS)) -fno-lto
+
 $(MUSL_PRINTSCAN_OBJS): CFLAGS += \
 	    -D__wasilibc_printscan_no_long_double \
 	    -D__wasilibc_printscan_full_support_option="\"add -lc-printscan-long-double to the link command\""
@@ -794,12 +827,14 @@ finish: startup_files libc dummy_libs
 	# The build succeeded! The generated sysroot is in $(SYSROOT).
 	#
 
+ifeq ($(LTO),no)
 # The check for defined and undefined symbols expects there to be a heap
 # alloctor (providing malloc, calloc, free, etc). Skip this step if the build
 # is done without a malloc implementation.
 ifneq ($(MALLOC_IMPL),none)
 finish: check-symbols
 endif
+endif
 
 DEFINED_SYMBOLS = $(SYSROOT_SHARE)/defined-symbols.txt
 UNDEFINED_SYMBOLS = $(SYSROOT_SHARE)/undefined-symbols.txt