Roll up msys2/clang/windows build fixes (#59003)

This rolls up everything I had to change to get a successful source
build of Julia under msys2. It's a misc collection of msys2, clang and
other fixes. With this, I can use the following Make.user:

```
USE_SYSTEM_CSL=1
USE_BINARYBUILDER_LLVM=0
CC=clang
CXX=clang++
FC=gfortran
```

The default USE_SYSTEM_CSL is broken due to #56840 With
USE_SYSTEM_CSL=1, LLVM is broken due to #57021 Clang is required because
gcc can't do an LLVM source build due to known export symbol size limits
(ref JuliaPackaging/Yggdrasil#11652).

That said, if we address the ABI issues in #56840, the default Make.user
should build again (with BB-provided LLVM).

Author: Keno

Make.inc

@@ -1580,7 +1580,9 @@ ifeq ($(OS), WINNT)
 HAVE_SSP := 1
 OSLIBS += -Wl,--export-all-symbols -Wl,--version-script=$(BUILDROOT)/src/julia.expmap \
 	$(NO_WHOLE_ARCHIVE) -lpsapi -lkernel32 -lws2_32 -liphlpapi -lwinmm -ldbghelp -luserenv -lsecur32 -latomic -lole32
-JLDFLAGS += -Wl,--stack,8388608 --disable-auto-import --disable-runtime-pseudo-reloc
+# N.B.: Unlike in the sysimage, we cannot -Wl,--disable-auto-import -Wl,--disable-runtime-pseudo-reloc here, because libstdc++/LLVM are not fully correct under
+# enforced visibility at this point.
+JLDFLAGS += -Wl,--stack,8388608
 ifeq ($(ARCH),i686)
 JLDFLAGS += -Wl,--large-address-aware
 endif

base/linking.jl

@@ -140,7 +140,15 @@ function link_image_cmd(path, out)
     LIBS = isdebugbuild() ? ("-ljulia-debug", "-ljulia-internal-debug") :
                         ("-ljulia", "-ljulia-internal")
     @static if Sys.iswindows()
-        LIBS = (LIBS..., "-lopenlibm", "-lssp", "-lgcc_s", "-lgcc", "-lmsvcrt")
+        LIBS = (LIBS..., "-lopenlibm", "-lgcc_s", "-lgcc", "-lmsvcrt")
+        if isdebugbuild()
+            LIBS = (LIBS..., "-lssp")
+            if isfile(joinpath(private_libdir(), "libmingwex.a"))
+                # In MinGW 11, the ssp implementation was moved from libssp to
+                # libmingwex with ssp only being a stub. See #59020.
+                LIBS = (LIBS..., "-lmingwex", "-lkernel32")
+            end
+        end
     end
 
     V = verbose_linking() ? "--verbose" : ""

cli/loader.h

@@ -36,9 +36,9 @@
 // Borrow definition from `support/dtypes.h`
 #ifdef _OS_WINDOWS_
 # ifdef JL_LIBRARY_EXPORTS
-#  define JL_DLLEXPORT __declspec(dllexport)
+#  define JL_DLLEXPORT __declspec(dllexport) __attribute__ ((visibility("default")))
 # endif
-#  define JL_DLLIMPORT __declspec(dllimport)
+#  define JL_DLLIMPORT __declspec(dllimport) __attribute__ ((visibility("default")))
 #define JL_HIDDEN
 #else
 # define JL_DLLIMPORT __attribute__ ((visibility("default")))

cli/loader_win_utils.c

@@ -12,6 +12,10 @@ static FILE _stderr = { INVALID_HANDLE_VALUE };
 FILE *stdout = &_stdout;
 FILE *stderr = &_stderr;
 
+void JL_HIDDEN free(void* mem) {
+    HeapFree(GetProcessHeap(), 0, mem);
+}
+
 int JL_HIDDEN fwrite(const char *str, size_t nchars, FILE *out) {
     DWORD written;
     if (out->isconsole) {
@@ -44,10 +48,6 @@ void JL_HIDDEN *realloc(void * mem, const size_t size) {
     return HeapReAlloc(GetProcessHeap(), HEAP_GENERATE_EXCEPTIONS, mem, size);
 }
 
-void JL_HIDDEN free(void* mem) {
-    HeapFree(GetProcessHeap(), 0, mem);
-}
-
 LPWSTR *CommandLineToArgv(LPWSTR lpCmdLine, int *pNumArgs) {
     LPWSTR out = lpCmdLine;
     LPWSTR cmd = out;

deps/Makefile

@@ -42,9 +42,10 @@ ifeq ($(USE_SYSTEM_LIBBLASTRAMPOLINE), 0)
 DEP_LIBS += blastrampoline
 endif
 
-ifeq ($(USE_SYSTEM_CSL), 0)
+# We need to run this whether or not USE_SYSTEM_CSL is set.
+# If it is, this target copies the system CSLs into the location our
+# build system expects.
 DEP_LIBS += csl
-endif
 
 ifeq ($(SANITIZE), 1)
 DEP_LIBS += sanitizers

deps/csl.mk

@@ -2,10 +2,10 @@
 STD_LIB_PATH := $(shell LANG=C $(FC) -print-search-dirs 2>/dev/null | grep '^programs: =' | sed -e "s/^programs: =//")
 STD_LIB_PATH += $(PATHSEP)$(shell LANG=C $(FC) -print-search-dirs 2>/dev/null | grep '^libraries: =' | sed -e "s/^libraries: =//")
 ifeq ($(BUILD_OS),WINNT)  # the mingw compiler lies about it search directory paths
-STD_LIB_PATH := $(shell echo '$(STD_LIB_PATH)' | sed -e "s!/lib/!/bin/!g")
+STD_LIB_PATH += $(shell echo '$(STD_LIB_PATH)' | sed -e "s!/lib/!/bin/!g")
 endif
 
-# Given a colon-separated list of paths in $(2), find the location of the library given in $(1)
+# Given a $(PATHSEP)-separated list of paths in $(2), find the location of the library given in $(1)
 define pathsearch
 $(firstword $(wildcard $(addsuffix /$(1),$(subst $(PATHSEP), ,$(2)))))
 endef
@@ -54,6 +54,13 @@ $$(build_shlibdir)/$(1): | $$(build_shlibdir)
 	[ -n "$$$${SRC_LIB}" ] && cp "$$$${SRC_LIB}" '$$(build_shlibdir)'
 endef
 
+define copy_csl_static_lib
+install-csl: | $$(build_private_libdir) $$(build_private_libdir)/$(1)
+$$(build_private_libdir)/$(1): | $$(build_private_libdir)
+	-@SRC_LIB='$$(call pathsearch,$(1),$$(STD_LIB_PATH))'; \
+	[ -n "$$$${SRC_LIB}" ] && cp "$$$${SRC_LIB}" '$$(build_private_libdir)'
+endef
+
 # libgfortran has multiple names; we're just going to copy any version we can find
 # Since we're only looking in the location given by `$(FC)` this should only succeed for one.
 $(eval $(call copy_csl,$(call versioned_libname,libgfortran,3)))
@@ -63,11 +70,20 @@ $(eval $(call copy_csl,$(call versioned_libname,libgfortran,5)))
 # These are all libraries that we should always have
 $(eval $(call copy_csl,$(call versioned_libname,libquadmath,0)))
 $(eval $(call copy_csl,$(call versioned_libname,libstdc++,6)))
-$(eval $(call copy_csl,$(call versioned_libname,libssp,0)))
 $(eval $(call copy_csl,$(call versioned_libname,libatomic,1)))
 $(eval $(call copy_csl,$(call versioned_libname,libgomp,1)))
 
+# Configurable either a static or dynamic library depending on the system
+$(eval $(call copy_csl,$(call versioned_libname,libssp,0)))
+$(eval $(call copy_csl_static_lib,libssp.a))
+
 ifeq ($(OS),WINNT)
+# On windows we need the static gcc runtime libraries for linking pkgimages
+$(eval $(call copy_csl_static_lib,libgcc.a))
+$(eval $(call copy_csl_static_lib,libgcc_s.a))
+$(eval $(call copy_csl_static_lib,libmsvcrt.a))
+$(eval $(call copy_csl_static_lib,libmingwex.a))
+$(eval $(call copy_csl_static_lib,libkernel32.a))
 # Windows has special gcc_s names
 ifeq ($(ARCH),i686)
 $(eval $(call copy_csl,$(call versioned_libname,libgcc_s_sjlj,1)))

deps/llvm.mk

@@ -292,6 +292,12 @@ $(LLVM_BUILDDIR_withtype)/build-configured: $(SRCCACHE)/$(LLVM_SRC_DIR)/source-e
 	echo 1 > $@
 
 $(LLVM_BUILDDIR_withtype)/build-compiled: $(LLVM_BUILDDIR_withtype)/build-configured
+ifeq ($(OS),WINNT)
+ifeq ($(USEGCC),1)
+	echo "LLVM source build is currently known to fail using GCC due to exceeded export table limits. Try clang."
+	exit 1
+endif
+endif
 	cd $(LLVM_BUILDDIR_withtype) && \
 		$(if $(filter $(CMAKE_GENERATOR),make), \
 		  $(MAKE), \
@@ -306,8 +312,8 @@ endif
 	echo 1 > $@
 
 LLVM_INSTALL = \
-	cd $1 && mkdir -p $2$$(build_depsbindir) && \
-	cp -r $$(SRCCACHE)/$$(LLVM_SRC_DIR)/llvm/utils/lit $2$$(build_depsbindir)/ && \
+	cd $1 && mkdir -p $2$$(build_depsbindir)/lit && \
+	cp -r $$(SRCCACHE)/$$(LLVM_SRC_DIR)/llvm/utils/lit/{*.py,*.toml,lit/} $2$$(build_depsbindir)/lit/ && \
 	$$(CMAKE) -DCMAKE_INSTALL_PREFIX="$2$$(build_prefix)" -P cmake_install.cmake
 ifeq ($(OS), WINNT)
 LLVM_INSTALL += && cp $2$$(build_shlibdir)/$(LLVM_SHARED_LIB_NAME).dll $2$$(build_depsbindir)

deps/p7zip.mk

@@ -19,7 +19,7 @@ checksum-p7zip: $(SRCCACHE)/p7zip-$(P7ZIP_VER).tar.gz
 
 $(BUILDDIR)/p7zip-$(P7ZIP_VER)/build-configured: $(BUILDDIR)/p7zip-$(P7ZIP_VER)/source-extracted
 $(BUILDDIR)/p7zip-$(P7ZIP_VER)/build-compiled: $(BUILDDIR)/p7zip-$(P7ZIP_VER)/build-configured
-	$(MAKE) -C $(dir $<) $(MAKE_COMMON) $(P7ZIP_BUILD_OPTS) 7za$(EXE)
+	$(MAKE) -C $(dir $<) $(MAKE_COMMON) $(P7ZIP_BUILD_OPTS) 7za
 	echo 1 > $@
 
 define P7ZIP_INSTALL

deps/tools/common.mk

@@ -5,8 +5,15 @@
 # apparently not on FreeBSD). Ref PR #22352
 
 CONFIGURE_COMMON = --prefix=$(abspath $(build_prefix)) --build=$(BUILD_MACHINE) --libdir=$(abspath $(build_libdir)) --bindir=$(abspath $(build_depsbindir)) $(CUSTOM_LD_LIBRARY_PATH)
+
+CMAKE_COMMON := -DCMAKE_INSTALL_PREFIX:PATH=$(build_prefix) -DCMAKE_PREFIX_PATH=$(build_prefix)
+CMAKE_COMMON += -DLIB_INSTALL_DIR=$(build_shlibdir)
+
 ifneq ($(XC_HOST),)
 CONFIGURE_COMMON += --host=$(XC_HOST)
+else
+# Defeat bad automatic cross compile detection (e.g. clang on mingw)
+# CMAKE_COMMON += -DCMAKE_CROSSCOMPILING=0
 endif
 ifeq ($(OS),WINNT)
 CONFIGURE_COMMON += LDFLAGS="$(LDFLAGS) -Wl,--stack,8388608"
@@ -15,8 +22,6 @@ CONFIGURE_COMMON += LDFLAGS="$(LDFLAGS) $(RPATH_ESCAPED_ORIGIN) $(SANITIZE_LDFLA
 endif
 CONFIGURE_COMMON += F77="$(FC)" CC="$(CC) $(SANITIZE_OPTS)" CXX="$(CXX) $(SANITIZE_OPTS)" LD="$(LD)"
 
-CMAKE_COMMON := -DCMAKE_INSTALL_PREFIX:PATH=$(build_prefix) -DCMAKE_PREFIX_PATH=$(build_prefix)
-CMAKE_COMMON += -DLIB_INSTALL_DIR=$(build_shlibdir)
 ifneq ($(OS),WINNT)
 CMAKE_COMMON += -DCMAKE_INSTALL_LIBDIR=$(build_libdir)
 endif
@@ -61,7 +66,12 @@ endif
 CMAKE_COMMON += -DCMAKE_LINKER="$$(which $(LD))" -DCMAKE_AR="$$(which $(AR))" -DCMAKE_RANLIB="$$(which $(RANLIB))"
 
 ifeq ($(OS),WINNT)
+ifeq ($(BUILD_OS),WINNT)
+# Don't make CMake think we're cross compiling, but do make sure it knows we're Windows
+CMAKE_COMMON += -DCMAKE_HOST_SYSTEM_NAME=Windows
+else
 CMAKE_COMMON += -DCMAKE_SYSTEM_NAME=Windows
+endif
 CMAKE_COMMON += -DCMAKE_RC_COMPILER="$$(which $(CROSS_COMPILE)windres)"
 endif
 

deps/zstd.mk

@@ -3,6 +3,8 @@ ifneq ($(USE_BINARYBUILDER_ZSTD), 1)
 ZSTD_GIT_URL := https://github.com/facebook/zstd.git
 ZSTD_TAR_URL = https://api.github.com/repos/facebook/zstd/tarball/$1
 $(eval $(call git-external,zstd,ZSTD,,,$(BUILDDIR)))
+# See note in llvm.mk for source tarballs with symlinks to non-existent targets
+$(BUILDDIR)/$(ZSTD_SRC_DIR)/source-extracted: export MSYS=winsymlinks:native
 
 ZSTD_BUILD_OPTS := MOREFLAGS="-DZSTD_MULTITHREAD $(fPIC)" bindir=$(build_private_libexecdir)