Add tests to ensure reproducibility on more platforms (#1234)

* Add tests to ensure reproducibility on more platforms

* Skip reproducibility test with GCC 5 for i686 Windows

Author: giordano

Manifest.toml

@@ -1,6 +1,6 @@
 # This file is machine-generated - editing it directly is not advised
 
-julia_version = "1.8.0"
+julia_version = "1.7.2"
 manifest_format = "2.0"
 project_hash = "1e87134dc5853b26c2f65af5a46eb0a2d987da86"
 
@@ -12,7 +12,6 @@ version = "1.1.4"
 
 [[deps.ArgTools]]
 uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f"
-version = "1.1.1"
 
 [[deps.Artifacts]]
 uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
@@ -33,11 +32,16 @@ uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 
 [[deps.BinaryBuilderBase]]
 deps = ["CodecZlib", "Downloads", "InteractiveUtils", "JSON", "LibGit2", "LibGit2_jll", "Libdl", "Logging", "OrderedCollections", "OutputCollectors", "Pkg", "ProgressMeter", "Random", "SHA", "Scratch", "SimpleBufferStream", "TOML", "Tar", "UUIDs", "p7zip_jll", "pigz_jll"]
-git-tree-sha1 = "29d27e44a3eb75a8487e08e7e26cf3873f1759db"
+git-tree-sha1 = "af7cdc65f6ef4e2434baff93cf2d805cb54efbe6"
 repo-rev = "master"
 repo-url = "https://github.com/JuliaPackaging/BinaryBuilderBase.jl.git"
 uuid = "7f725544-6523-48cd-82d1-3fa08ff4056e"
-version = "1.17.0"
+version = "1.18.0"
+
+[[deps.BitFlags]]
+git-tree-sha1 = "84259bb6172806304b9101094a7cc4bc6f56dbc6"
+uuid = "d1d4a3ce-64b1-5f1a-9ba4-7e7e69966f35"
+version = "0.1.5"
 
 [[deps.CodecZlib]]
 deps = ["TranscodingStreams", "Zlib_jll"]
@@ -54,12 +58,11 @@ version = "3.46.0"
 [[deps.CompilerSupportLibraries_jll]]
 deps = ["Artifacts", "Libdl"]
 uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae"
-version = "0.5.2+0"
 
 [[deps.DataAPI]]
-git-tree-sha1 = "fb5f5316dd3fd4c5e7c30a24d50643b73e37cd40"
+git-tree-sha1 = "1106fa7e1256b402a86a8e7b15c00c85036fef49"
 uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a"
-version = "1.10.0"
+version = "1.11.0"
 
 [[deps.DataValueInterfaces]]
 git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6"
@@ -79,9 +82,8 @@ deps = ["Random", "Serialization", "Sockets"]
 uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b"
 
 [[deps.Downloads]]
-deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"]
+deps = ["ArgTools", "LibCURL", "NetworkOptions"]
 uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
-version = "1.6.0"
 
 [[deps.ExprTools]]
 git-tree-sha1 = "56559bbef6ca5ea0c0818fa5c90320398a6fbf8d"
@@ -110,10 +112,10 @@ uuid = "bc5e4493-9b4d-5f90-b8aa-2b2bcaad7a26"
 version = "5.7.3"
 
 [[deps.HTTP]]
-deps = ["Base64", "CodecZlib", "Dates", "IniFile", "Logging", "LoggingExtras", "MbedTLS", "NetworkOptions", "Random", "SimpleBufferStream", "Sockets", "URIs", "UUIDs"]
-git-tree-sha1 = "59ba44e0aa49b87a8c7a8920ec76f8afe87ed502"
+deps = ["Base64", "CodecZlib", "Dates", "IniFile", "Logging", "LoggingExtras", "MbedTLS", "NetworkOptions", "OpenSSL", "Random", "SimpleBufferStream", "Sockets", "URIs", "UUIDs"]
+git-tree-sha1 = "4abede886fcba15cd5fd041fef776b230d004cee"
 uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3"
-version = "1.3.3"
+version = "1.4.0"
 
 [[deps.Hiccup]]
 deps = ["MacroTools", "Test"]
@@ -162,12 +164,10 @@ version = "0.3.4"
 [[deps.LibCURL]]
 deps = ["LibCURL_jll", "MozillaCACerts_jll"]
 uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21"
-version = "0.6.3"
 
 [[deps.LibCURL_jll]]
 deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"]
 uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0"
-version = "7.84.0+0"
 
 [[deps.LibGit2]]
 deps = ["Base64", "NetworkOptions", "Printf", "SHA"]
@@ -176,12 +176,10 @@ uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"
 [[deps.LibGit2_jll]]
 deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll"]
 uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5"
-version = "1.3.0+0"
 
 [[deps.LibSSH2_jll]]
 deps = ["Artifacts", "Libdl", "MbedTLS_jll"]
 uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8"
-version = "1.10.2+0"
 
 [[deps.Libdl]]
 uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
@@ -218,7 +216,6 @@ version = "1.1.6"
 [[deps.MbedTLS_jll]]
 deps = ["Artifacts", "Libdl"]
 uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1"
-version = "2.28.0+0"
 
 [[deps.Mmap]]
 uuid = "a63ad114-7e13-5084-954f-fe012c677804"
@@ -231,7 +228,6 @@ version = "0.7.3"
 
 [[deps.MozillaCACerts_jll]]
 uuid = "14a3606d-f60d-562e-9121-12d972cd8159"
-version = "2022.2.1"
 
 [[deps.Mustache]]
 deps = ["Printf", "Tables"]
@@ -247,7 +243,6 @@ version = "1.0.0"
 
 [[deps.NetworkOptions]]
 uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908"
-version = "1.2.0"
 
 [[deps.ObjectFile]]
 deps = ["Reexport", "StructIO"]
@@ -258,7 +253,18 @@ version = "0.3.7"
 [[deps.OpenBLAS_jll]]
 deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"]
 uuid = "4536629a-c528-5b80-bd46-f80d51c5b363"
-version = "0.3.20+0"
+
+[[deps.OpenSSL]]
+deps = ["BitFlags", "Dates", "MozillaCACerts_jll", "OpenSSL_jll", "Sockets"]
+git-tree-sha1 = "02be9f845cb58c2d6029a6d5f67f4e0af3237814"
+uuid = "4d8831e6-92b7-49fb-bdf8-b643e874388c"
+version = "1.1.3"
+
+[[deps.OpenSSL_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "e60321e3f2616584ff98f0a4f18d98ae6f89bbb3"
+uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95"
+version = "1.1.17+0"
 
 [[deps.OrderedCollections]]
 git-tree-sha1 = "85f8e6578bf1f9ee0d11e7bb1b1456435479d47c"
@@ -285,7 +291,6 @@ version = "1.3.0"
 [[deps.Pkg]]
 deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"]
 uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
-version = "1.8.0"
 
 [[deps.PkgLicenses]]
 deps = ["Test"]
@@ -342,7 +347,6 @@ version = "1.3.0"
 
 [[deps.SHA]]
 uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
-version = "0.7.0"
 
 [[deps.Scratch]]
 deps = ["Dates"]
@@ -388,7 +392,6 @@ version = "0.3.0"
 [[deps.TOML]]
 deps = ["Dates"]
 uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
-version = "1.0.0"
 
 [[deps.TableTraits]]
 deps = ["IteratorInterfaceExtensions"]
@@ -398,14 +401,13 @@ version = "1.0.1"
 
 [[deps.Tables]]
 deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "LinearAlgebra", "OrderedCollections", "TableTraits", "Test"]
-git-tree-sha1 = "7149a60b01bf58787a1b83dad93f90d4b9afbe5d"
+git-tree-sha1 = "2d7164f7b8a066bcfa6224e67736ce0eb54aef5b"
 uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
-version = "1.8.1"
+version = "1.9.0"
 
 [[deps.Tar]]
 deps = ["ArgTools", "SHA"]
 uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
-version = "1.10.0"
 
 [[deps.Test]]
 deps = ["InteractiveUtils", "Logging", "Random", "Serialization"]
@@ -455,7 +457,6 @@ version = "4.3.4+0"
 [[deps.Zlib_jll]]
 deps = ["Libdl"]
 uuid = "83775a58-1f1d-513f-b197-d71354ab007a"
-version = "1.2.12+3"
 
 [[deps.ghr_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
@@ -466,7 +467,6 @@ version = "0.14.0+0"
 [[deps.libblastrampoline_jll]]
 deps = ["Artifacts", "Libdl", "OpenBLAS_jll"]
 uuid = "8e850b90-86db-534c-a0d3-1478176c7d93"
-version = "5.1.1+0"
 
 [[deps.libsodium_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
@@ -477,12 +477,10 @@ version = "1.0.20+0"
 [[deps.nghttp2_jll]]
 deps = ["Artifacts", "Libdl"]
 uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d"
-version = "1.48.0+0"
 
 [[deps.p7zip_jll]]
 deps = ["Artifacts", "Libdl"]
 uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0"
-version = "17.4.0+0"
 
 [[deps.pigz_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Zlib_jll"]

Project.toml

@@ -33,7 +33,7 @@ ghr_jll = "07c12ed4-43bc-5495-8a2a-d5838ef8d533"
 
 [compat]
 ArgParse = "1.1"
-BinaryBuilderBase = "1.17"
+BinaryBuilderBase = "1.18"
 GitHub = "5.1"
 HTTP = "0.8, 0.9, 1"
 JLD2 = "0.1.6, 0.2, 0.3, 0.4"

test/building.jl

@@ -180,22 +180,48 @@ end
 
 @testset "gfortran linking specialty flags" begin
     # We test things like linking against libgfortran with `$FC` on a couple of troublesome platforms
+    x86_64_linux = Platform("x86_64", "linux"; libgfortran_version=v"3")
+    aarch64_linux = Platform("aarch64", "linux"; libgfortran_version=v"3")
+    ppc64le_linux = Platform("powerpc64le", "linux"; libgfortran_version=v"3")
+    armv7l_linux = Platform("armv7l", "linux"; libgfortran_version=v"3")
+    x86_64_macos = Platform("x86_64", "macos"; libgfortran_version=v"3")
+    i686_windows = Platform("i686", "windows"; libgfortran_version=v"3")
+    troublesome_platforms = [
+        x86_64_linux,
+        ppc64le_linux,
+        armv7l_linux,
+        aarch64_linux,
+        x86_64_macos,
+        i686_windows,
+    ]
     expected_git_shas = Dict(
         v"4" => Dict(
-            Platform("x86_64", "linux"; libgfortran_version=v"3")  => Base.SHA1("6b705c0ff5760e236ab414fb9a366193f36d3e45"),
-            Platform("aarch64", "linux"; libgfortran_version=v"3") => Base.SHA1("fb501d0fb03bbf8f034cff50d8d8a1b2956cdf1a"),
+            x86_64_linux  => Base.SHA1("6b705c0ff5760e236ab414fb9a366193f36d3e45"),
+            ppc64le_linux => Base.SHA1("2fa9900c01dc87dad914de469e662cacd9c48e01"),
+            armv7l_linux  => Base.SHA1("192f05a64cc969cde17c026517166f98048ce27a"),
+            aarch64_linux => Base.SHA1("fb501d0fb03bbf8f034cff50d8d8a1b2956cdf1a"),
+            x86_64_macos  => Base.SHA1("5a927c7c1b0c7813564bf47ec7ff287071ed5aae"),
+            i686_windows  => Base.SHA1("f39858ccc34a63a648cf21d33ae236bfdd706d09"),
         ),
         v"5" => Dict(
-            Platform("x86_64", "linux"; libgfortran_version=v"3")  => Base.SHA1("ef9653f47abba2895b14c1086ba40ad8c64ac098"),
-            Platform("aarch64", "linux"; libgfortran_version=v"3") => Base.SHA1("ed873d488f47fb8bf5b4b89bb31ed1cad27ee27c"),
+            x86_64_linux  => Base.SHA1("ef9653f47abba2895b14c1086ba40ad8c64ac098"),
+            ppc64le_linux => Base.SHA1("b402ba20da0b800d37f69e1fbfe0e30c3b11a8db"),
+            armv7l_linux  => Base.SHA1("b282db91a2c2a33485718d8e8a743b7f5c384719"),
+            aarch64_linux => Base.SHA1("ed873d488f47fb8bf5b4b89bb31ed1cad27ee27c"),
+            x86_64_macos  => Base.SHA1("099b6a75d4f417c9f67c4da9218a56327086ec72"),
+            i686_windows  => Base.SHA1("9390a3c24a8e274e6d7245c6c977f97b406bc3f5"),
         ),
         v"6" => Dict(
-            Platform("x86_64", "linux"; libgfortran_version=v"3")  => Base.SHA1("b3829adf90f6521e4a05f20924506c8358247e06"),
-            Platform("aarch64", "linux"; libgfortran_version=v"3") => Base.SHA1("641031469413f4fa776c74310098b1b181796720"),
+            x86_64_linux  => Base.SHA1("b3829adf90f6521e4a05f20924506c8358247e06"),
+            ppc64le_linux => Base.SHA1("a12153a57fc6d8c415dc77161a39ed8ab01b16b2"),
+            armv7l_linux  => Base.SHA1("a5c66513ca6a0348ca7b1e99412524fb1ee75e87"),
+            aarch64_linux => Base.SHA1("641031469413f4fa776c74310098b1b181796720"),
+            x86_64_macos  => Base.SHA1("c01c6e840c4268123fbbd8703c8a153219752800"),
+            i686_windows  => Base.SHA1("ae50af4ca8651cb3c8f71f34d0b66ca0d8f14a99"),
         ),
     )
 
-    for gcc_version in (v"4", v"5", v"6")
+    @testset "gcc version $(gcc_version)" for gcc_version in (v"4", v"5", v"6")
         mktempdir() do build_path
             build_output_meta = autobuild(
                 build_path,
@@ -211,27 +237,23 @@ end
                 install_license /usr/share/licenses/libuv/LICENSE
                 """,
                 # Build for a few troublesome platforms
-                [
-                    Platform("x86_64", "linux"; libgfortran_version=v"3"),
-                    Platform("powerpc64le", "linux"; libgfortran_version=v"3"),
-                    Platform("armv7l", "linux"; libgfortran_version=v"3"),
-                    Platform("aarch64", "linux"; libgfortran_version=v"3"),
-                    Platform("x86_64", "macos"; libgfortran_version=v"3"),
-                    Platform("i686", "windows"; libgfortran_version=v"3"),
-                ],
+                troublesome_platforms,
                 [ExecutableProduct("hello_world_fortran", :hello_world_fortran)],
                 # Express a dependency on CSL to silence warning for fortran code
                 [Dependency("CompilerSupportLibraries_jll")];
                 preferred_gcc_version=gcc_version,
             )
 
-            for p in (Platform("x86_64", "linux"; libgfortran_version=v"3"), Platform("aarch64", "linux"; libgfortran_version=v"3"))
+            for p in troublesome_platforms
                 # Test build reproducibility
-                @test build_output_meta[p][3] == expected_git_shas[gcc_version][p]
+                # Note: for some reasons, GCC 5 for i686 windows gives different results on
+                # different systems, while still always reproducible on each of them:
+                # https://github.com/JuliaPackaging/BinaryBuilder.jl/pull/1234#issuecomment-1264192726
+                @test build_output_meta[p][3] == expected_git_shas[gcc_version][p] skip=(Sys.iswindows(p) && gcc_version==v"5")
             end
 
             # Just a simple test to ensure that it worked.
-            @test length(keys(build_output_meta)) == 6
+            @test length(keys(build_output_meta)) == length(troublesome_platforms)
         end
     end