diff --git a/lib/mix/lib/mix/tasks/clean.ex b/lib/mix/lib/mix/tasks/clean.ex index b0dcf4c6e8c..41d6dc75472 100644 --- a/lib/mix/lib/mix/tasks/clean.ex +++ b/lib/mix/lib/mix/tasks/clean.ex @@ -57,6 +57,22 @@ defmodule Mix.Tasks.Clean do |> Path.join("*#{opts[:only]}") if opts[:deps] do + deps_path = Mix.Project.deps_path() + build_lib = Path.join(build, "lib") + loaded_deps = Mix.Dep.Converger.converge([]) + + apps = + build_lib + |> Path.join("*") + |> Path.wildcard() + |> Enum.filter(&File.dir?/1) + |> Enum.map(&Path.basename/1) + |> Enum.reject(&(&1 == to_string(Mix.Project.config()[:app]))) + + Mix.Project.with_deps_lock(fn -> + Mix.Tasks.Deps.Clean.clean_generated_sources(apps, build_lib, deps_path, loaded_deps) + end) + build |> Path.wildcard() |> Enum.each(&File.rm_rf/1) diff --git a/lib/mix/lib/mix/tasks/deps.clean.ex b/lib/mix/lib/mix/tasks/deps.clean.ex index 5e59334c80a..0d450724f51 100644 --- a/lib/mix/lib/mix/tasks/deps.clean.ex +++ b/lib/mix/lib/mix/tasks/deps.clean.ex @@ -68,6 +68,7 @@ defmodule Mix.Tasks.Deps.Clean do end Mix.Project.with_build_lock(fn -> + clean_generated_sources(apps_to_clean, build_path, deps_path, loaded_deps) clean_build(apps_to_clean, build_path) end) @@ -118,6 +119,60 @@ defmodule Mix.Tasks.Deps.Clean do end end + # Called from Mix.Tasks.Clean for --deps mode. + @doc false + def clean_generated_sources([], _build_path, _deps_path, _loaded_deps), do: :ok + + def clean_generated_sources([app | rest], build_path, deps_path, loaded_deps) do + dest = + case Enum.find(loaded_deps, &(Atom.to_string(&1.app) == app)) do + %{opts: opts} -> opts[:dest] + nil -> Path.expand(Path.join(deps_path, app)) + end + + # When no Mix manifests exist (e.g. rebar3 deps compiled externally), + # infer generated files from co-located .yrl/.xrl sources. + paths = + case get_manifest_paths(build_path, app) do + [] -> + erl_paths_from_grammar_sources(dest) + + manifest_paths -> + manifest_paths + end + + maybe_remove_files(paths, dest) + + clean_generated_sources(rest, build_path, deps_path, loaded_deps) + end + + defp maybe_remove_files(paths, dest) do + paths + |> Enum.uniq() + |> Enum.map(&Path.expand(&1, dest)) + |> Enum.filter(&String.starts_with?(&1, dest <> "/")) + |> Enum.each(fn expanded -> + case File.rm(expanded) do + :ok -> :ok + {:error, reason} -> maybe_warn_failed_file_deletion({:error, reason, expanded}) + end + end) + end + + defp erl_paths_from_grammar_sources(dest) do + ~w[yrl xrl] + |> Enum.flat_map(&Path.wildcard(Path.join([dest, "**", "*.#{&1}"]))) + |> Enum.map(&(Path.rootname(&1) <> ".erl")) + end + + defp get_manifest_paths(build_path, app) do + build_path + |> Path.join(to_string(app)) + |> Path.wildcard() + |> Enum.flat_map(&Path.wildcard(Path.join(&1, ".mix/compile.{yecc,leex}"))) + |> Enum.flat_map(&Mix.Compilers.Erlang.outputs/1) + end + defp clean_build(apps, build_path) do shell = Mix.shell() diff --git a/lib/mix/test/fixtures/clean_with_yecc/deps/parser_dep/mix.exs b/lib/mix/test/fixtures/clean_with_yecc/deps/parser_dep/mix.exs new file mode 100644 index 00000000000..46cf84a7a2c --- /dev/null +++ b/lib/mix/test/fixtures/clean_with_yecc/deps/parser_dep/mix.exs @@ -0,0 +1,4 @@ +defmodule ParserDep.MixProject do + use Mix.Project + def project, do: [app: :parser_dep, version: "0.1.0"] +end diff --git a/lib/mix/test/fixtures/clean_with_yecc/deps/parser_dep/src/handwritten.erl b/lib/mix/test/fixtures/clean_with_yecc/deps/parser_dep/src/handwritten.erl new file mode 100644 index 00000000000..e387885cf31 --- /dev/null +++ b/lib/mix/test/fixtures/clean_with_yecc/deps/parser_dep/src/handwritten.erl @@ -0,0 +1,4 @@ +%% Hand-authored — NOT generated, must never be removed by mix clean. +-module(handwritten). +-export([run/0]). +run() -> ok. diff --git a/lib/mix/test/fixtures/clean_with_yecc/deps/parser_dep/src/lexer.erl b/lib/mix/test/fixtures/clean_with_yecc/deps/parser_dep/src/lexer.erl new file mode 100644 index 00000000000..a9c146678f2 --- /dev/null +++ b/lib/mix/test/fixtures/clean_with_yecc/deps/parser_dep/src/lexer.erl @@ -0,0 +1,4 @@ +%% Generated by leex from lexer.xrl — this file should be removed by mix clean. +-module(lexer). +-export([string/1]). +string(_) -> ok. diff --git a/lib/mix/test/fixtures/clean_with_yecc/deps/parser_dep/src/lexer.xrl b/lib/mix/test/fixtures/clean_with_yecc/deps/parser_dep/src/lexer.xrl new file mode 100644 index 00000000000..4b2e7d07d3d --- /dev/null +++ b/lib/mix/test/fixtures/clean_with_yecc/deps/parser_dep/src/lexer.xrl @@ -0,0 +1,5 @@ +Definitions. +D = [0-9] +Rules. +{D}+ : {token, {number, TokenLine, list_to_integer(TokenChars)}}. +Erlang code. diff --git a/lib/mix/test/fixtures/clean_with_yecc/deps/parser_dep/src/parser.erl b/lib/mix/test/fixtures/clean_with_yecc/deps/parser_dep/src/parser.erl new file mode 100644 index 00000000000..a8154fc421f --- /dev/null +++ b/lib/mix/test/fixtures/clean_with_yecc/deps/parser_dep/src/parser.erl @@ -0,0 +1,4 @@ +%% Generated by yecc from parser.yrl — this file should be removed by mix clean. +-module(parser). +-export([parse/1]). +parse(_) -> ok. diff --git a/lib/mix/test/fixtures/clean_with_yecc/deps/parser_dep/src/parser.yrl b/lib/mix/test/fixtures/clean_with_yecc/deps/parser_dep/src/parser.yrl new file mode 100644 index 00000000000..84bd1908714 --- /dev/null +++ b/lib/mix/test/fixtures/clean_with_yecc/deps/parser_dep/src/parser.yrl @@ -0,0 +1,4 @@ +Nonterminals expr. +Terminals number. +Rootsymbol expr. +expr -> number : '$1'. diff --git a/lib/mix/test/fixtures/clean_with_yecc/deps/rebar_dep/rebar.config b/lib/mix/test/fixtures/clean_with_yecc/deps/rebar_dep/rebar.config new file mode 100644 index 00000000000..9fba03ceb84 --- /dev/null +++ b/lib/mix/test/fixtures/clean_with_yecc/deps/rebar_dep/rebar.config @@ -0,0 +1 @@ +{erl_opts, []}. diff --git a/lib/mix/test/fixtures/clean_with_yecc/deps/rebar_dep/src/rb_parser.erl b/lib/mix/test/fixtures/clean_with_yecc/deps/rebar_dep/src/rb_parser.erl new file mode 100644 index 00000000000..57c8c064939 --- /dev/null +++ b/lib/mix/test/fixtures/clean_with_yecc/deps/rebar_dep/src/rb_parser.erl @@ -0,0 +1,4 @@ +%% Generated by yecc from rb_parser.yrl — this file should be removed by mix clean. +-module(rb_parser). +-export([parse/1]). +parse(_) -> ok. diff --git a/lib/mix/test/fixtures/clean_with_yecc/deps/rebar_dep/src/rb_parser.yrl b/lib/mix/test/fixtures/clean_with_yecc/deps/rebar_dep/src/rb_parser.yrl new file mode 100644 index 00000000000..84bd1908714 --- /dev/null +++ b/lib/mix/test/fixtures/clean_with_yecc/deps/rebar_dep/src/rb_parser.yrl @@ -0,0 +1,4 @@ +Nonterminals expr. +Terminals number. +Rootsymbol expr. +expr -> number : '$1'. diff --git a/lib/mix/test/mix/tasks/clean_test.exs b/lib/mix/test/mix/tasks/clean_test.exs index 3af7d79328f..b8f678b8bf2 100644 --- a/lib/mix/test/mix/tasks/clean_test.exs +++ b/lib/mix/test/mix/tasks/clean_test.exs @@ -53,6 +53,81 @@ defmodule Mix.Tasks.CleanTest do end) end + defmodule ParserOnly do + def project do + [ + app: :clean_with_yecc, + version: "0.1.0", + deps: [{:parser_dep, path: "deps/parser_dep"}] + ] + end + end + + defp yecc_manifest(build_dir, relative_paths) do + write_manifest(build_dir, "compile.yecc", relative_paths) + end + + defp leex_manifest(build_dir, relative_paths) do + write_manifest(build_dir, "compile.leex", relative_paths) + end + + defp write_manifest(build_dir, name, relative_paths) do + mix_dir = Path.join(build_dir, ".mix") + File.mkdir_p!(mix_dir) + entries = Enum.map(relative_paths, &{&1, []}) + File.write!(Path.join(mix_dir, name), :erlang.term_to_binary({1, entries})) + end + + test "--deps removes yecc-generated .erl files from dep source" do + in_fixture("clean_with_yecc", fn -> + Mix.Project.push(ParserOnly) + + build_dir = "_build/dev/lib/parser_dep" + File.mkdir_p!(Path.join(build_dir, "ebin")) + yecc_manifest(build_dir, ["src/parser.erl"]) + + Mix.Tasks.Clean.run(["--deps"]) + + refute File.exists?("deps/parser_dep/src/parser.erl") + assert File.exists?("deps/parser_dep/src/parser.yrl") + refute File.exists?("_build/dev") + end) + end + + test "without --deps does not touch dep generated files" do + in_fixture("clean_with_yecc", fn -> + Mix.Project.push(ParserOnly) + + Mix.Tasks.Clean.run([]) + + assert File.exists?("deps/parser_dep/src/parser.erl") + end) + end + + test "--deps --only dev scopes generated-file cleanup to dev environment" do + in_fixture("clean_with_yecc", fn -> + Mix.Project.push(ParserOnly) + + dev_dir = "_build/dev/lib/parser_dep" + test_dir = "_build/test/lib/parser_dep" + + File.mkdir_p!(Path.join(dev_dir, "ebin")) + File.mkdir_p!(Path.join(test_dir, "ebin")) + + # dev tracks parser.erl; test tracks lexer.erl + yecc_manifest(dev_dir, ["src/parser.erl"]) + leex_manifest(test_dir, ["src/lexer.erl"]) + + Mix.Tasks.Clean.run(["--deps", "--only", "dev"]) + + refute File.exists?("deps/parser_dep/src/parser.erl") + # test manifest was not processed + assert File.exists?("deps/parser_dep/src/lexer.erl") + refute File.exists?("_build/dev") + assert File.exists?("_build/test") + end) + end + test "invokes compiler hook defined in project" do Mix.ProjectStack.post_config(compilers: Mix.compilers() ++ [:testc]) diff --git a/lib/mix/test/mix/tasks/deps.clean_test.exs b/lib/mix/test/mix/tasks/deps.clean_test.exs new file mode 100644 index 00000000000..c8ac393af03 --- /dev/null +++ b/lib/mix/test/mix/tasks/deps.clean_test.exs @@ -0,0 +1,261 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team + +Code.require_file("../../test_helper.exs", __DIR__) + +defmodule Mix.Tasks.Deps.CleanTest do + use MixTest.Case + + # Project with both a Mix dep (parser_dep) and a rebar3 dep (rebar_dep). + # Both use path: so they are available without network access in tests. + defmodule BothDeps do + def project do + [ + app: :clean_with_yecc, + version: "0.1.0", + deps: [ + {:parser_dep, path: "deps/parser_dep"}, + {:rebar_dep, path: "deps/rebar_dep"} + ] + ] + end + end + + defmodule ParserOnly do + def project do + [ + app: :clean_with_yecc, + version: "0.1.0", + deps: [{:parser_dep, path: "deps/parser_dep"}] + ] + end + end + + # Writes a compile.yecc manifest exactly as the real compiler does: + # relative paths from the dep's working directory, version tag 1. + defp yecc_manifest(build_dir, relative_paths) do + write_manifest(build_dir, "compile.yecc", relative_paths) + end + + defp leex_manifest(build_dir, relative_paths) do + write_manifest(build_dir, "compile.leex", relative_paths) + end + + defp write_manifest(build_dir, name, relative_paths) do + mix_dir = Path.join(build_dir, ".mix") + File.mkdir_p!(mix_dir) + entries = Enum.map(relative_paths, &{&1, []}) + File.write!(Path.join(mix_dir, name), :erlang.term_to_binary({1, entries})) + end + + defp touch_build(app) do + dir = "_build/dev/lib/#{app}" + File.mkdir_p!(Path.join(dir, "ebin")) + + File.write!( + Path.join([dir, "ebin", "#{app}.app"]), + "{application, #{app}, [{vsn,\"0.1.0\"}]}." + ) + + dir + end + + test "removes yecc-generated .erl, preserves .yrl source" do + in_fixture("clean_with_yecc", fn -> + Mix.Project.push(ParserOnly) + build_dir = touch_build(:parser_dep) + yecc_manifest(build_dir, ["src/parser.erl"]) + + Mix.Tasks.Deps.Clean.run(["parser_dep"]) + + refute File.exists?("deps/parser_dep/src/parser.erl") + assert File.exists?("deps/parser_dep/src/parser.yrl") + end) + end + + test "removes leex-generated .erl, preserves .xrl source" do + in_fixture("clean_with_yecc", fn -> + Mix.Project.push(ParserOnly) + build_dir = touch_build(:parser_dep) + leex_manifest(build_dir, ["src/lexer.erl"]) + + Mix.Tasks.Deps.Clean.run(["parser_dep"]) + + refute File.exists?("deps/parser_dep/src/lexer.erl") + assert File.exists?("deps/parser_dep/src/lexer.xrl") + end) + end + + test "hand-written .erl not tracked in any manifest is never deleted" do + in_fixture("clean_with_yecc", fn -> + Mix.Project.push(ParserOnly) + build_dir = touch_build(:parser_dep) + # manifest only tracks parser.erl — handwritten.erl is NOT listed + yecc_manifest(build_dir, ["src/parser.erl"]) + + Mix.Tasks.Deps.Clean.run(["parser_dep"]) + + assert File.exists?("deps/parser_dep/src/handwritten.erl") + end) + end + + test "cleans both yecc and leex generated files in one pass" do + in_fixture("clean_with_yecc", fn -> + Mix.Project.push(ParserOnly) + build_dir = touch_build(:parser_dep) + yecc_manifest(build_dir, ["src/parser.erl"]) + leex_manifest(build_dir, ["src/lexer.erl"]) + + Mix.Tasks.Deps.Clean.run(["parser_dep"]) + + refute File.exists?("deps/parser_dep/src/parser.erl") + refute File.exists?("deps/parser_dep/src/lexer.erl") + assert File.exists?("deps/parser_dep/src/parser.yrl") + assert File.exists?("deps/parser_dep/src/lexer.xrl") + end) + end + + test "generated .erl in non-standard source directory is deleted" do + in_fixture("clean_with_yecc", fn -> + Mix.Project.push(ParserOnly) + build_dir = touch_build(:parser_dep) + # dep uses erlc_paths: ["grammar"] instead of the default "src" + File.mkdir_p!("deps/parser_dep/grammar") + File.write!("deps/parser_dep/grammar/dialect.erl", "-module(dialect).") + yecc_manifest(build_dir, ["grammar/dialect.erl"]) + + Mix.Tasks.Deps.Clean.run(["parser_dep"]) + + refute File.exists?("deps/parser_dep/grammar/dialect.erl") + end) + end + + test "--build still removes generated source files, leaves dep directory" do + in_fixture("clean_with_yecc", fn -> + Mix.Project.push(ParserOnly) + build_dir = touch_build(:parser_dep) + yecc_manifest(build_dir, ["src/parser.erl"]) + + Mix.Tasks.Deps.Clean.run(["--build", "parser_dep"]) + + refute File.exists?("deps/parser_dep/src/parser.erl") + # --build keeps the dep source directory intact + assert File.dir?("deps/parser_dep") + assert File.exists?("deps/parser_dep/src/parser.yrl") + end) + end + + test "--all --build removes generated files from every dep" do + in_fixture("clean_with_yecc", fn -> + Mix.Project.push(BothDeps) + parser_build = touch_build(:parser_dep) + rebar_build = touch_build(:rebar_dep) + yecc_manifest(parser_build, ["src/parser.erl"]) + yecc_manifest(rebar_build, ["src/rb_parser.erl"]) + + Mix.Tasks.Deps.Clean.run(["--all", "--build"]) + + refute File.exists?("deps/parser_dep/src/parser.erl") + refute File.exists?("deps/rebar_dep/src/rb_parser.erl") + assert File.exists?("deps/parser_dep/src/parser.yrl") + assert File.exists?("deps/rebar_dep/src/rb_parser.yrl") + end) + end + + test "manifests in multiple envs point to same .erl — Enum.uniq deduplicates before deletion" do + in_fixture("clean_with_yecc", fn -> + Mix.Project.push(ParserOnly) + + for env <- ["dev", "test"] do + dir = "_build/#{env}/lib/parser_dep" + File.mkdir_p!(Path.join(dir, "ebin")) + yecc_manifest(dir, ["src/parser.erl"]) + end + + Mix.Tasks.Deps.Clean.run(["parser_dep"]) + + refute File.exists?("deps/parser_dep/src/parser.erl") + end) + end + + test "--only scopes manifest reading to the specified environment" do + in_fixture("clean_with_yecc", fn -> + Mix.Project.push(ParserOnly) + + dev_dir = "_build/dev/lib/parser_dep" + test_dir = "_build/test/lib/parser_dep" + + for dir <- [dev_dir, test_dir] do + File.mkdir_p!(Path.join(dir, "ebin")) + end + + # dev tracks src/parser.erl; test tracks src/lexer.erl + yecc_manifest(dev_dir, ["src/parser.erl"]) + leex_manifest(test_dir, ["src/lexer.erl"]) + + Mix.Tasks.Deps.Clean.run(["parser_dep", "--only", "dev"]) + + refute File.exists?("deps/parser_dep/src/parser.erl") + # test manifest was not processed + assert File.exists?("deps/parser_dep/src/lexer.erl") + end) + end + + test "rebar3 dep: deletes .erl files inferred from co-located .yrl/.xrl when no manifests exist" do + in_fixture("clean_with_yecc", fn -> + Mix.Project.push(BothDeps) + # No build dir created for rebar_dep — rebar3 never writes Mix manifests. + # The fixture has src/rb_parser.yrl and src/rb_parser.erl pre-populated. + + Mix.Tasks.Deps.Clean.run(["rebar_dep"]) + + refute File.exists?("deps/rebar_dep/src/rb_parser.erl") + assert File.exists?("deps/rebar_dep/src/rb_parser.yrl") + end) + end + + test "dep with no manifests removes .erl files inferred from .yrl/.xrl sources" do + in_fixture("clean_with_yecc", fn -> + Mix.Project.push(ParserOnly) + # no _build at all — dep was never compiled by Mix + # parser.erl/lexer.erl are inferred from parser.yrl/lexer.xrl and deleted + + Mix.Tasks.Deps.Clean.run(["parser_dep"]) + + refute File.exists?("deps/parser_dep/src/parser.erl") + refute File.exists?("deps/parser_dep/src/lexer.erl") + assert File.exists?("deps/parser_dep/src/parser.yrl") + assert File.exists?("deps/parser_dep/src/lexer.xrl") + # hand-written .erl with no grammar counterpart is preserved + assert File.exists?("deps/parser_dep/src/handwritten.erl") + end) + end + + test "idempotent: running deps.clean twice does not error" do + in_fixture("clean_with_yecc", fn -> + Mix.Project.push(ParserOnly) + build_dir = touch_build(:parser_dep) + yecc_manifest(build_dir, ["src/parser.erl"]) + + Mix.Tasks.Deps.Clean.run(["parser_dep"]) + # parser.erl is now gone and _build deleted; second run must not raise + Mix.Tasks.Deps.Clean.run(["parser_dep"]) + # the second run warns that the dep is not in the build directory + assert_received {:mix_shell, :error, ["warning: the dependency parser_dep " <> _]} + end) + end + + test "does not delete a file whose expanded path escapes the dep source root" do + in_fixture("clean_with_yecc", fn -> + Mix.Project.push(ParserOnly) + build_dir = touch_build(:parser_dep) + # A hypothetical manifest entry with path traversal + write_manifest(build_dir, "compile.yecc", ["../../some_other_file.erl"]) + File.write!("some_other_file.erl", "not to be deleted") + + Mix.Tasks.Deps.Clean.run(["parser_dep"]) + + assert File.exists?("some_other_file.erl") + end) + end +end