Fix shell function symlink_contents_to_dir, add test (#377)

Fix shell function symlink_contents_to_dir, add test

- test for the case when we symlink the contents of two directories with the same inner directories (include)
- the test data of the test for shell script helper also changes, as it uses the real shell toolchain implementation text
- Fixes the issue from #330

examples/BUILD

@@ -17,6 +17,7 @@ tests = [
     "@rules_foreign_cc//test:shell_script_conversion_suite",
     "@rules_foreign_cc//test:utils_test_suite",
     "@rules_foreign_cc//test:shell_script_inner_fun_test",
+    "@rules_foreign_cc//test:shell_method_symlink_contents_to_dir_test",
     "//cc_configure_make:libevent_echosrv1",
 ]
 

test/BUILD

@@ -3,6 +3,7 @@ load(":convert_shell_script_test.bzl", "shell_script_conversion_suite")
 load(":utils_test.bzl", "utils_test_suite")
 load(":shell_script_helper_test_rule.bzl", "shell_script_helper_test_rule")
 load("@bazel_skylib//rules:diff_test.bzl", "diff_test")
+load(":symlink_contents_to_dir_test_rule.bzl", "symlink_contents_to_dir_test_rule")
 
 cmake_script_test_suite()
 
@@ -21,8 +22,36 @@ result_file = select({
     "//conditions:default": "expected/inner_fun_text.txt",
 })
 
+result_file_symlink_dirs = select({
+    "@bazel_tools//src/conditions:darwin": "expected/out_symlinked_dirs_osx.txt",
+    "//conditions:default": "expected/out_symlinked_dirs.txt",
+})
+
 diff_test(
     name = "shell_script_inner_fun_test",
     file1 = ":shell_script_inner_fun",
     file2 = result_file,
 )
+
+filegroup(
+    name = "dir1_fg",
+    srcs = glob(["dir1/**"], exclude_directories = 0),
+)
+
+filegroup(
+    name = "dir2_fg",
+    srcs = glob(["dir2/**"], exclude_directories = 0),
+)
+
+symlink_contents_to_dir_test_rule(
+    name = "symlink_dirs",
+    out = "out_symlinked_dirs.txt",
+    dir1 = ":dir1_fg",
+    dir2 = ":dir2_fg",
+)
+
+diff_test(
+    name = "shell_method_symlink_contents_to_dir_test",
+    file1 = result_file_symlink_dirs,
+    file2 = ":symlink_dirs",
+)

test/dir1/include/header1.h

@@ -0,0 +1 @@
+#include <stdio>

test/dir2/include/header2.h

@@ -0,0 +1 @@
+#include <string>

test/expected/inner_fun_text.txt

@@ -1,13 +1,13 @@
 function symlink_contents_to_dir() {
 local target="$2"
-mkdir -p $target
-if [[ -f $1 ]]; then
+mkdir -p "$target"
+if [[ -f "$1" ]]; then
 symlink_to_dir $1 $target
-return 0
-fi
-
-if [[ -d $1 || -L $1 ]]; then
-local children=$(find -H $1 -maxdepth 1 -mindepth 1)
+elif [[ -L "$1" ]]; then
+local actual=$(readlink "$1")
+symlink_contents_to_dir $actual $target
+elif [[ -d "$1" ]]; then
+local children=$(find -H "$1" -maxdepth 1 -mindepth 1)
 for child in $children; do
 symlink_to_dir $child $target
 done
@@ -15,14 +15,18 @@ fi
 }
 function symlink_to_dir() {
 local target="$2"
-mkdir -p ${target}
-
-if [[ -d $1 ]]; then
-ln -s -t ${target} $1
-elif [[ -f $1 ]]; then
-ln -s -t ${target} $1
-elif [[ -L $1 ]]; then
-cp --no-target-directory $1 ${target}
+mkdir -p "$target"
+if [[ -f "$1" ]]; then
+ln -s -f -t "$target" "$1"
+elif [[ -L "$1" ]]; then
+cp $1 $2
+elif [[ -d "$1" ]]; then
+local children=$(find -H "$1" -maxdepth 1 -mindepth 1)
+local dirname=$(basename "$1")
+mkdir -p "$target/$dirname"
+for child in $children; do
+symlink_to_dir $child $target/$dirname
+done
 else
 echo "Can not copy $1"
 fi

test/expected/inner_fun_text_osx.txt

@@ -1,31 +1,32 @@
 function symlink_contents_to_dir() {
-
 local target="$2"
-mkdir -p $target
-if [[ -f $1 ]]; then
+mkdir -p "$target"
+if [[ -f "$1" ]]; then
 symlink_to_dir $1 $target
-return 0
-fi
-
-if [[ -d $1 || -L $1 ]]; then
-local children=$(find -H $1 -maxdepth 1 -mindepth 1)
+elif [[ -L "$1" ]]; then
+local actual=$(readlink "$1")
+symlink_contents_to_dir $actual $target
+elif [[ -d "$1" ]]; then
+local children=$(find "$1" -maxdepth 1 -mindepth 1)
 for child in $children; do
 symlink_to_dir $child $target
 done
 fi
 }
 function symlink_to_dir() {
-
 local target="$2"
-mkdir -p ${target}
-
-if [[ -d $1 ]]; then
-local dir_name="$(basename "$1")"
-ln -s $1 ${target}/${dir_name}
-elif [[ -f $1 ]]; then
-ln -s $1 ${target}
-elif [[ -L $1 ]]; then
-cp $1 ${target}
+mkdir -p "$target"
+if [[ -f "$1" ]]; then
+ln -s -f "$1" "$target"
+elif [[ -L "$1" ]]; then
+cp $1 $2
+elif [[ -d "$1" ]]; then
+local children=$(find "$1" -maxdepth 1 -mindepth 1)
+local dirname=$(basename "$1")
+mkdir -p "$target/$dirname"
+for child in $children; do
+symlink_to_dir $child $target/$dirname
+done
 else
 echo "Can not copy $1"
 fi

test/expected/out_symlinked_dirs.txt

@@ -0,0 +1,6 @@
+aaa:
+include
+
+aaa/include:
+header1.h
+header2.h

test/expected/out_symlinked_dirs_osx.txt

@@ -0,0 +1,5 @@
+include
+
+aaa/include:
+header1.h
+header2.h

test/symlink_contents_to_dir_test_rule.bzl

@@ -0,0 +1,41 @@
+load(
+    "@rules_foreign_cc//tools/build_defs:shell_script_helper.bzl",
+    "convert_shell_script",
+)
+load("@rules_foreign_cc//tools/build_defs:detect_root.bzl", "detect_root", "filter_containing_dirs_from_inputs")
+
+def _impl(ctx):
+    out = ctx.actions.declare_file(ctx.attr.out)
+    dir1 = detect_root(ctx.attr.dir1)
+    dir2 = detect_root(ctx.attr.dir2)
+    script_lines = [
+        "##mkdirs## aaa",
+        "##symlink_contents_to_dir## %s aaa" % dir1,
+        "##symlink_contents_to_dir## %s aaa" % dir2,
+        "ls -R aaa > %s" % out.path,
+    ]
+    converted_script = convert_shell_script(ctx, script_lines)
+    ctx.actions.run_shell(
+        mnemonic = "TestSymlinkContentsToDir",
+        inputs = depset(
+            direct =
+                filter_containing_dirs_from_inputs(ctx.attr.dir1.files.to_list()) +
+                filter_containing_dirs_from_inputs(ctx.attr.dir2.files.to_list()),
+        ),
+        outputs = [out],
+        command = converted_script,
+        execution_requirements = {"block-network": ""},
+    )
+    return [DefaultInfo(files = depset([out]))]
+
+symlink_contents_to_dir_test_rule = rule(
+    implementation = _impl,
+    attrs = {
+        "dir1": attr.label(allow_files = True),
+        "dir2": attr.label(allow_files = True),
+        "out": attr.string(),
+    },
+    toolchains = [
+        "@rules_foreign_cc//tools/build_defs/shell_toolchain/toolchains:shell_commands",
+    ],
+)

tools/build_defs/detect_root.bzl

@@ -39,3 +39,20 @@ def _get_level(path):
         normalized = new_normalized
 
     return normalized.count("/")
+
+"""When the directories are also passed in the filegroup with the sources,
+we get into a situation when we have containing in the sources list,
+which is not allowed by Bazel (execroot creation code fails).
+The parent directories will be created for us in the execroot anyway,
+so we filter them out."""
+
+def filter_containing_dirs_from_inputs(input_files_list):
+    # This puts directories in front of their children in list
+    sorted_list = sorted(input_files_list)
+    contains_map = {}
+    for input in input_files_list:
+        # If the immediate parent directory is already in the list, remove it
+        if contains_map.get(input.dirname):
+            contains_map.pop(input.dirname)
+        contains_map[input.path] = input
+    return contains_map.values()

tools/build_defs/framework.bzl

@@ -11,7 +11,7 @@ load(
     "get_env_vars",
     "targets_windows",
 )
-load("@rules_foreign_cc//tools/build_defs:detect_root.bzl", "detect_root")
+load("@rules_foreign_cc//tools/build_defs:detect_root.bzl", "detect_root", "filter_containing_dirs_from_inputs")
 load(
     "@rules_foreign_cc//tools/build_defs:run_shell_file_utils.bzl",
     "copy_directory",
@@ -345,6 +345,9 @@ rm -rf $BUILD_TMPDIR $EXT_BUILD_DEPS""",
 rules_foreign_cc: Printing build logs:\\n\\n_____ BEGIN BUILD LOGS _____\\n"
 ##cat## $$BUILD_LOG$$
 ##echo## "\\n_____ END BUILD LOGS _____\\n"
+##echo## "Printing build script:\\n\\n_____ BEGIN BUILD SCRIPT _____\\n"
+##cat## $$BUILD_SCRIPT$$
+##echo## "\\n_____ END BUILD SCRIPT _____\\n"
 ##echo## "rules_foreign_cc: Build script location: $$BUILD_SCRIPT$$\\n"
 ##echo## "rules_foreign_cc: Build log location: $$BUILD_LOG$$\\n\\n"
 """,
@@ -596,23 +599,6 @@ def _define_inputs(attrs):
                           ext_build_dirs,
     )
 
-"""When the directories are also passed in the filegroup with the sources,
-we get into a situation when we have containing in the sources list,
-which is not allowed by Bazel (execroot creation code fails).
-The parent directories will be created for us in the execroot anyway,
-so we filter them out."""
-
-def filter_containing_dirs_from_inputs(input_files_list):
-    # This puts directories in front of their children in list
-    sorted_list = sorted(input_files_list)
-    contains_map = {}
-    for input in input_files_list:
-        # If the immediate parent directory is already in the list, remove it
-        if contains_map.get(input.dirname):
-            contains_map.pop(input.dirname)
-        contains_map[input.path] = input
-    return contains_map.values()
-
 def uniq_list_keep_order(list):
     result = []
     contains_map = {}

tools/build_defs/shell_toolchain/toolchains/impl/default_commands.bzl

@@ -65,16 +65,16 @@ def copy_dir_contents_to_dir(source, target):
 
 def symlink_contents_to_dir(source, target):
     text = """local target="$2"
-mkdir -p $target
-if [[ -f $1 ]]; then
-  ##symlink_to_dir## $1 $target
-  return 0
-fi
-
-if [[ -d $1 || -L $1 ]]; then
-  local children=$(find -H $1 -maxdepth 1 -mindepth 1)
+mkdir -p "$target"
+if [[ -f "$1" ]]; then
+  ##symlink_to_dir## "$1" "$target"
+elif [[ -L "$1" ]]; then
+  local actual=$(readlink "$1")
+  ##symlink_contents_to_dir## "$actual" "$target"
+elif [[ -d "$1" ]]; then
+  local children=$(find -H "$1" -maxdepth 1 -mindepth 1)
   for child in $children; do
-    ##symlink_to_dir## $child $target
+    ##symlink_to_dir## "$child" "$target"
   done
 fi
 """
@@ -82,14 +82,18 @@ fi
 
 def symlink_to_dir(source, target):
     text = """local target="$2"
-mkdir -p ${target}
-
-if [[ -d $1 ]]; then
-  ln -s -t ${target} $1
-elif [[ -f $1 ]]; then
-  ln -s -t ${target} $1
-elif [[ -L $1 ]]; then
-  cp --no-target-directory $1 ${target}
+mkdir -p "$target"
+if [[ -f "$1" ]]; then
+  ln -s -f -t "$target" "$1"
+elif [[ -L "$1" ]]; then
+  local actual=$(readlink "$1")
+  ##symlink_to_dir## "$actual" "$target"
+elif [[ -d "$1" ]]; then
+  local children=$(find -H "$1" -maxdepth 1 -mindepth 1)
+  local dirname=$(basename "$1")
+  for child in $children; do
+    ##symlink_to_dir## "$child" "$target/$dirname"
+  done
 else
   echo "Can not copy $1"
 fi

tools/build_defs/shell_toolchain/toolchains/impl/linux_commands.bzl

@@ -65,16 +65,16 @@ def copy_dir_contents_to_dir(source, target):
 
 def symlink_contents_to_dir(source, target):
     text = """local target="$2"
-mkdir -p $target
-if [[ -f $1 ]]; then
-  ##symlink_to_dir## $1 $target
-  return 0
-fi
-
-if [[ -d $1 || -L $1 ]]; then
-  local children=$(find -H $1 -maxdepth 1 -mindepth 1)
+mkdir -p "$target"
+if [[ -f "$1" ]]; then
+  ##symlink_to_dir## "$1" "$target"
+elif [[ -L "$1" ]]; then
+  local actual=$(readlink "$1")
+  ##symlink_contents_to_dir## "$actual" "$target"
+elif [[ -d "$1" ]]; then
+  local children=$(find -H "$1" -maxdepth 1 -mindepth 1)
   for child in $children; do
-    ##symlink_to_dir## $child $target
+    ##symlink_to_dir## "$child" "$target"
   done
 fi
 """
@@ -82,14 +82,18 @@ fi
 
 def symlink_to_dir(source, target):
     text = """local target="$2"
-mkdir -p ${target}
-
-if [[ -d $1 ]]; then
-  ln -s -t ${target} $1
-elif [[ -f $1 ]]; then
-  ln -s -t ${target} $1
-elif [[ -L $1 ]]; then
-  cp --no-target-directory $1 ${target}
+mkdir -p "$target"
+if [[ -f "$1" ]]; then
+  ln -s -f -t "$target" "$1"
+elif [[ -L "$1" ]]; then
+  cp $1 $2
+elif [[ -d "$1" ]]; then
+  local children=$(find -H "$1" -maxdepth 1 -mindepth 1)
+  local dirname=$(basename "$1")
+  mkdir -p "$target/$dirname"
+  for child in $children; do
+    ##symlink_to_dir## "$child" "$target/$dirname"
+  done
 else
   echo "Can not copy $1"
 fi

tools/build_defs/shell_toolchain/toolchains/impl/osx_commands.bzl

@@ -62,45 +62,61 @@ fi
 
 def copy_dir_contents_to_dir(source, target):
     text = """
-local children=$(find $1 -maxdepth 1 -mindepth 1)
+local children=$(find "$1" -maxdepth 1 -mindepth 1)
 local target="$2"
-mkdir -p ${target}
+mkdir -p "${target}"
 for child in $children; do
-  cp -R -L $child ${target}
+  if [[ -f "$child" ]]; then
+    cp "$child" "$target"
+  elif [[ -L "$child" ]]; then
+    local $actual=$(readlink "$child")
+    if [[ -f "$actual" ]]; then
+      cp "$actual" "$target"
+    else
+      local dirn=$(basename "$actual")
+      mkdir -p "$target/$dirn"
+      ##copy_dir_contents_to_dir## "$actual" "$target/$dirn"
+    fi
+  elif [[ -d "$child" ]]; then
+    local dirn=$(basename "$child")
+    mkdir -p "$target/$dirn"
+    ##copy_dir_contents_to_dir## "$child" "$target/$dirn"
+  fi
 done
 """
     return FunctionAndCall(text = text)
 
 def symlink_contents_to_dir(source, target):
-    text = """
-local target="$2"
-mkdir -p $target
-if [[ -f $1 ]]; then
-  ##symlink_to_dir## $1 $target
-  return 0
-fi
-
-if [[ -d $1 || -L $1 ]]; then
-  local children=$(find -H $1 -maxdepth 1 -mindepth 1)
+    text = """local target="$2"
+mkdir -p "$target"
+if [[ -f "$1" ]]; then
+  ##symlink_to_dir## "$1" "$target"
+elif [[ -L "$1" ]]; then
+  local actual=$(readlink "$1")
+  ##symlink_contents_to_dir## "$actual" "$target"
+elif [[ -d "$1" ]]; then
+  local children=$(find "$1" -maxdepth 1 -mindepth 1)
   for child in $children; do
-    ##symlink_to_dir## $child $target
+    ##symlink_to_dir## "$child" "$target"
   done
 fi
 """
     return FunctionAndCall(text = text)
 
 def symlink_to_dir(source, target):
-    text = """
-local target="$2"
-mkdir -p ${target}
-
-if [[ -d $1 ]]; then
-  local dir_name="$(basename "$1")"
-  ln -s $1 ${target}/${dir_name}
-elif [[ -f $1 ]]; then
-  ln -s $1 ${target}
-elif [[ -L $1 ]]; then
-  cp $1 ${target}
+    text = """local target="$2"
+mkdir -p "$target"
+if [[ -f "$1" ]]; then
+  ln -s -f "$1" "$target"
+elif [[ -L "$1" ]]; then
+  cp $1 $2
+elif [[ -d "$1" ]]; then
+  local children=$(find "$1" -maxdepth 1 -mindepth 1)
+  local dirname=$(basename "$1")
+  mkdir -p "$target/$dirname"
+  for child in $children; do
+    ##symlink_to_dir## "$child" "$target/$dirname"
+  done
 else
   echo "Can not copy $1"
 fi

tools/build_defs/shell_toolchain/toolchains/impl/windows_commands.bzl

@@ -65,16 +65,16 @@ def copy_dir_contents_to_dir(source, target):
 
 def symlink_contents_to_dir(source, target):
     text = """local target="$2"
-mkdir -p $target
-if [[ -f $1 ]]; then
-  ##symlink_to_dir## $1 $target
-  return 0
-fi
-
-if [[ -d $1 || -L $1 ]]; then
-  local children=$(find -H $1 -maxdepth 1 -mindepth 1)
+mkdir -p "$target"
+if [[ -f "$1" ]]; then
+  ##symlink_to_dir## "$1" "$target"
+elif [[ -L "$1" ]]; then
+  local actual=$(readlink "$1")
+  ##symlink_contents_to_dir## "$actual" "$target"
+elif [[ -d "$1" ]]; then
+  local children=$($REAL_FIND -H "$1" -maxdepth 1 -mindepth 1)
   for child in $children; do
-    ##symlink_to_dir## $child $target
+    ##symlink_to_dir## "$child" "$target"
   done
 fi
 """
@@ -82,14 +82,17 @@ fi
 
 def symlink_to_dir(source, target):
     text = """local target="$2"
-mkdir -p ${target}
-
-if [[ -d $1 ]]; then
-  ln -s -t ${target} $1
-elif [[ -f $1 ]]; then
-  ln -s -t ${target} $1
-elif [[ -L $1 ]]; then
-  cp --no-target-directory $1 ${target}
+mkdir -p "$target"
+if [[ -f "$1" ]]; then
+  ln -s -f -t "$target" "$1"
+elif [[ -L "$1" ]]; then
+  local actual=$(readlink "$1")
+  ##symlink_to_dir## "$actual" "$target"
+elif [[ -d "$1" ]]; then
+  local children=$($REAL_FIND -H "$1" -maxdepth 1 -mindepth 1)
+  for child in $children; do
+    ##symlink_to_dir## "$child" "$target/$(basename $1)"
+  done
 else
   echo "Can not copy $1"
 fi