Adding a preserve time feature to copy_to_directory and copy_directory (#898)

docs/copy_directory.md

@@ -41,7 +41,7 @@ for more context.
 ## copy_directory_bin_action
 
 <pre>
-copy_directory_bin_action(<a href="#copy_directory_bin_action-ctx">ctx</a>, <a href="#copy_directory_bin_action-src">src</a>, <a href="#copy_directory_bin_action-dst">dst</a>, <a href="#copy_directory_bin_action-copy_directory_bin">copy_directory_bin</a>, <a href="#copy_directory_bin_action-hardlink">hardlink</a>, <a href="#copy_directory_bin_action-verbose">verbose</a>)
+copy_directory_bin_action(<a href="#copy_directory_bin_action-ctx">ctx</a>, <a href="#copy_directory_bin_action-src">src</a>, <a href="#copy_directory_bin_action-dst">dst</a>, <a href="#copy_directory_bin_action-copy_directory_bin">copy_directory_bin</a>, <a href="#copy_directory_bin_action-hardlink">hardlink</a>, <a href="#copy_directory_bin_action-verbose">verbose</a>, <a href="#copy_directory_bin_action-preserve_mtime">preserve_mtime</a>)
 </pre>
 
 Factory function that creates an action to copy a directory from src to dst using a tool binary.
@@ -64,5 +64,6 @@ within other rule implementations.
 | <a id="copy_directory_bin_action-copy_directory_bin"></a>copy_directory_bin |  Copy to directory tool binary.   |  none |
 | <a id="copy_directory_bin_action-hardlink"></a>hardlink |  Controls when to use hardlinks to files instead of making copies.<br><br>See copy_directory rule documentation for more details.   |  `"auto"` |
 | <a id="copy_directory_bin_action-verbose"></a>verbose |  If true, prints out verbose logs to stdout   |  `False` |
+| <a id="copy_directory_bin_action-preserve_mtime"></a>preserve_mtime |  If true, preserve the modified time from the source.   |  `False` |
 
 

docs/copy_to_directory.md

@@ -9,7 +9,7 @@ Copy files and directories to an output directory.
 <pre>
 copy_to_directory(<a href="#copy_to_directory-name">name</a>, <a href="#copy_to_directory-srcs">srcs</a>, <a href="#copy_to_directory-out">out</a>, <a href="#copy_to_directory-allow_overwrites">allow_overwrites</a>, <a href="#copy_to_directory-exclude_srcs_packages">exclude_srcs_packages</a>, <a href="#copy_to_directory-exclude_srcs_patterns">exclude_srcs_patterns</a>,
                   <a href="#copy_to_directory-hardlink">hardlink</a>, <a href="#copy_to_directory-include_external_repositories">include_external_repositories</a>, <a href="#copy_to_directory-include_srcs_packages">include_srcs_packages</a>,
-                  <a href="#copy_to_directory-include_srcs_patterns">include_srcs_patterns</a>, <a href="#copy_to_directory-replace_prefixes">replace_prefixes</a>, <a href="#copy_to_directory-root_paths">root_paths</a>, <a href="#copy_to_directory-verbose">verbose</a>)
+                  <a href="#copy_to_directory-include_srcs_patterns">include_srcs_patterns</a>, <a href="#copy_to_directory-preserve_mtime">preserve_mtime</a>, <a href="#copy_to_directory-replace_prefixes">replace_prefixes</a>, <a href="#copy_to_directory-root_paths">root_paths</a>, <a href="#copy_to_directory-verbose">verbose</a>)
 </pre>
 
 Copies files and directories to an output directory.
@@ -57,6 +57,7 @@ for more information on supported globbing patterns.
 | <a id="copy_to_directory-include_external_repositories"></a>include_external_repositories |  List of external repository names (with glob support) to include in the output directory.<br><br>Files from external repositories are only copied into the output directory if the external repository they come from matches one of the external repository patterns specified or if they are in the same external repository as this target.<br><br>When copied from an external repository, the file path in the output directory defaults to the file's path within the external repository. The external repository name is _not_ included in that path.<br><br>For example, the following copies `@external_repo//path/to:file` to `path/to/file` within the output directory.<br><br><pre><code>copy_to_directory(&#10;    name = "dir",&#10;    include_external_repositories = ["external_*"],&#10;    srcs = ["@external_repo//path/to:file"],&#10;)</code></pre><br><br>Files that come from matching external are subject to subsequent filters and transformations to determine if they are copied and what their path in the output directory will be. The external repository name of the file from an external repository is not included in the output directory path and is considered in subsequent filters and transformations.<br><br>Globs are supported (see rule docstring above).   | List of strings | optional |  `[]`  |
 | <a id="copy_to_directory-include_srcs_packages"></a>include_srcs_packages |  List of Bazel packages (with glob support) to include in output directory.<br><br>Files in srcs are only copied to the output directory if the Bazel package of the file matches one of the patterns specified.<br><br>Forward slashes (`/`) should be used as path separators. A first character of `"."` will be replaced by the target's package path.<br><br>Defaults to `["**"]` which includes sources from all packages.<br><br>Files that have matching Bazel packages are subject to subsequent filters and transformations to determine if they are copied and what their path in the output directory will be.<br><br>Globs are supported (see rule docstring above).   | List of strings | optional |  `["**"]`  |
 | <a id="copy_to_directory-include_srcs_patterns"></a>include_srcs_patterns |  List of paths (with glob support) to include in output directory.<br><br>Files in srcs are only copied to the output directory if their output directory path, after applying `root_paths`, matches one of the patterns specified.<br><br>Forward slashes (`/`) should be used as path separators.<br><br>Defaults to `["**"]` which includes all sources.<br><br>Files that have matching output directory paths are subject to subsequent filters and transformations to determine if they are copied and what their path in the output directory will be.<br><br>Globs are supported (see rule docstring above).   | List of strings | optional |  `["**"]`  |
+| <a id="copy_to_directory-preserve_mtime"></a>preserve_mtime |  If True, the last modified time of copied files is preserved.   | Boolean | optional |  `False`  |
 | <a id="copy_to_directory-replace_prefixes"></a>replace_prefixes |  Map of paths prefixes (with glob support) to replace in the output directory path when copying files.<br><br>If the output directory path for a file starts with or fully matches a a key in the dict then the matching portion of the output directory path is replaced with the dict value for that key. The final path segment matched can be a partial match of that segment and only the matching portion will be replaced. If there are multiple keys that match, the longest match wins.<br><br>Forward slashes (`/`) should be used as path separators.<br><br>Replace prefix transformation are the final step in the list of filters and transformations. The final output path of a file being copied into the output directory is determined at this step.<br><br>Globs are supported (see rule docstring above).   | <a href="https://bazel.build/rules/lib/dict">Dictionary: String -> String</a> | optional |  `{}`  |
 | <a id="copy_to_directory-root_paths"></a>root_paths |  List of paths (with glob support) that are roots in the output directory.<br><br>If any parent directory of a file being copied matches one of the root paths patterns specified, the output directory path will be the path relative to the root path instead of the path relative to the file's workspace. If there are multiple root paths that match, the longest match wins.<br><br>Matching is done on the parent directory of the output file path so a trailing '**' glob patterm will match only up to the last path segment of the dirname and will not include the basename. Only complete path segments are matched. Partial matches on the last segment of the root path are ignored.<br><br>Forward slashes (`/`) should be used as path separators.<br><br>A `"."` value expands to the target's package path (`ctx.label.package`).<br><br>Defaults to `["."]` which results in the output directory path of files in the target's package and and sub-packages are relative to the target's package and files outside of that retain their full workspace relative paths.<br><br>Globs are supported (see rule docstring above).   | List of strings | optional |  `["."]`  |
 | <a id="copy_to_directory-verbose"></a>verbose |  If true, prints out verbose logs to stdout   | Boolean | optional |  `False`  |
@@ -70,7 +71,7 @@ for more information on supported globbing patterns.
 copy_to_directory_bin_action(<a href="#copy_to_directory_bin_action-ctx">ctx</a>, <a href="#copy_to_directory_bin_action-name">name</a>, <a href="#copy_to_directory_bin_action-dst">dst</a>, <a href="#copy_to_directory_bin_action-copy_to_directory_bin">copy_to_directory_bin</a>, <a href="#copy_to_directory_bin_action-files">files</a>, <a href="#copy_to_directory_bin_action-targets">targets</a>, <a href="#copy_to_directory_bin_action-root_paths">root_paths</a>,
                              <a href="#copy_to_directory_bin_action-include_external_repositories">include_external_repositories</a>, <a href="#copy_to_directory_bin_action-include_srcs_packages">include_srcs_packages</a>,
                              <a href="#copy_to_directory_bin_action-exclude_srcs_packages">exclude_srcs_packages</a>, <a href="#copy_to_directory_bin_action-include_srcs_patterns">include_srcs_patterns</a>, <a href="#copy_to_directory_bin_action-exclude_srcs_patterns">exclude_srcs_patterns</a>,
-                             <a href="#copy_to_directory_bin_action-replace_prefixes">replace_prefixes</a>, <a href="#copy_to_directory_bin_action-allow_overwrites">allow_overwrites</a>, <a href="#copy_to_directory_bin_action-hardlink">hardlink</a>, <a href="#copy_to_directory_bin_action-verbose">verbose</a>)
+                             <a href="#copy_to_directory_bin_action-replace_prefixes">replace_prefixes</a>, <a href="#copy_to_directory_bin_action-allow_overwrites">allow_overwrites</a>, <a href="#copy_to_directory_bin_action-hardlink">hardlink</a>, <a href="#copy_to_directory_bin_action-preserve_mtime">preserve_mtime</a>, <a href="#copy_to_directory_bin_action-verbose">verbose</a>)
 </pre>
 
 Factory function to copy files to a directory using a tool binary.
@@ -102,6 +103,7 @@ other rule implementations where additional_files can also be passed in.
 | <a id="copy_to_directory_bin_action-replace_prefixes"></a>replace_prefixes |  Map of paths prefixes to replace in the output directory path when copying files.<br><br>See copy_to_directory rule documentation for more details.   |  `{}` |
 | <a id="copy_to_directory_bin_action-allow_overwrites"></a>allow_overwrites |  If True, allow files to be overwritten if the same output file is copied to twice.<br><br>See copy_to_directory rule documentation for more details.   |  `False` |
 | <a id="copy_to_directory_bin_action-hardlink"></a>hardlink |  Controls when to use hardlinks to files instead of making copies.<br><br>See copy_to_directory rule documentation for more details.   |  `"auto"` |
+| <a id="copy_to_directory_bin_action-preserve_mtime"></a>preserve_mtime |  If true, preserve the modified time from the source.   |  `False` |
 | <a id="copy_to_directory_bin_action-verbose"></a>verbose |  If true, prints out verbose logs to stdout   |  `False` |
 
 

e2e/smoke/BUILD.bazel

@@ -99,3 +99,28 @@ bats_test(
         "basic.bats",
     ],
 )
+
+copy_to_directory(
+    name = "copy_to_directory_mtime_case",
+    srcs = ["d"],
+    out = "copy_to_directory_mtime_out",
+    preserve_mtime = True,
+)
+
+copy_directory(
+    name = "copy_directory_mtime_case",
+    src = "d",
+    out = "copy_directory_mtime_out",
+    preserve_mtime = True,
+)
+
+sh_test(
+    name = "test_preserve_mtime",
+    srcs = ["test_preserve_mtime.sh"],
+    data = [
+        "d",
+        ":copy_to_directory_mtime_case",
+        ":copy_directory_mtime_case",
+    ],
+    size = "small",
+)

e2e/smoke/test_preserve_mtime.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+function main {
+    compareMTimes d/1 copy_to_directory_mtime_out/d/1
+    compareMTimes d/1 copy_directory_mtime_out/1
+}
+
+function compareMTimes {
+    local originalFile="$1"
+    local copiedFile="$2"
+
+    local mtimeOriginal
+    mtimeOriginal="$(stat --dereference --format=%y "$originalFile")"
+
+    local mtimeCopy
+    mtimeCopy="$(stat --dereference --format=%y "$copiedFile")"
+
+    if [[ "$mtimeOriginal" != "$mtimeCopy" ]]; then
+        echo "Preserve mtime test failed. Modify times do not match for $originalFile and $copiedFile"
+        echo "  Original modify time: $mtimeOriginal"
+        echo "  Copied modify time:   $mtimeCopy"
+        return 1
+    fi
+
+    echo "Preserve mtime test passed for $originalFile and $copiedFile"
+}
+
+main "$@"

lib/private/copy_directory.bzl

@@ -12,7 +12,8 @@ def copy_directory_bin_action(
         dst,
         copy_directory_bin,
         hardlink = "auto",
-        verbose = False):
+        verbose = False,
+        preserve_mtime = False):
     """Factory function that creates an action to copy a directory from src to dst using a tool binary.
 
     The tool binary will typically be the `@aspect_bazel_lib//tools/copy_directory` `go_binary`
@@ -35,6 +36,8 @@ def copy_directory_bin_action(
             See copy_directory rule documentation for more details.
 
         verbose: If true, prints out verbose logs to stdout
+
+        preserve_mtime: If true, preserve the modified time from the source.
     """
     args = [
         src.path,
@@ -48,6 +51,9 @@ def copy_directory_bin_action(
     elif hardlink == "auto" and not src.is_source:
         args.append("--hardlink")
 
+    if preserve_mtime:
+        args.append("--preserve-mtime")
+
     ctx.actions.run(
         inputs = [src],
         outputs = [dst],
@@ -71,6 +77,7 @@ def _copy_directory_impl(ctx):
         copy_directory_bin = copy_directory_bin,
         hardlink = ctx.attr.hardlink,
         verbose = ctx.attr.verbose,
+        preserve_mtime = ctx.attr.preserve_mtime,
     )
 
     return [
@@ -93,6 +100,10 @@ _copy_directory = rule(
             default = "auto",
         ),
         "verbose": attr.bool(),
+        "preserve_mtime": attr.bool(
+            doc = "If True, the last modified time of copied files is preserved.",
+            default = False,
+        ),
         # use '_tool' attribute for development only; do not commit with this attribute active since it
         # propagates a dependency on rules_go which would be breaking for users
         # "_tool": attr.label(

lib/private/copy_to_directory.bzl

@@ -205,6 +205,7 @@ removed from sources files.
 - `off`: all files are copied
 - `on`: hardlinks are used for all files (not recommended)
     """,
+    "preserve_mtime": """If True, the last modified time of copied files is preserved.""",
     # verbose
     "verbose": """If true, prints out verbose logs to stdout""",
 }
@@ -251,6 +252,10 @@ _copy_to_directory_attr = {
         default = "auto",
         doc = _copy_to_directory_attr_doc["hardlink"],
     ),
+    "preserve_mtime": attr.bool(
+        default = False,
+        doc = _copy_to_directory_attr_doc["preserve_mtime"],
+    ),
     "verbose": attr.bool(
         doc = _copy_to_directory_attr_doc["verbose"],
     ),
@@ -285,6 +290,7 @@ def _copy_to_directory_impl(ctx):
         replace_prefixes = ctx.attr.replace_prefixes,
         allow_overwrites = ctx.attr.allow_overwrites,
         hardlink = ctx.attr.hardlink,
+        preserve_mtime = ctx.attr.preserve_mtime,
         verbose = ctx.attr.verbose,
     )
 
@@ -324,6 +330,7 @@ def copy_to_directory_bin_action(
         replace_prefixes = {},
         allow_overwrites = False,
         hardlink = "auto",
+        preserve_mtime = False,
         verbose = False):
     """Factory function to copy files to a directory using a tool binary.
 
@@ -382,6 +389,8 @@ def copy_to_directory_bin_action(
 
             See copy_to_directory rule documentation for more details.
 
+        preserve_mtime: If true, preserve the modified time from the source.
+
         verbose: If true, prints out verbose logs to stdout
     """
 
@@ -472,6 +481,7 @@ def copy_to_directory_bin_action(
         "include_srcs_patterns": include_srcs_patterns,
         "replace_prefixes": replace_prefixes,
         "root_paths": root_paths,
+        "preserve_mtime": preserve_mtime,
         "verbose": verbose,
     }
 

tools/common/copy.go

@@ -7,6 +7,7 @@ import (
 	"log"
 	"os"
 	"sync"
+	"time"
 )
 
 // From https://opensource.com/article/18/6/copying-files-go
@@ -27,7 +28,7 @@ func CopyFile(src string, dst string) error {
 }
 
 func Copy(opts CopyOpts) {
-	if !opts.info.Mode().IsRegular() {
+	if !opts.srcInfo.Mode().IsRegular() {
 		log.Fatalf("%s is not a regular file", opts.src)
 	}
 
@@ -79,6 +80,14 @@ func Copy(opts CopyOpts) {
 	if err != nil {
 		log.Fatal(err)
 	}
+
+	if opts.preserveMTime {
+		accessTime := time.Now()
+		err := os.Chtimes(opts.dst, accessTime, opts.srcInfo.ModTime())
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
 }
 
 type CopyWorker struct {
@@ -97,12 +106,13 @@ func (w *CopyWorker) Run(wg *sync.WaitGroup) {
 }
 
 type CopyOpts struct {
-	src, dst string
+	src, dst     string
-	info     fs.FileInfo
+	srcInfo      fs.FileInfo
-	hardlink bool
+	hardlink     bool
-	verbose  bool
+	verbose      bool
+	preserveMTime bool
 }
 
-func NewCopyOpts(src string, dst string, info fs.FileInfo, hardlink bool, verbose bool) CopyOpts {
+func NewCopyOpts(src string, dst string, srcInfo fs.FileInfo, hardlink bool, verbose bool, preserveMTime bool) CopyOpts {
-	return CopyOpts{src: src, dst: dst, info: info, hardlink: hardlink, verbose: verbose}
+	return CopyOpts{src: src, dst: dst, srcInfo: srcInfo, hardlink: hardlink, verbose: verbose, preserveMTime: preserveMTime}
 }

tools/copy_directory/main.go

@@ -16,6 +16,7 @@ type pathSet map[string]bool
 var srcPaths = pathSet{}
 var hardlink = false
 var verbose = false
+var preserveMTime = false
 
 type walker struct {
 	queue chan<- common.CopyOpts
@@ -68,13 +69,13 @@ func (w *walker) copyDir(src string, dst string) error {
 				return w.copyDir(linkPath, d)
 			} else {
 				// symlink points to a regular file
-				w.queue <- common.NewCopyOpts(linkPath, d, stat, hardlink, verbose)
+				w.queue <- common.NewCopyOpts(linkPath, d, stat, hardlink, verbose, preserveMTime)
 				return nil
 			}
 		}
 
 		// a regular file
-		w.queue <- common.NewCopyOpts(p, d, info, hardlink, verbose)
+		w.queue <- common.NewCopyOpts(p, d, info, hardlink, verbose, preserveMTime)
 		return nil
 	})
 }
@@ -83,7 +84,7 @@ func main() {
 	args := os.Args[1:]
 
 	if len(args) < 2 {
-		fmt.Println("Usage: copy_directory src dst [--hardlink] [--verbose]")
+		fmt.Println("Usage: copy_directory src dst [--hardlink] [--verbose] [--preserve-mtime]")
 		os.Exit(1)
 	}
 
@@ -96,6 +97,8 @@ func main() {
 				hardlink = true
 			} else if a == "--verbose" {
 				verbose = true
+			} else if a == "--preserve-mtime" {
+				preserveMTime = true
 			}
 		}
 	}

tools/copy_to_directory/main.go

@@ -40,6 +40,7 @@ type config struct {
 	IncludeSrcsPatterns         []string          `json:"include_srcs_patterns"`
 	ReplacePrefixes             map[string]string `json:"replace_prefixes"`
 	RootPaths                   []string          `json:"root_paths"`
+	PreserveMTime               bool              `json:"preserve_mtime"`
 	Verbose                     bool              `json:"verbose"`
 
 	ReplacePrefixesKeys []string
@@ -317,7 +318,7 @@ func (w *walker) copyPath(cfg *config, file fileInfo, outputPath string) error {
 
 	if !cfg.AllowOverwrites {
 		// if we don't allow overwrites then we can start copying as soon as a copy is calculated
-		w.queue <- common.NewCopyOpts(file.Path, outputPath, file.FileInfo, file.Hardlink, cfg.Verbose)
+		w.queue <- common.NewCopyOpts(file.Path, outputPath, file.FileInfo, file.Hardlink, cfg.Verbose, cfg.PreserveMTime)
 	}
 
 	return nil
@@ -419,7 +420,7 @@ func main() {
 		// if we allow overwrites then we must wait until all copy paths are calculated before starting
 		// any copy operations
 		for outputPath, file := range copySet {
-			queue <- common.NewCopyOpts(file.Path, outputPath, file.FileInfo, file.Hardlink, cfg.Verbose)
+			queue <- common.NewCopyOpts(file.Path, outputPath, file.FileInfo, file.Hardlink, cfg.Verbose, cfg.PreserveMTime)
 		}
 	}