cli: make paths to auto-track configurable, add jj track

It's a pretty frequent request to have support for turning off
auto-tracking of new files and to have a command to manually track
them instead. This patch adds a `snapshot.auto-track` config to decide
which paths to auto-track (defaults to `all()`). It also adds a `jj
track` command to manually track the untracked paths.

This patch does not include displaying the untracked paths in `jj
status`, so for now this is probably only useful in colocated repos
where you can run `git status` to find the untracked files.

#323

CHANGELOG.md

@@ -20,6 +20,13 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ### New features
 
+* The new config option `snapshot.auto-track` lets you automatically track only
+  the specified paths (all paths by default). Use the new `jj file track`
+  command to manually tracks path that were not automatically tracked. There is
+  no way to list untracked files yet. Use `git status` in a colocated workspace
+  as a workaround.
+  [#323](https://github.com/martinvonz/jj/issues/323)
+
 * `jj fix` now allows fixing unchanged files with the `--include-unchanged-files` flag. This
   can be used to more easily introduce automatic formatting changes in a new
   commit separate from other changes.

cli/src/cli_util.rs

@@ -878,6 +878,18 @@ impl WorkspaceCommandHelper {
         Ok(FilesetExpression::union_all(expressions))
     }
 
+    pub fn auto_tracking_matcher(&self) -> Result<Box<dyn Matcher>, CommandError> {
+        let pattern = self.settings().config().get_string("snapshot.auto-track")?;
+        let expression = fileset::parse(
+            &pattern,
+            &RepoPathUiConverter::Fs {
+                cwd: "".into(),
+                base: "".into(),
+            },
+        )?;
+        Ok(expression.to_matcher())
+    }
+
     pub(crate) fn path_converter(&self) -> &RepoPathUiConverter {
         &self.path_converter
     }
@@ -1316,6 +1328,7 @@ impl WorkspaceCommandHelper {
             return Ok(());
         };
         let base_ignores = self.base_ignores()?;
+        let auto_tracking_matcher = self.auto_tracking_matcher()?;
 
         // Compare working-copy tree and operation with repo's, and reload as needed.
         let fsmonitor_settings = self.settings().fsmonitor_settings()?;
@@ -1371,6 +1384,7 @@ See https://martinvonz.github.io/jj/latest/working-copy/#stale-working-copy \
             base_ignores,
             fsmonitor_settings,
             progress: progress.as_ref().map(|x| x as _),
+            start_tracking_matcher: &auto_tracking_matcher,
             max_new_file_size,
         })?;
         drop(progress);

cli/src/commands/file/mod.rs

@@ -15,6 +15,7 @@
 pub mod chmod;
 pub mod list;
 pub mod show;
+pub mod track;
 pub mod untrack;
 
 use crate::cli_util::CommandHelper;
@@ -27,6 +28,7 @@ pub enum FileCommand {
     Chmod(chmod::FileChmodArgs),
     List(list::FileListArgs),
     Show(show::FileShowArgs),
+    Track(track::FileTrackArgs),
     Untrack(untrack::FileUntrackArgs),
 }
 
@@ -39,6 +41,7 @@ pub fn cmd_file(
         FileCommand::Chmod(args) => chmod::cmd_file_chmod(ui, command, args),
         FileCommand::List(args) => list::cmd_file_list(ui, command, args),
         FileCommand::Show(args) => show::cmd_file_show(ui, command, args),
+        FileCommand::Track(args) => track::cmd_file_track(ui, command, args),
         FileCommand::Untrack(args) => untrack::cmd_file_untrack(ui, command, args),
     }
 }

cli/src/commands/file/track.rs

@@ -0,0 +1,68 @@
+// Copyright 2024 The Jujutsu Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::io::Write;
+
+use jj_lib::working_copy::SnapshotOptions;
+use tracing::instrument;
+
+use crate::cli_util::CommandHelper;
+use crate::command_error::CommandError;
+use crate::ui::Ui;
+
+/// Start tracking specified paths in the working copy
+///
+/// Without arguments, all paths that are not ignored will be tracked.
+///
+/// New files in the working copy can be automatically tracked.  
+/// You can configure which paths to automatically track by setting
+/// `snapshot.auto-track` (e.g. to `"none()"` or `"glob:**/*.rs"`). Files that
+/// don't match the pattern can be manually tracked using this command. The
+/// default pattern is `all()` and this command has no effect.
+#[derive(clap::Args, Clone, Debug)]
+pub(crate) struct FileTrackArgs {
+    /// Paths to track
+    #[arg(required = true, value_hint = clap::ValueHint::AnyPath)]
+    paths: Vec<String>,
+}
+
+#[instrument(skip_all)]
+pub(crate) fn cmd_file_track(
+    ui: &mut Ui,
+    command: &CommandHelper,
+    args: &FileTrackArgs,
+) -> Result<(), CommandError> {
+    let mut workspace_command = command.workspace_helper(ui)?;
+    let matcher = workspace_command
+        .parse_file_patterns(&args.paths)?
+        .to_matcher();
+
+    let mut tx = workspace_command.start_transaction().into_inner();
+    let base_ignores = workspace_command.base_ignores()?;
+    let (mut locked_ws, _wc_commit) = workspace_command.start_working_copy_mutation()?;
+    locked_ws.locked_wc().snapshot(&SnapshotOptions {
+        base_ignores,
+        fsmonitor_settings: command.settings().fsmonitor_settings()?,
+        progress: None,
+        start_tracking_matcher: &matcher,
+        max_new_file_size: command.settings().max_new_file_size()?,
+    })?;
+    let num_rebased = tx.repo_mut().rebase_descendants(command.settings())?;
+    if num_rebased > 0 {
+        writeln!(ui.status(), "Rebased {num_rebased} descendant commits")?;
+    }
+    let repo = tx.commit("track paths");
+    locked_ws.finish(repo.op_id().clone())?;
+    Ok(())
+}

cli/src/commands/file/untrack.rs

@@ -51,6 +51,7 @@ pub(crate) fn cmd_file_untrack(
 
     let mut tx = workspace_command.start_transaction().into_inner();
     let base_ignores = workspace_command.base_ignores()?;
+    let auto_tracking_matcher = workspace_command.auto_tracking_matcher()?;
     let (mut locked_ws, wc_commit) = workspace_command.start_working_copy_mutation()?;
     // Create a new tree without the unwanted files
     let mut tree_builder = MergedTreeBuilder::new(wc_commit.tree_id().clone());
@@ -72,6 +73,7 @@ pub(crate) fn cmd_file_untrack(
         base_ignores,
         fsmonitor_settings: command.settings().fsmonitor_settings()?,
         progress: None,
+        start_tracking_matcher: &auto_tracking_matcher,
         max_new_file_size: command.settings().max_new_file_size()?,
     })?;
     if wc_tree_id != *new_commit.tree_id() {

cli/src/config/misc.toml

@@ -24,3 +24,4 @@ edit = false
 
 [snapshot]
 max-new-file-size = "1MiB"
+auto-track = "all()"

cli/src/merge_tools/diff_working_copies.rs

@@ -12,6 +12,7 @@ use jj_lib::fsmonitor::FsmonitorSettings;
 use jj_lib::gitignore::GitIgnoreFile;
 use jj_lib::local_working_copy::TreeState;
 use jj_lib::local_working_copy::TreeStateError;
+use jj_lib::matchers::EverythingMatcher;
 use jj_lib::matchers::Matcher;
 use jj_lib::merged_tree::MergedTree;
 use jj_lib::merged_tree::TreeDiffEntry;
@@ -286,6 +287,7 @@ diff editing in mind and be a little inaccurate.
             base_ignores,
             fsmonitor_settings: FsmonitorSettings::None,
             progress: None,
+            start_tracking_matcher: &EverythingMatcher,
             max_new_file_size: u64::MAX,
         })?;
         Ok(output_tree_state.current_tree_id().clone())

cli/tests/cli-reference@.md.snap

@@ -40,6 +40,7 @@ This document contains the help content for the `jj` command-line program.
 * [`jj file chmod`↴](#jj-file-chmod)
 * [`jj file list`↴](#jj-file-list)
 * [`jj file show`↴](#jj-file-show)
+* [`jj file track`↴](#jj-file-track)
 * [`jj file untrack`↴](#jj-file-untrack)
 * [`jj fix`↴](#jj-fix)
 * [`jj git`↴](#jj-git)
@@ -739,6 +740,7 @@ File operations
 * `chmod` — Sets or removes the executable bit for paths in the repo
 * `list` — List files in a revision
 * `show` — Print contents of files in a revision
+* `track` — Start tracking specified paths in the working copy
 * `untrack` — Stop tracking specified paths in the working copy
 
 
@@ -809,6 +811,22 @@ If the given path is a directory, files in the directory will be visited recursi
 
 
 
+## `jj file track`
+
+Start tracking specified paths in the working copy
+
+Without arguments, all paths that are not ignored will be tracked.
+
+New files in the working copy can be automatically tracked. You can configure which paths to automatically track by setting `snapshot.auto-track` (e.g. to `"none()"` or `"glob:**/*.rs"`). Files that don't match the pattern can be manually tracked using this command. The default pattern is `all()` and this command has no effect.
+
+**Usage:** `jj file track <PATHS>...`
+
+###### **Arguments:**
+
+* `<PATHS>` — Paths to track
+
+
+
 ## `jj file untrack`
 
 Stop tracking specified paths in the working copy

cli/tests/runner.rs

@@ -30,7 +30,7 @@ mod test_edit_command;
 mod test_evolog_command;
 mod test_file_chmod_command;
 mod test_file_print_command;
-mod test_file_untrack_command;
+mod test_file_track_untrack_commands;
 mod test_fix_command;
 mod test_generate_md_cli_help;
 mod test_git_clone;

cli/tests/test_file_track_untrack_commands.rs

@@ -17,7 +17,7 @@ use std::path::PathBuf;
 use crate::common::TestEnvironment;
 
 #[test]
-fn test_untrack() {
+fn test_track_untrack() {
     let test_env = TestEnvironment::default();
     test_env.add_config(r#"ui.allow-init-native = true"#);
     test_env.jj_cmd_ok(test_env.env_root(), &["init", "repo"]);
@@ -103,7 +103,7 @@ fn test_untrack() {
 }
 
 #[test]
-fn test_untrack_sparse() {
+fn test_track_untrack_sparse() {
     let test_env = TestEnvironment::default();
     test_env.add_config(r#"ui.allow-init-native = true"#);
     test_env.jj_cmd_ok(test_env.env_root(), &["init", "repo"]);
@@ -128,4 +128,95 @@ fn test_untrack_sparse() {
     insta::assert_snapshot!(stdout, @r###"
     file1
     "###);
+    // Trying to manually track a file that's not included in the sparse working has
+    // no effect. TODO: At least a warning would be useful
+    let (stdout, stderr) = test_env.jj_cmd_ok(&repo_path, &["file", "track", "file2"]);
+    insta::assert_snapshot!(stdout, @"");
+    insta::assert_snapshot!(stderr, @"");
+    let stdout = test_env.jj_cmd_success(&repo_path, &["file", "list"]);
+    insta::assert_snapshot!(stdout, @r###"
+    file1
+    "###);
+}
+
+#[test]
+fn test_auto_track() {
+    let test_env = TestEnvironment::default();
+    test_env.add_config(r#"snapshot.auto-track = 'glob:*.rs'"#);
+    test_env.jj_cmd_ok(test_env.env_root(), &["git", "init", "repo"]);
+    let repo_path = test_env.env_root().join("repo");
+
+    std::fs::write(repo_path.join("file1.rs"), "initial").unwrap();
+    std::fs::write(repo_path.join("file2.md"), "initial").unwrap();
+    std::fs::write(repo_path.join("file3.md"), "initial").unwrap();
+
+    // Only configured paths get auto-tracked
+    let stdout = test_env.jj_cmd_success(&repo_path, &["file", "list"]);
+    insta::assert_snapshot!(stdout, @r###"
+    file1.rs
+    "###);
+
+    // Can manually track paths
+    let stdout = test_env.jj_cmd_success(&repo_path, &["file", "track", "file3.md"]);
+    insta::assert_snapshot!(stdout, @"");
+    let stdout = test_env.jj_cmd_success(&repo_path, &["file", "list"]);
+    insta::assert_snapshot!(stdout, @r###"
+    file1.rs
+    file3.md
+    "###);
+
+    // Can manually untrack paths
+    let stdout = test_env.jj_cmd_success(&repo_path, &["file", "untrack", "file3.md"]);
+    insta::assert_snapshot!(stdout, @"");
+    let stdout = test_env.jj_cmd_success(&repo_path, &["file", "list"]);
+    insta::assert_snapshot!(stdout, @r###"
+    file1.rs
+    "###);
+
+    // CWD-relative paths in `snapshot.auto-track` are evaluated from the repo root
+    let subdir = repo_path.join("sub");
+    std::fs::create_dir(&subdir).unwrap();
+    std::fs::write(subdir.join("file1.rs"), "initial").unwrap();
+    let stdout = test_env.jj_cmd_success(&subdir, &["file", "list"]);
+    insta::assert_snapshot!(stdout.replace('\\', "/"), @r###"
+    ../file1.rs
+    "###);
+
+    // But `jj file track` wants CWD-relative paths
+    let stdout = test_env.jj_cmd_success(&subdir, &["file", "track", "file1.rs"]);
+    insta::assert_snapshot!(stdout, @"");
+    let stdout = test_env.jj_cmd_success(&subdir, &["file", "list"]);
+    insta::assert_snapshot!(stdout.replace('\\', "/"), @r###"
+    ../file1.rs
+    file1.rs
+    "###);
+}
+
+#[test]
+fn test_track_ignored() {
+    let test_env = TestEnvironment::default();
+    test_env.add_config(r#"snapshot.auto-track = 'none()'"#);
+    test_env.jj_cmd_ok(test_env.env_root(), &["git", "init", "repo"]);
+    let repo_path = test_env.env_root().join("repo");
+
+    std::fs::write(repo_path.join(".gitignore"), "*.bak\n").unwrap();
+    std::fs::write(repo_path.join("file1"), "initial").unwrap();
+    std::fs::write(repo_path.join("file1.bak"), "initial").unwrap();
+
+    // Track an unignored path
+    let stdout = test_env.jj_cmd_success(&repo_path, &["file", "track", "file1"]);
+    insta::assert_snapshot!(stdout, @"");
+    let stdout = test_env.jj_cmd_success(&repo_path, &["file", "list"]);
+    insta::assert_snapshot!(stdout, @r###"
+    file1
+    "###);
+    // Track an ignored path
+    let stdout = test_env.jj_cmd_success(&repo_path, &["file", "track", "file1.bak"]);
+    insta::assert_snapshot!(stdout, @"");
+    // TODO: We should teach `jj file track` to track ignored paths (possibly
+    // requiring a flag)
+    let stdout = test_env.jj_cmd_success(&repo_path, &["file", "list"]);
+    insta::assert_snapshot!(stdout, @r###"
+    file1
+    "###);
 }

cli/tests/test_working_copy.rs

@@ -51,4 +51,9 @@ fn test_snapshot_large_file() {
       - Run `jj --config-toml 'snapshot.max-new-file-size=11264' st`
         This will increase the maximum file size allowed for new files, for this command only.
     "###);
+
+    // No error if we disable auto-tracking of the path
+    test_env.add_config(r#"snapshot.auto-track = 'none()'"#);
+    let stdout = test_env.jj_cmd_success(&repo_path, &["file", "list"]);
+    insta::assert_snapshot!(stdout, @"");
 }

docs/FAQ.md

@@ -101,6 +101,10 @@ To squash or split commits, use `jj squash` and `jj split`.
 
 ### How can I keep my scratch files in the repository without committing them?
 
+You can set `snapshot.auto-track` to only start tracking new files matching the
+configured pattern (e.g. `"none()"`). Changes to already tracked files will
+still be snapshotted by every command.
+
 You can keep your notes and other scratch files in the repository, if you add
 a wildcard pattern to either the repo's `gitignore` or your global `gitignore`.
 Something like `*.scratch` or `*.scratchpad` should do, after that rename the

docs/working-copy.md

@@ -12,12 +12,19 @@ working-copy contents when they have changed. Most `jj` commands you run will
 commit the working-copy changes if they have changed. The resulting revision
 will replace the previous working-copy revision.
 
-Also unlike most other VCSs, added files are implicitly tracked. That means that
-if you add a new file to the working copy, it will be automatically committed
-once you run e.g. `jj st`. Similarly, if you remove a file from the working
-copy, it will implicitly be untracked. To untrack a file while keeping it in
-the working copy, first make sure it's [ignored](#ignored-files) and then run
-`jj file untrack <path>`.
+Also unlike most other VCSs, added files are implicitly tracked by default. That
+means that if you add a new file to the working copy, it will be automatically
+committed once you run e.g. `jj st`. Similarly, if you remove a file from the
+working copy, it will implicitly be untracked.
+
+The `snapshot.auto-track` config option controls which paths get automatically
+tracked when they're added to the working copy. See the
+[fileset documentation](filesets.md) for the syntax. Files with paths matching
+[ignore files](#ignored-files) are never tracked automatically
+
+You can use `jj file untrack` to untrack a file while keeping it in the working
+copy. However, first [ignore](#ignored-files) them or remove them from the
+`snapshot.auto-track` patterns; otherwise they will be immediately tracked again.
 
 
 ## Conflicts
@@ -59,6 +66,14 @@ See https://git-scm.com/docs/gitignore for details about the format.
 `.gitignore` files are supported in any directory in the working copy, as well
 as in `$HOME/.gitignore` and `$GIT_DIR/info/exclude`.
 
+Ignored files are never tracked automatically (regardless of the value of
+`snapshot.auto-track`), but they can still end up being tracked for a few reasons:
+
+* if they were tracked in the parent commit
+* because of an explicit `jj file track` command
+
+You can untrack such files with the jj file untrack command.
+
 
 ## Workspaces
 

lib/src/local_working_copy.rs

@@ -797,6 +797,7 @@ impl TreeState {
             base_ignores,
             fsmonitor_settings,
             progress,
+            start_tracking_matcher,
             max_new_file_size,
         } = options;
 
@@ -834,6 +835,7 @@ impl TreeState {
             };
             self.visit_directory(
                 &matcher,
+                start_tracking_matcher,
                 &current_tree,
                 tree_entries_tx,
                 file_states_tx,
@@ -907,6 +909,7 @@ impl TreeState {
     fn visit_directory(
         &self,
         matcher: &dyn Matcher,
+        start_tracking_matcher: &dyn Matcher,
         current_tree: &MergedTree,
         tree_entries_tx: Sender<(RepoPathBuf, MergedTreeValue)>,
         file_states_tx: Sender<(RepoPathBuf, FileState)>,
@@ -963,7 +966,12 @@ impl TreeState {
 
                 if file_type.is_dir() {
                     let file_states = file_states.prefixed(&path);
-                    if git_ignore.matches(&path.to_internal_dir_string()) {
+                    if git_ignore.matches(&path.to_internal_dir_string())
+                        || start_tracking_matcher.visit(&path).is_nothing()
+                    {
+                        // TODO: Report this directory to the caller if there are unignored paths we
+                        // should not start tracking.
+
                         // If the whole directory is ignored, visit only paths we're already
                         // tracking.
                         for (tracked_path, current_file_state) in file_states {
@@ -1016,6 +1024,7 @@ impl TreeState {
                         };
                         self.visit_directory(
                             matcher,
+                            start_tracking_matcher,
                             current_tree,
                             tree_entries_tx.clone(),
                             file_states_tx.clone(),
@@ -1033,8 +1042,12 @@ impl TreeState {
                         && git_ignore.matches(path.as_internal_file_string())
                     {
                         // If it wasn't already tracked and it matches
-                        // the ignored paths, then
-                        // ignore it.
+                        // the ignored paths, then ignore it.
+                    } else if maybe_current_file_state.is_none()
+                        && !start_tracking_matcher.matches(&path)
+                    {
+                        // Leave the file untracked
+                        // TODO: Report this path to the caller
                     } else {
                         let metadata = entry.metadata().map_err(|err| SnapshotError::Other {
                             message: format!("Failed to stat file {}", entry.path().display()),
@@ -1042,6 +1055,7 @@ impl TreeState {
                         })?;
                         if maybe_current_file_state.is_none() && metadata.len() > max_new_file_size
                         {
+                            // TODO: Maybe leave the file untracked instead
                             return Err(SnapshotError::NewFileTooLarge {
                                 path: entry.path().clone(),
                                 size: HumanByteSize(metadata.len()),

lib/src/working_copy.rs

@@ -28,6 +28,8 @@ use crate::commit::Commit;
 use crate::fsmonitor::FsmonitorSettings;
 use crate::gitignore::GitIgnoreError;
 use crate::gitignore::GitIgnoreFile;
+use crate::matchers::EverythingMatcher;
+use crate::matchers::Matcher;
 use crate::op_store::OperationId;
 use crate::op_store::WorkspaceId;
 use crate::repo_path::RepoPath;
@@ -194,6 +196,9 @@ pub struct SnapshotOptions<'a> {
     pub fsmonitor_settings: FsmonitorSettings,
     /// A callback for the UI to display progress.
     pub progress: Option<&'a SnapshotProgress<'a>>,
+    /// For new files that are not already tracked, start tracking them if they
+    /// match this.
+    pub start_tracking_matcher: &'a dyn Matcher,
     /// The size of the largest file that should be allowed to become tracked
     /// (already tracked files are always snapshotted). If there are larger
     /// files in the working copy, then `LockedWorkingCopy::snapshot()` may
@@ -209,6 +214,7 @@ impl SnapshotOptions<'_> {
             base_ignores: GitIgnoreFile::empty(),
             fsmonitor_settings: FsmonitorSettings::None,
             progress: None,
+            start_tracking_matcher: &EverythingMatcher,
             max_new_file_size: u64::MAX,
         }
     }