Merge virtualfilesystem hook

Add virtual file system settings and hook proc.  On index load,
clear/set the skip worktree bits based on the virtual file system data.
Use virtual file system data to update skip-worktree bit in
unpack-trees. Use virtual file system data to exclude files and folders
not explicitly requested.

The hook was first contributed in private, but was extended via the
following pull requests:

	#15
	#27
	git#33
	git#70

Signed-off-by: Ben Peart <Ben.Peart@microsoft.com>
Signed-off-by: Derrick Stolee <dstolee@microsoft.com>

Author: dscho

Documentation/config/core.txt

@@ -111,6 +111,14 @@ Version 2 uses an opaque string so that the monitor can return
 something that can be used to determine what files have changed
 without race conditions.
 
+core.virtualFilesystem::
+	If set, the value of this variable is used as a command which
+	will identify all files and directories that are present in
+	the working directory.  Git will only track and update files
+	listed in the virtual file system.  Using the virtual file system
+	will supersede the sparse-checkout settings which will be ignored.
+	See the "virtual file system" section of linkgit:githooks[5].
+
 core.trustctime::
 	If false, the ctime differences between the index and the
 	working tree are ignored; useful when the inode change time

Documentation/githooks.txt

@@ -758,6 +758,26 @@ and "0" meaning they were not.
 Only one parameter should be set to "1" when the hook runs.  The hook
 running passing "1", "1" should not be possible.
 
+virtualFilesystem
+~~~~~~~~~~~~~~~~~~
+
+"Virtual File System" allows populating the working directory sparsely.
+The projection data is typically automatically generated by an external
+process.  Git will limit what files it checks for changes as well as which
+directories are checked for untracked files based on the path names given.
+Git will also only update those files listed in the projection.
+
+The hook is invoked when the configuration option core.virtualFilesystem
+is set.  It takes one argument, a version (currently 1).
+
+The hook should output to stdout the list of all files in the working
+directory that git should track.  The paths are relative to the root
+of the working directory and are separated by a single NUL.  Full paths
+('dir1/a.txt') as well as directories are supported (ie 'dir1/').
+
+The exit status determines whether git will use the data from the
+hook.  On error, git will abort the command with an error message.
+
 SEE ALSO
 --------
 linkgit:git-hook[1]

Makefile

@@ -1201,6 +1201,7 @@ LIB_OBJS += utf8.o
 LIB_OBJS += varint.o
 LIB_OBJS += version.o
 LIB_OBJS += versioncmp.o
+LIB_OBJS += virtualfilesystem.o
 LIB_OBJS += walker.o
 LIB_OBJS += wildmatch.o
 LIB_OBJS += worktree.o

config.c

@@ -1649,7 +1649,11 @@ int git_default_core_config(const char *var, const char *value,
 	}
 
 	if (!strcmp(var, "core.sparsecheckout")) {
-		core_apply_sparse_checkout = git_config_bool(var, value);
+		/* virtual file system relies on the sparse checkout logic so force it on */
+		if (core_virtualfilesystem)
+			core_apply_sparse_checkout = 1;
+		else
+			core_apply_sparse_checkout = git_config_bool(var, value);
 		return 0;
 	}
 
@@ -2740,6 +2744,44 @@ int repo_config_get_max_percent_split_change(struct repository *r)
 	return -1; /* default value */
 }
 
+int repo_config_get_virtualfilesystem(struct repository *r)
+{
+	/* Run only once. */
+	static int virtual_filesystem_result = -1;
+	if (virtual_filesystem_result >= 0)
+		return virtual_filesystem_result;
+
+	if (repo_config_get_pathname(r, "core.virtualfilesystem", &core_virtualfilesystem))
+		core_virtualfilesystem = getenv("GIT_VIRTUALFILESYSTEM_TEST");
+
+	if (core_virtualfilesystem && !*core_virtualfilesystem)
+		core_virtualfilesystem = NULL;
+
+	if (core_virtualfilesystem) {
+		/*
+		 * Some git commands spawn helpers and redirect the index to a different
+		 * location.  These include "difftool -d" and the sequencer
+		 * (i.e. `git rebase -i`, `git cherry-pick` and `git revert`) and others.
+		 * In those instances we don't want to update their temporary index with
+		 * our virtualization data.
+		 */
+		char *default_index_file = xstrfmt("%s/%s", the_repository->gitdir, "index");
+		int should_run_hook = !strcmp(default_index_file, the_repository->index_file);
+
+		free(default_index_file);
+		if (should_run_hook) {
+			/* virtual file system relies on the sparse checkout logic so force it on */
+			core_apply_sparse_checkout = 1;
+			virtual_filesystem_result = 1;
+			return 1;
+		}
+		core_virtualfilesystem = NULL;
+	}
+
+	virtual_filesystem_result = 0;
+	return 0;
+}
+
 int repo_config_get_index_threads(struct repository *r, int *dest)
 {
 	int is_bool, val;

config.h

@@ -684,6 +684,8 @@ int repo_config_get_index_threads(struct repository *r, int *dest);
 int repo_config_get_split_index(struct repository *r);
 int repo_config_get_max_percent_split_change(struct repository *r);
 
+int repo_config_get_virtualfilesystem(struct repository *r);
+
 /* This dies if the configured or default date is in the future */
 int repo_config_get_expiry(struct repository *r, const char *key, char **output);
 

dir.c

@@ -10,6 +10,7 @@
 
 #include "git-compat-util.h"
 #include "abspath.h"
+#include "virtualfilesystem.h"
 #include "config.h"
 #include "convert.h"
 #include "dir.h"
@@ -1475,6 +1476,19 @@ enum pattern_match_result path_matches_pattern_list(
 	int result = NOT_MATCHED;
 	size_t slash_pos;
 
+	if (core_virtualfilesystem) {
+		/*
+		* The virtual file system data is used to prevent git from traversing
+		* any part of the tree that is not in the virtual file system.  Return
+		* 1 to exclude the entry if it is not found in the virtual file system,
+		* else fall through to the regular excludes logic as it may further exclude.
+		*/
+		if (*dtype == DT_UNKNOWN)
+			*dtype = resolve_dtype(DT_UNKNOWN, istate, pathname, pathlen);
+		if (is_excluded_from_virtualfilesystem(pathname, pathlen, *dtype) > 0)
+			return 1;
+	}
+
 	if (!pl->use_cone_patterns) {
 		pattern = last_matching_pattern_from_list(pathname, pathlen, basename,
 							dtype, pl, istate);
@@ -1819,8 +1833,22 @@ struct path_pattern *last_matching_pattern(struct dir_struct *dir,
 int is_excluded(struct dir_struct *dir, struct index_state *istate,
 		const char *pathname, int *dtype_p)
 {
-	struct path_pattern *pattern =
-		last_matching_pattern(dir, istate, pathname, dtype_p);
+	struct path_pattern *pattern;
+
+	if (core_virtualfilesystem) {
+		/*
+		* The virtual file system data is used to prevent git from traversing
+		* any part of the tree that is not in the virtual file system.  Return
+		* 1 to exclude the entry if it is not found in the virtual file system,
+		* else fall through to the regular excludes logic as it may further exclude.
+		*/
+		if (*dtype_p == DT_UNKNOWN)
+			*dtype_p = resolve_dtype(DT_UNKNOWN, istate, pathname, strlen(pathname));
+		if (is_excluded_from_virtualfilesystem(pathname, strlen(pathname), *dtype_p) > 0)
+			return 1;
+	}
+
+	pattern = last_matching_pattern(dir, istate, pathname, dtype_p);
 	if (pattern)
 		return pattern->flags & PATTERN_FLAG_NEGATIVE ? 0 : 1;
 	return 0;
@@ -2438,6 +2466,8 @@ static enum path_treatment treat_path(struct dir_struct *dir,
 						ignore_case);
 	if (dtype != DT_DIR && has_path_in_index)
 		return path_none;
+	if (is_excluded_from_virtualfilesystem(path->buf, path->len, dtype) > 0)
+		return path_excluded;
 
 	/*
 	 * When we are looking at a directory P in the working tree,
@@ -2642,6 +2672,8 @@ static void add_path_to_appropriate_result_list(struct dir_struct *dir,
 	/* add the path to the appropriate result list */
 	switch (state) {
 	case path_excluded:
+		if (is_excluded_from_virtualfilesystem(path->buf, path->len, DT_DIR) > 0)
+			break;
 		if (dir->flags & DIR_SHOW_IGNORED)
 			dir_add_name(dir, istate, path->buf, path->len);
 		else if ((dir->flags & DIR_SHOW_IGNORED_TOO) ||

environment.c

@@ -72,6 +72,7 @@ int core_apply_sparse_checkout;
 int core_sparse_checkout_cone;
 int sparse_expect_files_outside_of_patterns;
 int core_gvfs;
+char *core_virtualfilesystem;
 int merge_log_config = -1;
 int precomposed_unicode = -1; /* see probe_utf8_pathname_composition() */
 unsigned long pack_size_limit_cfg;

environment.h

@@ -171,6 +171,7 @@ extern unsigned long pack_size_limit_cfg;
 extern int max_allowed_tree_depth;
 
 extern int core_preload_index;
+extern char *core_virtualfilesystem;
 extern int core_gvfs;
 extern int precomposed_unicode;
 extern int protect_hfs;

hook.c

@@ -196,7 +196,7 @@ int run_hooks_opt(struct repository *r, const char *hook_name,
 		.hook_name = hook_name,
 		.options = options,
 	};
-	const char *const hook_path = find_hook(r, hook_name);
+	const char *hook_path = find_hook(r, hook_name);
 	int ret = 0;
 	const struct run_process_parallel_opts opts = {
 		.tr2_category = "hook",
@@ -212,6 +212,18 @@ int run_hooks_opt(struct repository *r, const char *hook_name,
 		.data = &cb_data,
 	};
 
+	/*
+	 * Backwards compatibility hack in VFS for Git: when originally
+	 * introduced (and used!), it was called `post-indexchanged`, but this
+	 * name was changed during the review on the Git mailing list.
+	 *
+	 * Therefore, when the `post-index-change` hook is not found, let's
+	 * look for a hook with the old name (which would be found in case of
+	 * already-existing checkouts).
+	 */
+	if (!hook_path && !strcmp(hook_name, "post-index-change"))
+		hook_path = find_hook(r, "post-indexchanged");
+
 	if (!options)
 		BUG("a struct run_hooks_opt must be provided to run_hooks");
 

read-cache.c

@@ -8,6 +8,7 @@
 
 #include "git-compat-util.h"
 #include "bulk-checkin.h"
+#include "virtualfilesystem.h"
 #include "config.h"
 #include "date.h"
 #include "diff.h"
@@ -1979,6 +1980,7 @@ static void post_read_index_from(struct index_state *istate)
 	tweak_untracked_cache(istate);
 	tweak_split_index(istate);
 	tweak_fsmonitor(istate);
+	apply_virtualfilesystem(istate);
 }
 
 static size_t estimate_cache_size_from_compressed(unsigned int entries)

sparse-index.c

@@ -267,7 +267,7 @@ static int add_path_to_index(const struct object_id *oid,
 	size_t len = base->len;
 
 	if (S_ISDIR(mode)) {
-		int dtype;
+		int dtype = DT_DIR;
 		size_t baselen = base->len;
 		if (!ctx->pl)
 			return READ_TREE_RECURSIVE;
@@ -391,7 +391,7 @@ void expand_index(struct index_state *istate, struct pattern_list *pl)
 		struct cache_entry *ce = istate->cache[i];
 		struct tree *tree;
 		struct pathspec ps;
-		int dtype;
+		int dtype = DT_UNKNOWN;
 
 		if (!S_ISSPARSEDIR(ce->ce_mode)) {
 			set_index_entry(full, full->cache_nr++, ce);
@@ -666,6 +666,7 @@ static void clear_skip_worktree_from_present_files_full(struct index_state *ista
 void clear_skip_worktree_from_present_files(struct index_state *istate)
 {
 	if (!core_apply_sparse_checkout ||
+	    core_virtualfilesystem ||
 	    sparse_expect_files_outside_of_patterns)
 		return;
 

t/t1090-sparse-checkout-scope.sh

@@ -108,9 +108,9 @@ test_expect_success 'in partial clone, sparse checkout only fetches needed blobs
 '
 
 test_expect_success 'checkout does not delete items outside the sparse checkout file' '
-	# The "sparse.expectfilesoutsideofpatterns" config will prevent the
+	# The "core.virtualfilesystem" config will prevent the
 	# SKIP_WORKTREE flag from being dropped on files present on-disk.
-	test_config sparse.expectfilesoutsideofpatterns true &&
+	test_config core.virtualfilesystem true &&
 
 	test_config core.gvfs 8 &&
 	git checkout -b outside &&